From a5b04bb7c853ad08668a9cb6c7c6eb97610abe38 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 00:06:45 -0400 Subject: [PATCH 01/94] perf(j2k): pack 8-bit RGB region rows in one vectorizable loop U8RowSink packs three 8-bit component rows with one indexed loop over RGB triples (round and clamp to 0..=255, as sample_as_u8 does for 8 bits) instead of a per-sample call that branches on the bit depth. Multi-threaded region decodes run this pack serially after the IDWT. Measured together with the byte-run reads of stuffed packet-header bits, on an 18-core ARM64 machine, 5 interleaved rounds, median of per-round ratios, on regions of the 18000 x 12000 PLT test image (where packet parsing is small): top-left -11%, middle -8.5% and bottom-right -9% on 18 threads, middle -6.6% on 6 and -1.3% on one. --- crates/j2k/src/decode/output/u8.rs | 35 +++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/crates/j2k/src/decode/output/u8.rs b/crates/j2k/src/decode/output/u8.rs index 0797d3aa..83c29acd 100644 --- a/crates/j2k/src/decode/output/u8.rs +++ b/crates/j2k/src/decode/output/u8.rs @@ -209,6 +209,29 @@ fn write_rgb_component_rows_u8( width: usize, height: usize, ) { + if planes[..3].iter().all(|plane| plane.bit_depth() == 8) { + // One indexed loop over whole pixels per row, which LLVM vectorizes. + for y in 0..height { + let row = y * width..(y + 1) * width; + let (red, green, blue) = ( + &planes[0].samples()[row.clone()], + &planes[1].samples()[row.clone()], + &planes[2].samples()[row], + ); + let pixels = out[y * stride..y * stride + width * 3] + .as_chunks_mut::<3>() + .0; + let len = pixels.len().min(red.len()).min(green.len()).min(blue.len()); + for x in 0..len { + pixels[x] = [ + sample_as_u8_depth8(red[x]), + sample_as_u8_depth8(green[x]), + sample_as_u8_depth8(blue[x]), + ]; + } + } + return; + } for y in 0..height { let row = y * width; let dst = &mut out[y * stride..y * stride + width * 3]; @@ -258,6 +281,16 @@ fn write_samples_as_u8(src: &[f32], bit_depth: u8, dst: &mut [u8]) { } } +/// [`sample_as_u8`] for an 8-bit sample. +#[expect( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + reason = "samples are rounded and clamped to the unsigned 8-bit output domain before conversion" +)] +fn sample_as_u8_depth8(sample: f32) -> u8 { + sample.round().clamp(0.0, f32::from(u8::MAX)) as u8 +} + #[expect( clippy::cast_possible_truncation, clippy::cast_sign_loss, @@ -266,7 +299,7 @@ fn write_samples_as_u8(src: &[f32], bit_depth: u8, dst: &mut [u8]) { fn sample_as_u8(sample: f32, bit_depth: u8) -> u8 { let rounded = sample.round(); if bit_depth == 8 { - return rounded.clamp(0.0, f32::from(u8::MAX)) as u8; + return sample_as_u8_depth8(sample); } let max_value = if bit_depth >= 16 { f32::from(u16::MAX) From b1c74b126eb981fc86ee99e96215a88b5f75b375 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 02:17:25 -0400 Subject: [PATCH 02/94] perf(j2k-native): stream classic region rows from all components' pool bands A parallel region decode on the row-streamed IDWT decoded one component's bands on the pool while that component's IDWT ran, then the next component's, and wrote component planes. At reduced resolutions each sub-band holds few bands, so one component's bands left most of the pool idle, and classic reduced-resolution regions decoded more slowly at 18 threads than before the band pool. Now a classic tile whose rows stream to a row sink runs the bands of all its components on one executor, and the IDWT advances the components together, a row of each at a time, handing the rows to the sink as the serial bands of `banded_tier1::decode_tile_rows` do. The region needs no component planes. - The components' rings and bands are planned together, and workers take the band the IDWT reads first across all components. - `streams_rows` accepts tiles that decode their bands on the pool, and the line buffer charged for lockstep rows covers them. - A tile the route declines decodes its bands serially; one stranded before any row streams does too. A failure reruns whole-window Tier-1 serially for the error, as before. - HTJ2K tiles keep component planes: their bands decode fast enough that the planes' inverse transform on the pool costs less than transforming rows on the calling thread. Measured with region decodes to RGB8 (5 interleaved rounds, median of per-round ratios; output byte-identical at 1, 6 and 18 threads) on regions of an 18000 x 12000 RGB image: classic with PLT markers, classic without, and HTJ2K. Against the parent: - Classic whole image at 1/16 scale: -29% at 6 threads, -53% at 18; at 1/8 scale: -9% and -47%. Against the commit before the band pool, the 18-thread rows are 20% and 30% faster. - Classic PLT 2048 x 2048 at 1/2 scale: -11% at 6 threads, -22% at 18. - Classic PLT 1024 x 1024: -3% to -5% at 6 threads, 0% to -5% at 18; without PLT -2%. - HTJ2K rows: -1.2% to +0.1%. - Peak heap: -29% to -59% on the classic rows at 6 and 18 threads, which hold no component planes; HTJ2K within +0.2%. --- crates/j2k-native/src/image/output_api.rs | 7 +- crates/j2k-native/src/j2c/decode.rs | 47 ++- .../j2k-native/src/j2c/decode/banded_tier1.rs | 58 +-- .../src/j2c/decode/parallel_bands.rs | 336 +++++++++++++++++- crates/j2k-native/src/j2c/decode/subband.rs | 2 +- .../src/j2c/decode/subband/tile_parallel.rs | 20 +- crates/j2k-native/src/tests/row_sink.rs | 143 +++++++- 7 files changed, 550 insertions(+), 63 deletions(-) diff --git a/crates/j2k-native/src/image/output_api.rs b/crates/j2k-native/src/image/output_api.rs index ec455c83..5adfd2d4 100644 --- a/crates/j2k-native/src/image/output_api.rs +++ b/crates/j2k-native/src/image/output_api.rs @@ -476,9 +476,10 @@ impl<'a> Image<'a> { /// [`Self::decode_region_components_for_integer_output_with_context`] /// that hands the region's samples to `sink` instead of returning - /// component planes. A plain region decode on one thread streams them as - /// they are finished and never holds the whole planes; any other decode - /// hands over its planes as one rectangle at the end. + /// component planes. A plain region decode streams them as they are + /// finished and never holds the whole planes, its Tier-1 bands decoded + /// serially or on the worker pool; any other decode hands over its planes + /// as one rectangle at the end. /// /// # Errors /// diff --git a/crates/j2k-native/src/j2c/decode.rs b/crates/j2k-native/src/j2c/decode.rs index 7c208723..b621632a 100644 --- a/crates/j2k-native/src/j2c/decode.rs +++ b/crates/j2k-native/src/j2c/decode.rs @@ -605,7 +605,8 @@ fn choose_region_idwt( /// Whether a region decode on the row-streamed IDWT can hand its samples to /// a row sink instead of component planes: every tile decodes its Tier-1 in -/// bands (see [`tile_bands`]), so all components can advance together; the +/// bands, serially (see [`tile_bands`]) or on the pool (see [`pool_bands`]), +/// so all components can advance together; the /// image and tiles have no offsets; there are one to four components; and /// each tile with an inverse MCT has three components or more, the first /// three with one wavelet, as `apply_inverse_mct_per_tile` requires. @@ -628,7 +629,8 @@ fn streams_rows( let infos = &tile.component_infos; infos.len() == components // The row-streamed IDWT already excludes hooks. - && tile_bands(header, tile, tile_ctx, false, parallelism) + && (tile_bands(header, tile, tile_ctx, false, parallelism) + || pool_bands(header, tile, tile_ctx, false, parallelism)) && infos .iter() .all(|component| !component.requires_exact_integer_decode()) @@ -891,13 +893,14 @@ fn prepare_region_storage( let clip_to_idwt_windows = clips_to_idwt_windows(tile_ctx, hooked); storage.banded_tier1 = storage.line_region_idwt && tile_bands(header, tile, tile_ctx, hooked, cpu_decode_parallelism); - storage.lockstep_lines = storage.banded_tier1 && tile_ctx.stream_rows; #[cfg(feature = "parallel")] { storage.parallel_bands = storage.line_region_idwt && !storage.banded_tier1 && parallel_bands::applies(header, tile, tile_ctx, hooked, cpu_decode_parallelism); } + storage.lockstep_lines = + (storage.banded_tier1 || storage.parallel_bands) && tile_ctx.stream_rows; build::allocate_deferred_coefficients( storage, header.skipped_resolution_levels, @@ -940,6 +943,26 @@ fn tile_bands( bands } +/// Whether `tile` decodes its bands on the worker pool ahead of this +/// decode's row-streamed IDWT (see [`parallel_bands::applies`]) and can +/// stream its rows from them (see [`parallel_bands::streams_rows`]). +fn pool_bands( + header: &Header<'_>, + tile: &Tile<'_>, + tile_ctx: &TileDecodeContext, + hooked: bool, + parallelism: CpuDecodeParallelism, +) -> bool { + #[cfg(feature = "parallel")] + return parallel_bands::applies(header, tile, tile_ctx, hooked, parallelism) + && parallel_bands::streams_rows(tile); + #[cfg(not(feature = "parallel"))] + { + let _ = (header, tile, tile_ctx, hooked, parallelism); + false + } +} + /// Where a decoded tile's samples go. enum TileOutput<'o, 'p> { /// The decode's component planes. @@ -974,6 +997,24 @@ fn decode_components<'a>( mut output: TileOutput<'_, '_>, ) -> Result<()> { if let TileOutput::Sink(sink) = output { + #[cfg(feature = "parallel")] + if storage.parallel_bands { + if parallel_bands::decode_tile_rows( + tile, + header, + tile_ctx, + storage, + sink, + profile_enabled, + profile_timings, + )? { + return Ok(()); + } + // The route declined the bands: decode them serially, which the + // pool's band slots also hold. + storage.parallel_bands = false; + storage.banded_tier1 = true; + } return banded_tier1::decode_tile_rows( tile, header, diff --git a/crates/j2k-native/src/j2c/decode/banded_tier1.rs b/crates/j2k-native/src/j2c/decode/banded_tier1.rs index d64209f4..581d4f3c 100644 --- a/crates/j2k-native/src/j2c/decode/banded_tier1.rs +++ b/crates/j2k-native/src/j2c/decode/banded_tier1.rs @@ -180,22 +180,7 @@ fn stream_rows<'a>( return Err(error); } }; - let mut output = RowOutput { - tile, - sink, - round_irreversible_output: tile_ctx.round_irreversible_output, - depths: core::array::from_fn(|index| { - tile_ctx - .channel_data - .get(index) - .map_or((0, false), |component| { - (component.bit_depth, component.signed) - }) - }), - copy, - region, - streamed: 0, - }; + let mut output = RowOutput::new(tile, sink, tile_ctx, copy, region); let mut output_samples = 0usize; let result = (|| { let rings_len = buffer.len() - scratch_len; @@ -281,7 +266,7 @@ fn plan_tile_lines( /// Take the context's ring buffer, grown to hold every plan's rings and /// `scratch_len` more samples, and return it with each plan's ring length. -fn take_line_buffer( +pub(super) fn take_line_buffer( tile_ctx: &mut TileDecodeContext, storage: &DecompositionStorage<'_>, line_plans: &[LinePlan], @@ -318,7 +303,7 @@ fn take_line_buffer( } /// Where a tile's reconstructed rows go: transformed, then to the sink. -struct RowOutput<'o, 't> { +pub(super) struct RowOutput<'o, 't> { tile: &'o Tile<'t>, sink: &'o mut dyn ComponentRowSink, round_irreversible_output: bool, @@ -328,16 +313,43 @@ struct RowOutput<'o, 't> { copy: IntRect, region: IntRect, /// Samples handed to the sink. - streamed: usize, + pub(super) streamed: usize, } -impl RowOutput<'_, '_> { +impl<'o, 't> RowOutput<'o, 't> { + /// The output of `tile`'s rows `copy`, in the coordinates of `region`, + /// to `sink`. + pub(super) fn new( + tile: &'o Tile<'t>, + sink: &'o mut dyn ComponentRowSink, + tile_ctx: &TileDecodeContext, + copy: IntRect, + region: IntRect, + ) -> Self { + Self { + tile, + sink, + round_irreversible_output: tile_ctx.round_irreversible_output, + depths: core::array::from_fn(|index| { + tile_ctx + .channel_data + .get(index) + .map_or((0, false), |component| { + (component.bit_depth, component.signed) + }) + }), + copy, + region, + streamed: 0, + } + } + /// Advance every component's reconstruction a row at a time, copying /// each row into `scratch`, one row per component, and hand them over. - fn write( + pub(super) fn write( &mut self, lines: &mut [LineReconstruction<'_>], - rows: &mut BandedRows<'_, '_>, + rows: &mut dyn SubBandRows, scratch: &mut [f32], ) -> Result<()> { let (copy, components) = (self.copy, lines.len()); @@ -386,7 +398,7 @@ impl RowOutput<'_, '_> { /// component at a time, in the order the planes path decodes them, and /// return the first error. If the whole windows don't fit the cap, or the /// rerun succeeds, keep the banded error. -fn whole_window_tile_error<'a>( +pub(super) fn whole_window_tile_error<'a>( tile: &Tile<'a>, header: &Header<'_>, tile_ctx: &mut TileDecodeContext, diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index 0be196c2..63df4f60 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -23,6 +23,12 @@ //! does. Workers fail in any order, so a failure reruns whole-window Tier-1 //! serially for the error the whole-window path reports. //! +//! A region decode that streams its rows to a sink runs every component's +//! bands on one executor instead, while the IDWT advances the components +//! together, a row of each at a time, as the serial bands of +//! `banded_tier1::decode_tile_rows` do. The pool then holds every +//! component's next bands at once, and the region needs no planes. +//! //! A running band always belongs to a worker that is still active. A wait //! that finds a running band with no worker active would otherwise last //! forever, so it gives up the pool instead, and the component decodes over @@ -35,19 +41,22 @@ use core::mem::size_of; use core::ops::Range; use std::sync::{Condvar, Mutex, MutexGuard}; -use super::banded_tier1; -use super::subband::{plan_band_route, routes_tile, BandRoute, BandRouting}; +use super::banded_tier1::{self, RowOutput}; +use super::subband::{ + plan_band_route, plan_band_route_beside, routes_tile, BandRoute, BandRouting, +}; use super::workspace::{TaskScratch, TaskWorkspace}; use super::{ - clips_to_idwt_windows, CpuDecodeParallelism, DecodeProfileTimings, DecompositionStorage, - Header, Result, Tile, TileDecodeContext, + clips_to_idwt_windows, record_workspace, CpuDecodeParallelism, DecodeAllocationBudget, + DecodeProfileTimings, DecompositionStorage, Header, Result, Tile, TileDecodeContext, }; use crate::error::{DecodeError, DecodingError, ValidationError}; use crate::j2c::build::{self, CodeBlockCoding}; use crate::j2c::codestream::WaveletTransform; -use crate::j2c::idwt::{ComponentLines, SubBandRows}; +use crate::j2c::idwt::{ComponentLines, LinePlan, LineReconstruction, SubBandRows}; use crate::j2c::rect::IntRect; use crate::profile; +use crate::ComponentRowSink; /// Sub-band rows of support on each side of the rows a level's lifting /// reads, in rows of that level, for ordering bands: a 9/7 lift reads two @@ -58,6 +67,22 @@ const SUPPORT_ROWS: i64 = 4; /// to finish it. const STRANDED_CHECK: std::time::Duration = std::time::Duration::from_millis(50); +/// Whether a tile whose bands decode on the pool (see [`applies`]) streams +/// its rows to a row sink, every component advancing together: when its +/// code-blocks are classic. One component's bands at a time then leave the +/// pool idle at reduced resolutions, where each sub-band holds few bands. +/// HTJ2K bands decode fast enough that its component planes, whose inverse +/// transform runs on the pool, cost less. +pub(super) fn streams_rows(tile: &Tile<'_>) -> bool { + tile.component_infos.iter().all(|component| { + !component + .coding_style + .parameters + .code_block_style + .uses_high_throughput_block_coding() + }) +} + /// Whether a tile's components decode their bands on the worker pool: it /// feeds the streamed IDWT, in stored windows clipped to it, under `Auto` /// with two threads or more, every component has a level to reconstruct, and @@ -229,7 +254,7 @@ fn decode_route( }; match route.coding() { CodeBlockCoding::Classic => run_bands( - route, + route.workers(), &rings, &bands, slots, @@ -243,7 +268,7 @@ fn decode_route( strand_once, ), CodeBlockCoding::HighThroughput => run_bands( - route, + route.workers(), &rings, &bands, slots, @@ -277,6 +302,291 @@ fn decode_route( } } +/// Decode every component of a tile whose rows stream to a sink: the bands +/// of all its components on the pool at once, while the streamed IDWT +/// advances the components together on this thread and hands their rows +/// over, as `banded_tier1::decode_tile_rows` does with serial bands. +/// Returns `false`, before anything is decoded or streamed, when the route +/// declines a component, or when a band was stranded before any row was +/// streamed; the tile's bands then decode serially. +#[expect( + clippy::too_many_lines, + reason = "the streamed tile keeps its plans, executor, rings, and failure handling together" +)] +pub(super) fn decode_tile_rows<'a>( + tile: &Tile<'a>, + header: &Header<'_>, + tile_ctx: &mut TileDecodeContext, + storage: &mut DecompositionStorage<'a>, + sink: &mut dyn ComponentRowSink, + profile_enabled: bool, + profile_timings: &mut DecodeProfileTimings, +) -> Result { + let stage_start = profile::profile_now(profile_enabled); + let components = tile.component_infos.len(); + let mut routes = Vec::new(); + routes + .try_reserve_exact(components) + .map_err(|_| DecodingError::HostAllocationFailed)?; + // Every component's route stays live beside the others' while the + // executor runs them all, so each is planned in the budget the last one + // left, as one budget. + let mut live = None; + for component_idx in 0..components { + match plan_band_route_beside(tile, component_idx, tile_ctx, storage, header, live) { + Ok(BandRouting::Route(route)) => { + live = Some(route.budget()); + routes.push(route); + } + Ok(BandRouting::Serial | BandRouting::WholeWindows) => return Ok(false), + // Planning can fail in a later band before earlier blocks decode. + Err(error) => { + // The whole-window rerun's budget leaves the plans out. + drop(routes); + return Err(banded_tier1::whole_window_tile_error( + tile, header, tile_ctx, storage, error, + )); + } + } + } + let (rings, bands, strip_starts, active_levels) = plan_tile_rings(&routes, storage, header)?; + let line_plans = plan_tile_lines(tile, header, tile_ctx, storage, &active_levels)?; + // The executor's records live beside every route's plan and scratch + // while the bands decode. + let mut live = match live { + Some(live) => live, + None => DecodeAllocationBudget::for_storage(storage)?, + }; + live.include_bytes(executor_bytes(&rings, &bands)?)?; + let empty = IntRect::from_xywh(0, 0, 0, 0); + let (copy, region) = line_plans + .first() + .map_or((empty, empty), |plan| (plan.copy, plan.region)); + let scratch_len = (copy.width() as usize) + .checked_mul(components) + .ok_or(ValidationError::ImageTooLarge)?; + let (mut buffer, ring_lens) = + banded_tier1::take_line_buffer(tile_ctx, storage, &line_plans, scratch_len)?; + let mut output = RowOutput::new(tile, sink, tile_ctx, copy, region); + let workers = routes + .iter() + .map(|route| route.workers()) + .max() + .unwrap_or(0); + // Every route plans the tile's one block coding (see `routes_tile`). + let coding = routes + .first() + .map_or(CodeBlockCoding::Classic, |route| route.coding()); + #[cfg(test)] + let strand_once = tile_ctx.strand_pool_band; + #[cfg(not(test))] + let strand_once = false; + let mut tier1_us = 0; + let mut output_samples = 0usize; + let mut coefficients = core::mem::take(&mut storage.coefficients); + let slots = split_slots(&rings, &bands, storage, &mut coefficients); + let result = slots.and_then(|slots| { + let rings_len = buffer.len() - scratch_len; + let (mut line_rings, scratch) = buffer.split_at_mut(rings_len); + let mut lines = Vec::new(); + lines + .try_reserve_exact(line_plans.len()) + .map_err(|_| DecodingError::HostAllocationFailed)?; + for (plan, len) in line_plans.iter().zip(&ring_lens) { + let (ring, tail) = core::mem::take(&mut line_rings).split_at_mut(*len); + line_rings = tail; + lines.push(plan.reconstruction(ring)?); + } + let run = |rows: &mut dyn SubBandRows| -> Result<()> { + let written = output.write(&mut lines, rows, scratch); + output_samples = lines.iter().map(LineReconstruction::output_samples).sum(); + written + }; + match coding { + CodeBlockCoding::Classic => run_bands( + workers, + &rings, + &bands, + slots, + &mut tile_ctx.classic_task_workspaces, + |strip, output, workspace, scratch| { + let (route, strip) = route_strip(&routes, &strip_starts, strip)?; + route.decode_classic(strip, output, workspace, scratch) + }, + run, + profile_enabled, + &mut tier1_us, + strand_once, + ), + CodeBlockCoding::HighThroughput => run_bands( + workers, + &rings, + &bands, + slots, + &mut tile_ctx.ht_task_workspaces, + |strip, output, workspace, scratch| { + let (route, strip) = route_strip(&routes, &strip_starts, strip)?; + route.decode_ht(strip, output, workspace, scratch) + }, + run, + profile_enabled, + &mut tier1_us, + strand_once, + ), + } + }); + storage.coefficients = coefficients; + // Each band zeroes its slot before it decodes, and the last bands leave + // theirs written. + storage.stale_coefficients = true; + tile_ctx.idwt_line_rings = buffer; + let streamed = output.streamed; + tile_ctx.streamed_samples = tile_ctx.streamed_samples.saturating_add(streamed); + tile_ctx.debug_counters.idwt_output_samples = tile_ctx + .debug_counters + .idwt_output_samples + .saturating_add(output_samples); + let elapsed = profile::elapsed_us(stage_start); + profile_timings.codeblock_us += tier1_us; + profile_timings.idwt_us += elapsed.saturating_sub(tier1_us); + if matches!(result, Ok(true)) { + for route in &routes { + route.record(tile_ctx, storage)?; + } + } + // Neither the serial bands' nor the whole-window rerun's budget includes + // the plans' live allocations. + drop(routes); + match result { + Ok(true) => { + tile_ctx.debug_counters.pool_band_components += components; + record_workspace(tile_ctx, storage, 0)?; + Ok(true) + } + Ok(false) => { + // A band was left running with no worker to finish it. + #[cfg(test)] + let simulated = tile_ctx.strand_pool_band; + #[cfg(not(test))] + let simulated = false; + debug_assert!(simulated, "a running band had no active worker"); + tile_ctx.debug_counters.pool_band_fallbacks += 1; + // Rows the sink holds can't be taken back. + if streamed > 0 { + return Err(DecodingError::CodeBlockDecodeFailure.into()); + } + Ok(false) + } + Err(error) => Err(banded_tier1::whole_window_tile_error( + tile, header, tile_ctx, storage, error, + )), + } +} + +/// The route of a band's `strip`, counted over every component's route from +/// `strip_starts`, and the strip in that route. +fn route_strip<'r, 'a>( + routes: &'r [Box>], + strip_starts: &[usize], + strip: usize, +) -> Result<(&'r BandRoute<'a>, usize)> { + let component = strip_starts + .partition_point(|start| *start <= strip) + .checked_sub(1) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let route = routes + .get(component) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + Ok((route, strip - strip_starts[component])) +} + +/// Every component's rings and bands, in component order, as one executor +/// takes them, with the first strip of each component's route among the +/// strips of all routes, and the levels each component reconstructs. +type TileRings = (Vec, Vec, Vec, Vec); + +fn plan_tile_rings( + routes: &[Box>], + storage: &DecompositionStorage<'_>, + header: &Header<'_>, +) -> Result { + let (mut rings, mut bands) = (Vec::new(), Vec::new()); + let mut strip_starts = Vec::new(); + strip_starts + .try_reserve_exact(routes.len()) + .map_err(|_| DecodingError::HostAllocationFailed)?; + let mut active_levels = Vec::new(); + active_levels + .try_reserve_exact(routes.len()) + .map_err(|_| DecodingError::HostAllocationFailed)?; + let mut strips = 0usize; + for (component_idx, route) in routes.iter().enumerate() { + let (component_rings, component_bands, levels) = + plan_rings(route, storage, header, component_idx)?; + let (ring_offset, band_offset) = (rings.len(), bands.len()); + rings + .try_reserve(component_rings.len()) + .map_err(|_| DecodingError::HostAllocationFailed)?; + bands + .try_reserve(component_bands.len()) + .map_err(|_| DecodingError::HostAllocationFailed)?; + rings.extend(component_rings.into_iter().map(|ring| Ring { + bands: ring.bands.start + band_offset..ring.bands.end + band_offset, + ..ring + })); + bands.extend(component_bands.into_iter().map(|band| Band { + strip: band.strip + strips, + ring: band.ring + ring_offset, + ..band + })); + strip_starts.push(strips); + strips += route.strips().count(); + active_levels.push(levels); + } + Ok((rings, bands, strip_starts, active_levels)) +} + +/// Plan every component's streamed reconstruction of its `active_levels`. +/// No plans when no component has rows to reconstruct; every component of +/// a 1x1-sampled tile reconstructs the same rows. +fn plan_tile_lines( + tile: &Tile<'_>, + header: &Header<'_>, + tile_ctx: &TileDecodeContext, + storage: &DecompositionStorage<'_>, + active_levels: &[usize], +) -> Result> { + let mut line_plans = Vec::new(); + line_plans + .try_reserve_exact(active_levels.len()) + .map_err(|_| DecodingError::HostAllocationFailed)?; + for (component_idx, (component_info, levels)) in + tile.component_infos.iter().zip(active_levels).enumerate() + { + if let Some(plan) = LinePlan::new( + storage, + tile_ctx, + component_idx, + *levels, + header, + component_info.wavelet_transform(), + )? { + line_plans.push(plan); + } + } + if line_plans.is_empty() { + return Ok(line_plans); + } + if line_plans.len() != tile.component_infos.len() + || line_plans + .iter() + .any(|plan| plan.copy != line_plans[0].copy || plan.region != line_plans[0].region) + { + return Err(DecodingError::CodeBlockDecodeFailure.into()); + } + Ok(line_plans) +} + /// The bytes the executor holds for `rings` and `bands`, besides the slots: /// the records, each band's status, slot, queue entry and decoded-list entry, /// and each ring's cursors and slot list. @@ -641,15 +951,15 @@ impl<'c, 'w, W> Shared<'_, 'c, 'w, W> { } } -/// Decode `bands` into `slots` on up to `route.workers()` pool threads -/// while `run` drives the IDWT through them on this thread, then decode the -/// bands it didn't read. Returns `false` when a band was stranded. +/// Decode `bands` into `slots` on up to `workers` pool threads while `run` +/// drives the IDWT through them on this thread, then decode the bands it +/// didn't read. Returns `false` when a band was stranded. #[expect( clippy::too_many_arguments, reason = "the executor takes the plan, its storage and its two drivers explicitly" )] fn run_bands( - route: &BandRoute<'_>, + workers: usize, rings: &[Ring], bands: &[Band], slots: Slots<'_>, @@ -662,7 +972,7 @@ fn run_bands( ) -> Result { #[cfg(not(test))] let _ = strand_once; - let workspaces_needed = route.workers() + 1; + let workspaces_needed = workers + 1; let mut workspaces: Vec<&mut W> = Vec::new(); workspaces .try_reserve_exact(workspaces_needed) @@ -705,7 +1015,7 @@ fn run_bands( let shared = Shared { rings, bands, - max_workers: route.workers(), + max_workers: workers, #[cfg(test)] strand_once: core::sync::atomic::AtomicBool::new(strand_once), state: Mutex::new(state), diff --git a/crates/j2k-native/src/j2c/decode/subband.rs b/crates/j2k-native/src/j2c/decode/subband.rs index a8405a23..6aea6dd7 100644 --- a/crates/j2k-native/src/j2c/decode/subband.rs +++ b/crates/j2k-native/src/j2c/decode/subband.rs @@ -44,7 +44,7 @@ pub(crate) use self::placement::BlockPlacement; pub(crate) use self::tile_parallel::MIN_BAND_WORK; #[cfg(feature = "parallel")] pub(in crate::j2c::decode) use self::tile_parallel::{ - plan_band_route, routes_tile, BandRoute, BandRouting, + plan_band_route, plan_band_route_beside, routes_tile, BandRoute, BandRouting, }; /// Decode the code-blocks of the tile's `components` into their sub-bands. diff --git a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs index 0d3b8f90..aaabfb16 100644 --- a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs +++ b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs @@ -1512,6 +1512,21 @@ pub(in crate::j2c::decode) fn plan_band_route<'a>( tile_ctx: &mut TileDecodeContext, storage: &mut DecompositionStorage<'a>, header: &Header<'_>, +) -> Result> { + plan_band_route_beside(tile, component_idx, tile_ctx, storage, header, None) +} + +/// [`plan_band_route`], charging the route to `live` when other routes stay +/// live beside it: the budget of the route planned last for the same +/// executor, which holds every earlier route's charges. Without it, the +/// route is charged to the decode's storage budget. +pub(in crate::j2c::decode) fn plan_band_route_beside<'a>( + tile: &Tile<'a>, + component_idx: usize, + tile_ctx: &mut TileDecodeContext, + storage: &mut DecompositionStorage<'a>, + header: &Header<'_>, + live: Option, ) -> Result> { let components = component_idx..component_idx + 1; let Some(coding) = tile_route_coding(tile, components.clone()) else { @@ -1533,7 +1548,10 @@ pub(in crate::j2c::decode) fn plan_band_route<'a>( ) { return Ok(declined(route)); } - let mut budget = DecodeAllocationBudget::for_storage(storage)?; + let mut budget = match live { + Some(live) => live, + None => DecodeAllocationBudget::for_storage(storage)?, + }; let mut plan = match plan_tile( tile, components, diff --git a/crates/j2k-native/src/tests/row_sink.rs b/crates/j2k-native/src/tests/row_sink.rs index 60bf21c9..79ac7b83 100644 --- a/crates/j2k-native/src/tests/row_sink.rs +++ b/crates/j2k-native/src/tests/row_sink.rs @@ -1,9 +1,10 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -//! A serial region decode on the streamed IDWT hands its rows to a row sink -//! as they are finished, with every component decoded together and no -//! component planes. Its samples must match the planes bit for bit, every -//! sample must arrive once, and it must hold less. +//! A region decode on the streamed IDWT hands its rows to a row sink as they +//! are finished, with every component decoded together and no component +//! planes, its bands decoded serially or on the worker pool. Its samples must +//! match the planes bit for bit, every sample must arrive once, and it must +//! hold less. use alloc::vec::Vec; @@ -69,6 +70,9 @@ struct Decoded { streamed: bool, channel_bytes: usize, peak_bytes: usize, + pool_components: usize, + decoded_blocks: usize, + skipped_blocks: usize, } fn decode_planes(image: &Image<'_>, roi: (u32, u32, u32, u32)) -> Decoded { @@ -101,12 +105,16 @@ fn decode_planes(image: &Image<'_>, roi: (u32, u32, u32, u32)) -> Decoded { if std::env::var_os("ROW_SINK_DEBUG").is_some() { std::eprintln!("planes {roi:?} {peak:?}"); } + let counters = context.tile_decode_context.debug_counters; Decoded { planes, depths, streamed: false, channel_bytes: peak.channels, peak_bytes: peak.total(), + pool_components: counters.pool_band_components, + decoded_blocks: counters.decoded_code_blocks, + skipped_blocks: counters.skipped_code_blocks, } } @@ -133,12 +141,16 @@ fn decode_rows( if std::env::var_os("ROW_SINK_DEBUG").is_some() { std::eprintln!("rows {roi:?} {peak:?}"); } + let counters = context.tile_decode_context.debug_counters; Decoded { planes: sink.planes, depths: sink.depths, streamed: context.tile_decode_context.stream_rows, channel_bytes: peak.channels, peak_bytes: peak.total(), + pool_components: counters.pool_band_components, + decoded_blocks: counters.decoded_code_blocks, + skipped_blocks: counters.skipped_code_blocks, } } @@ -235,22 +247,102 @@ fn row_sink_matches_component_planes_bit_for_bit() { rows.peak_bytes, planes.peak_bytes ); - // A parallel decode keeps its planes and hands them over at - // the end. + // A parallel decode streams its rows when its bands can + // decode on the pool, and otherwise hands over its planes at + // the end (see `pool_band_rows_match_component_planes_bit_for_bit`). let parallel = decode_rows(&image, roi, CpuDecodeParallelism::Auto); - #[cfg(feature = "parallel")] - if rayon::current_num_threads() > 1 { - assert!(!parallel.streamed, "{label}: parallel planes"); - } assert_eq!(parallel.planes, planes.planes, "{label}: parallel"); } } } } -/// Outcome of a serial region decode, through the planes or the row sink: -/// a hash of the samples, or the error. -fn outcome(bytes: &[u8], strict: bool, roi: (u32, u32, u32, u32), rows: bool) -> String { +/// With two threads or more, a region decode whose bands the route can +/// decode streams its rows too, with every component's bands on the pool at +/// once. Its rows must match the serial planes bit for bit, decode and skip +/// the same code-blocks, and hold no planes. +#[cfg(feature = "parallel")] +#[test] +fn pool_band_rows_match_component_planes_bit_for_bit() { + for threads in [2, 4, 12] { + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("build pool"); + for case in row_sink_cases() { + let pixels = noisy_pixels(case.components); + let bytes = encode( + &pixels, + WIDTH, + HEIGHT, + u16::from(case.components), + 8, + false, + &case.options, + ) + .expect("encode pool row fixture"); + let mut pool_components = 0; + for reduction in [0_u8, 1, 2] { + let image = + Image::new_with_reduction(&bytes, &DecodeSettings::default(), reduction) + .expect("parse pool row fixture"); + let (width, height) = (image.width(), image.height()); + let tile = TILE >> reduction; + for roi in [ + (0, 0, width, height), + ( + width / 20, + height / 20, + width - width / 10, + height - height / 10, + ), + (width / 3, height / 3, width / 7, height / 9), + (tile - 5, tile - 3, 11, 7), + (0, height - 9, width, 9), + (width - 1, height - 1, 1, 1), + ] { + let label = format!( + "{} threads={threads} reduction={reduction} roi={roi:?}", + case.label + ); + let planes = decode_planes(&image, roi); + let rows = + pool.install(|| decode_rows(&image, roi, CpuDecodeParallelism::Auto)); + assert_eq!(rows.planes, planes.planes, "{label}"); + assert_eq!(rows.depths, planes.depths, "{label}"); + // HT tiles decode their bands on the pool into component + // planes (see `parallel_bands::streams_rows`). + if rows.pool_components > 0 && !case.options.use_ht_block_coding { + assert!(rows.streamed, "{label}: streamed"); + assert!(rows.channel_bytes <= 256, "{label}: no planes"); + assert_eq!(rows.decoded_blocks, planes.decoded_blocks, "{label}"); + assert_eq!(rows.skipped_blocks, planes.skipped_blocks, "{label}"); + } + pool_components += rows.pool_components; + } + } + // As in `pool_bands_match_whole_window_tier1_bit_for_bit`: the + // route takes HT blocks and classic blocks of one arithmetic + // segment, and two threads reach every such case. + let routed = case.options.use_ht_block_coding || case.options.num_layers <= 1; + if !routed { + assert_eq!(pool_components, 0, "{}", case.label); + } else if threads == 2 { + assert!(pool_components > 0, "{} never used the pool", case.label); + } + } + } +} + +/// Outcome of a region decode, through the planes or the row sink: a hash +/// of the samples, or the error. +fn outcome( + bytes: &[u8], + strict: bool, + roi: (u32, u32, u32, u32), + rows: bool, + parallelism: CpuDecodeParallelism, +) -> String { let settings = DecodeSettings { strict, ..DecodeSettings::default() @@ -260,7 +352,7 @@ fn outcome(bytes: &[u8], strict: bool, roi: (u32, u32, u32, u32), rows: bool) -> Err(error) => return format!("parse {error:?}"), }; let mut context = DecoderContext::default(); - context.set_cpu_decode_parallelism(CpuDecodeParallelism::Serial); + context.set_cpu_decode_parallelism(parallelism); let hash = |planes: &[Vec]| { planes .iter() @@ -300,8 +392,14 @@ fn outcome(bytes: &[u8], strict: bool, roi: (u32, u32, u32, u32), rows: bool) -> fn row_sink_rejects_corrupt_code_blocks_as_component_planes_do() { // Small code-blocks give each component many bands, and the components // decode together, so two corrupt blocks often fail in a different order - // than the planes path decodes them. + // than the planes path decodes them. On the pool they also fail in any + // order across components. const SIZE: u32 = 48; + #[cfg(feature = "parallel")] + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(4) + .build() + .expect("build pool"); let mut state = 0x9e37_79b9_7f4a_7c15_u64; let mut next = move || { state ^= state << 13; @@ -340,12 +438,19 @@ fn row_sink_rejects_corrupt_code_blocks_as_component_planes_do() { for strict in [false, true] { let roi = [(0, 0, SIZE, SIZE), (30, 30, 18, 18), (5, 9, 13, 7)] [usize::try_from(next() % 3).expect("region index")]; - let planes = outcome(&mutant, strict, roi, false); - let rows = outcome(&mutant, strict, roi, true); - assert_eq!( - rows, planes, + let label = format!( "ht={use_ht_block_coding} iteration={iteration} strict={strict} roi={roi:?}" ); + let serial = CpuDecodeParallelism::Serial; + let planes = outcome(&mutant, strict, roi, false, serial); + let rows = outcome(&mutant, strict, roi, true, serial); + assert_eq!(rows, planes, "{label}"); + #[cfg(feature = "parallel")] + { + let auto = CpuDecodeParallelism::Auto; + let pool_rows = pool.install(|| outcome(&mutant, strict, roi, true, auto)); + assert_eq!(pool_rows, planes, "pool {label}"); + } failures += usize::from(planes.starts_with("decode")); } } From afab6a00b05412906d706b194b895769a2e62496 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 02:17:25 -0400 Subject: [PATCH 03/94] perf(j2k-native): synthesize streamed region rows with the SIMD row kernel The row-streamed IDWT interleaved each row's low- and high-pass samples into a row and lifted it with the scalar filter. It now runs the banded IDWT's SIMD horizontal synthesis on the stored band rows directly. The kernel applies the same operations in the same order, so the output is bit-identical. - A row whose band rows lie wholly inside their stored windows reads them in place; a band row wholly outside reads zeros. Rows that straddle a window edge, rows narrower than the SIMD kernel's minimum, and levels one row tall keep the scalar path. - The line rings carry one row of horizontal scratch for the widest level, charged with them. - `synthesize_row` takes the level's row geometry instead of the whole level, so the streamed rows can call it. A test compares one- and two-level streamed rows with the whole-window synthesis, bit for bit, for both wavelets, odd and even origins, and windows clipped on every side. Measured with region decodes to RGB8 (5 interleaved rounds, median of per-round ratios; output byte-identical at 1, 6 and 18 threads) on regions of an 18000 x 12000 RGB image: HTJ2K, classic with PLT markers, and classic without. Against the parent / against f33e40c6: - HTJ2K whole image at 1/4 scale: -52% / -53% at 6 and 18 threads; at 1/16 scale: -36% / -36% at 6 threads, -34% / -35% at 18. - HTJ2K 1024 x 1024: -29% / -30% at 6 threads, -27% / -28% at 18. - Classic PLT 1024 x 1024: -16% to -28% / -20% to -30% at 6 threads, -32% to -36% / -34% to -37% at 18. - Classic PLT 2048 x 2048 at 1/2 scale: 0% / -11% at 6 threads, -26% / -42% at 18. - Classic whole image at 1/8 and 1/16 scale: -1% to -2% / -10% to -30% at 6 threads, -6% to -15% / -54% to -56% at 18. - One thread, the serial row path the parent leaves alone, against f33e40c6: classic 1024 x 1024 -5% to -9%, 2048 x 2048 at 1/2 scale -4%, whole image at 1/16 scale -2%; HTJ2K 1024 x 1024 -27%, whole image at 1/8 scale -35%. --- crates/j2k-native/src/j2c/idwt/banded.rs | 69 +++++++- crates/j2k-native/src/j2c/idwt/lines.rs | 127 +++++++++++++- crates/j2k-native/src/j2c/idwt/tests.rs | 214 +++++++++++++++++++++++ 3 files changed, 397 insertions(+), 13 deletions(-) diff --git a/crates/j2k-native/src/j2c/idwt/banded.rs b/crates/j2k-native/src/j2c/idwt/banded.rs index afa8993e..2b3fc0c3 100644 --- a/crates/j2k-native/src/j2c/idwt/banded.rs +++ b/crates/j2k-native/src/j2c/idwt/banded.rs @@ -58,7 +58,7 @@ const RING_ROWS: usize = MAX_HALO_ROWS + 2; const MIN_STRIP_ROWS: usize = 64; /// Scratch elements for the two horizontally lifted bands of one row. -fn horizontal_scratch_elements(width: usize) -> Result { +pub(super) fn horizontal_scratch_elements(width: usize) -> Result { if width < MIN_SIMD_ROW_WIDTH { return Ok(0); } @@ -305,6 +305,15 @@ impl<'a> Level2d<'a> { } } + fn row_geometry(&self) -> RowGeometry { + RowGeometry { + width: self.width, + x0: self.x0, + transform: self.transform, + high_pass: self.high_pass, + } + } + /// This level with its LL band rows `first_row..` taken from `rows`. #[cfg(feature = "parallel")] fn with_ll_rows(mut self, rows: &'a [f32], first_row: usize) -> Self { @@ -319,6 +328,17 @@ impl<'a> Level2d<'a> { } } +/// What the horizontal synthesis of one row needs from its level: the row's +/// width, its first column, whose parity says whether it starts low-pass, the +/// transform, and the 9-7I high-pass scale. +#[derive(Clone, Copy)] +pub(super) struct RowGeometry { + pub(super) width: usize, + pub(super) x0: usize, + pub(super) transform: WaveletTransform, + pub(super) high_pass: f32, +} + pub(super) fn filter_2d_banded( input: IDWTInput<'_>, decomposition: &Decomposition, @@ -752,7 +772,16 @@ fn synthesize_level_row( }; match source { Some((first, second)) => { - synthesize_row(simd, level, first, second, row, low, high, row_scale); + synthesize_row( + simd, + level.row_geometry(), + first, + second, + row, + low, + high, + row_scale, + ); } // Lifting maps an all-zero row to zeros. None => row.fill(0.0), @@ -1498,7 +1527,39 @@ pub(in crate::j2c) fn lift_rows( } /// Rows narrower than this use the scalar interleaved `1D_SR`. -const MIN_SIMD_ROW_WIDTH: usize = 2 * SIMD_WIDTH; +pub(super) const MIN_SIMD_ROW_WIDTH: usize = 2 * SIMD_WIDTH; + +/// [`synthesize_row`] at the CPU's SIMD level, for callers outside a +/// dispatch: the horizontal synthesis of one row of `geometry` from its two +/// band rows, `first` holding the samples of column `x0`'s parity, written +/// interleaved into `row` and, for 9-7I, scaled by `row_scale`. `scratch` +/// holds at least [`horizontal_scratch_elements`] samples. +pub(super) fn synthesize_band_row( + geometry: RowGeometry, + first: &[f32], + second: &[f32], + row: &mut [f32], + scratch: &mut [f32], + row_scale: f32, +) { + let (low_len, high_len) = if geometry.x0.is_multiple_of(2) { + (first.len(), second.len()) + } else { + (second.len(), first.len()) + }; + let (low, rest) = scratch.split_at_mut(low_len + 2); + let high = &mut rest[..high_len + 2]; + dispatch!(Level::new(), simd => synthesize_row( + simd, + geometry, + first, + second, + row, + low, + high, + row_scale, + )); +} /// Horizontal `1D_SR` (F.3.6) of one row from its two band rows, written /// interleaved into `row`. @@ -1518,7 +1579,7 @@ const MIN_SIMD_ROW_WIDTH: usize = 2 * SIMD_WIDTH; #[inline(always)] fn synthesize_row( simd: S, - level: Level2d<'_>, + level: RowGeometry, first: &[f32], second: &[f32], row: &mut [f32], diff --git a/crates/j2k-native/src/j2c/idwt/lines.rs b/crates/j2k-native/src/j2c/idwt/lines.rs index d3f30b4a..3ff112a8 100644 --- a/crates/j2k-native/src/j2c/idwt/lines.rs +++ b/crates/j2k-native/src/j2c/idwt/lines.rs @@ -20,6 +20,9 @@ use alloc::vec::Vec; use super::super::codestream::WaveletTransform; use super::super::decode::DecompositionStorage; use super::super::rect::IntRect; +use super::banded::{ + horizontal_scratch_elements, synthesize_band_row, RowGeometry, MIN_SIMD_ROW_WIDTH, +}; use super::filter_common::periodic_symmetric_extension_left; use super::horizontal::filter_row; use super::model::CoefficientSource; @@ -37,17 +40,36 @@ pub(crate) const fn ring_rows(transform: WaveletTransform) -> usize { vertical_step_count(transform) + 2 } -/// Samples the rings of one component's levels hold together. +/// Samples the rings of one component's levels hold together, with the +/// scratch their rows' horizontal synthesis shares. pub(crate) fn ring_samples( windows: impl IntoIterator, transform: WaveletTransform, ) -> Result { - windows.into_iter().try_fold(0usize, |total, window| { - (window.width() as usize) - .checked_mul(ring_rows(transform)) - .and_then(|samples| total.checked_add(samples)) - .ok_or(ValidationError::ImageTooLarge.into()) - }) + let mut widest = 0; + let rings = windows.into_iter().try_fold(0usize, |total, window| { + widest = widest.max(window.width() as usize); + level_ring_samples(window, transform).and_then(|samples| { + total + .checked_add(samples) + .ok_or(ValidationError::ImageTooLarge.into()) + }) + })?; + rings + .checked_add(synthesis_scratch_samples(widest)?) + .ok_or(ValidationError::ImageTooLarge.into()) +} + +/// Samples the ring of a level with this window holds. +fn level_ring_samples(window: IntRect, transform: WaveletTransform) -> Result { + (window.width() as usize) + .checked_mul(ring_rows(transform)) + .ok_or(ValidationError::ImageTooLarge.into()) +} + +/// Scratch for the horizontal synthesis of a row of the widest level. +fn synthesis_scratch_samples(widest: usize) -> Result { + horizontal_scratch_elements(widest) } /// Where the streamed IDWT reads the rows of a component's stored sub-bands. @@ -128,11 +150,13 @@ impl LevelRows<'_> { } } -/// What every level reads besides its own LL input. +/// What every level reads besides its own LL input, and the scratch its +/// rows' horizontal synthesis shares. struct Inputs<'r> { rows: &'r mut dyn SubBandRows, ll_sub_band: usize, transform: WaveletTransform, + scratch: &'r mut [f32], } /// Reconstruct a component's active levels from its stored sub-bands, read @@ -174,6 +198,8 @@ pub(super) fn reconstruct_rows( /// advance together. pub(crate) struct LineReconstruction<'r> { levels: Vec>, + /// Scratch for the horizontal synthesis of one row at a time. + scratch: &'r mut [f32], ll_sub_band: usize, transform: WaveletTransform, /// The rows of the last level still to return. @@ -200,7 +226,7 @@ impl<'r> LineReconstruction<'r> { .map_err(|_| DecodingError::HostAllocationFailed)?; let mut rest = ring; for (window, sub_bands) in windows.iter().zip(sub_bands) { - let len = ring_samples([*window], transform)?; + let len = level_ring_samples(*window, transform)?; if window.is_empty() || rest.len() < len { return Err(DecodingError::CodeBlockDecodeFailure.into()); } @@ -214,6 +240,12 @@ impl<'r> LineReconstruction<'r> { finished: [0; 4], }); } + let widest = levels.iter().map(LevelRows::width).max().unwrap_or(0); + let scratch_len = synthesis_scratch_samples(widest)?; + if rest.len() < scratch_len { + return Err(DecodingError::CodeBlockDecodeFailure.into()); + } + let (scratch, _) = core::mem::take(&mut rest).split_at_mut(scratch_len); let window = levels[levels.len() - 1].window; let (rows, columns) = if copy.is_empty() { @@ -232,6 +264,7 @@ impl<'r> LineReconstruction<'r> { }; Ok(Self { levels, + scratch, ll_sub_band, transform, rows, @@ -252,6 +285,7 @@ impl<'r> LineReconstruction<'r> { rows, ll_sub_band: self.ll_sub_band, transform: self.transform, + scratch: &mut *self.scratch, }; finish_row(&mut self.levels, last, row, &mut inputs)?; Ok(Some((y, &self.levels[last].row(row)[self.columns.clone()]))) @@ -340,6 +374,54 @@ fn feed_row(levels: &mut [LevelRows<'_>], level: usize, inputs: &mut Inputs<'_>) let width = current.width(); let slot = current.slot(local); let target = &mut current.ring[slot * width..][..width]; + // The low band holds the even columns, the high band the odd ones. + let (low, high) = if y.is_multiple_of(2) { + (ll, band(sub_bands[0])) + } else { + (band(sub_bands[1]), band(sub_bands[2])) + }; + let low_len = (window.x1.div_ceil(2) - window.x0.div_ceil(2)) as usize; + // Rows of one sample keep their own scaling, and narrow rows and band + // rows their rectangles don't hold whole take the windowed path below. + let band_rows = (height > 1 && width >= MIN_SIMD_ROW_WIDTH) + .then(|| { + Some(( + stored_row(low, window.x0.div_ceil(2), low_len, band_y)?, + stored_row(high, window.x0 / 2, width - low_len, band_y)?, + )) + }) + .flatten(); + if let Some((low_row, high_row)) = band_rows { + let (first, second) = if window.x0.is_multiple_of(2) { + (low_row, high_row) + } else { + (high_row, low_row) + }; + let row_scale = if inputs.transform == WaveletTransform::Irreversible97 { + irreversible_row_scale( + window.y0, + local.is_multiple_of(2), + super::OPENJPEG_NORMALIZED_HIGH_PASS_F32, + ) + } else { + 1.0 + }; + synthesize_band_row( + RowGeometry { + width, + x0: window.x0 as usize, + transform: inputs.transform, + high_pass: super::OPENJPEG_NORMALIZED_HIGH_PASS_F32, + }, + first, + second, + target, + inputs.scratch, + row_scale, + ); + current.fed += 1; + return Ok(()); + } interleave_samples_roi( ll, band(sub_bands[0]), @@ -373,6 +455,33 @@ fn feed_row(levels: &mut [LevelRows<'_>], level: usize, inputs: &mut Inputs<'_>) Ok(()) } +/// Zero samples for band rows that lie wholly outside their stored +/// rectangles, as sub-bands without code-blocks in a row do. +static ZERO_ROW: [f32; 4096] = [0.0; 4096]; + +/// The `len` samples of row `band_y` of `source` from band column `band_x0`, +/// which read zero outside its rectangle as [`CoefficientSource::get`] does: +/// read in place when the rectangle holds them all, or zeros when it holds +/// none. `None` when it holds only some. +fn stored_row( + source: CoefficientSource<'_>, + band_x0: u32, + len: usize, + band_y: u32, +) -> Option<&[f32]> { + let rect = source.rect; + let band_x1 = u64::from(band_x0) + len as u64; + if band_y < rect.y0 || band_y >= rect.y1 || band_x1 <= u64::from(rect.x0) || band_x0 >= rect.x1 + { + return ZERO_ROW.get(..len); + } + if band_x0 < rect.x0 || band_x1 > u64::from(rect.x1) { + return None; + } + let start = (band_y - rect.y0) as usize * source.stride as usize + (band_x0 - rect.x0) as usize; + source.coefficients.get(start..start.checked_add(len)?) +} + /// Let every vertical step finish as many rows as the rows fed so far allow. fn advance_steps(level: &mut LevelRows<'_>, transform: WaveletTransform) { let height = level.height(); diff --git a/crates/j2k-native/src/j2c/idwt/tests.rs b/crates/j2k-native/src/j2c/idwt/tests.rs index 696ab5ff..58ef9102 100644 --- a/crates/j2k-native/src/j2c/idwt/tests.rs +++ b/crates/j2k-native/src/j2c/idwt/tests.rs @@ -627,3 +627,217 @@ fn banded_2d_matches_plain_2d_bit_for_bit() { } } } + +/// Sub-bands for the streamed IDWT, each stored in its own rectangle. +struct StoredBands { + bands: Vec<(Vec, IntRect)>, +} + +impl super::lines::SubBandRows for StoredBands { + fn prepare_row(&mut self, _sub_band: usize, _y: u32) -> Result<()> { + Ok(()) + } + + fn samples(&self, sub_band: usize) -> (&[f32], IntRect, u32) { + let (samples, rect) = &self.bands[sub_band]; + (samples, *rect, rect.width()) + } +} + +/// `rect` grown by `[left, top, right, bottom]` samples; negative values +/// shrink it. +#[expect( + clippy::cast_sign_loss, + clippy::cast_possible_truncation, + reason = "the fixture rectangles stay at small non-negative coordinates" +)] +fn adjusted(rect: IntRect, [left, top, right, bottom]: [i64; 4]) -> IntRect { + let at = |value: u32, delta: i64| (i64::from(value) + delta).max(0) as u32; + IntRect::from_ltrb( + at(rect.x0, -left), + at(rect.y0, -top), + at(rect.x1, right).max(at(rect.x0, -left)), + at(rect.y1, bottom).max(at(rect.y0, -top)), + ) +} + +/// The whole-window region IDWT of one level, as `roi::fill_level_roi` runs it. +fn whole_window_level( + sources: [CoefficientSource<'_>; 4], + window: IntRect, + transform: WaveletTransform, +) -> Vec { + let mut output = vec![0.0; (window.width() * window.height()) as usize]; + interleave_samples_roi( + sources[0], + sources[1], + sources[2], + sources[3], + &mut output, + window, + ); + super::horizontal::filter_horizontal_codestream(&mut output, window, transform); + super::vertical::filter_vertical_codestream(&mut output, window, transform); + output +} + +/// The streamed IDWT synthesizes a row from its two band rows in place with +/// the banded SIMD kernel when their stored rectangles hold them whole, and +/// otherwise through the windowed interleave. Every row must match the +/// whole-window region IDWT, which reads every sample through +/// `CoefficientSource::get` and filters with the scalar `filter_row`, bit for +/// bit: both transforms, both column and row parities, stored rectangles +/// wider than the window's bands and clipped on either side, sub-bands with +/// nothing stored, and a second level reading the first one's rows. +#[test] +#[expect(clippy::too_many_lines, reason = "one table-driven equivalence sweep")] +fn streamed_rows_match_the_whole_window_levels_bit_for_bit() { + use super::banded::packed_band_rects; + use super::lines::{reconstruct_rows, ring_samples}; + + let mut seed = 7_u64; + for transform in [ + WaveletTransform::Reversible53, + WaveletTransform::Irreversible97, + ] { + let integral = transform == WaveletTransform::Reversible53; + for (x0, y0) in [(0, 0), (1, 0), (0, 1), (1, 1), (5, 2), (8, 3)] { + for (width, height) in [(16, 9), (17, 20), (31, 2), (40, 33), (64, 12), (161, 40)] { + for clip in [ + [0, 0, 0, 0], + [2, 1, 3, 2], + [-2, 0, 0, 0], + [0, 0, -3, 0], + [-1, -1, -2, -1], + [3, 0, -1, 1], + [1, 0, 1, 0], + ] { + let label = format!( + "{transform:?} origin=({x0},{y0}) size={width}x{height} clip={clip:?}" + ); + let window = IntRect::from_xywh(x0, y0, width, height); + // A sub-band without code-blocks stores nothing and + // reads as zero: here HH, or LH with a clipped HL. + let empty = match clip { + [1, 0, 1, 0] => Some(3), + [-2, 0, 0, 0] => Some(2), + _ => None, + }; + let mut bands = Vec::new(); + for (index, rect) in packed_band_rects(window).into_iter().enumerate() { + let stored = if empty == Some(index) { + IntRect::from_xywh(0, 0, 0, 0) + } else { + adjusted(rect, clip) + }; + seed += 1; + let samples = pseudo_random_coefficients( + (stored.width() * stored.height()) as usize, + seed, + integral, + ); + bands.push((samples, stored)); + } + let sources: [CoefficientSource<'_>; 4] = core::array::from_fn(|index| { + let (samples, rect) = &bands[index]; + CoefficientSource::new(samples, *rect, rect.width()) + }); + let expected = whole_window_level(sources, window, transform); + + let mut rows = StoredBands { + bands: bands.clone(), + }; + let mut ring = vec![f32::NAN; ring_samples([window], transform).expect("ring")]; + let mut streamed = vec![f32::NAN; expected.len()]; + let mut output_samples = 0; + reconstruct_rows( + &mut rows, + &[[1, 2, 3]], + &[window], + 0, + transform, + &mut ring, + window, + &mut |y, samples| { + let start = ((y - window.y0) * width) as usize; + streamed[start..][..samples.len()].copy_from_slice(samples); + Ok(()) + }, + &mut output_samples, + ) + .expect("streamed level"); + assert_eq!(bits(&streamed), bits(&expected), "one level {label}"); + + // A second level whose LL is the first level's window, + // clipped or grown the same way. + let next = IntRect::from_xywh(2 * x0 + 1, 2 * y0, 2 * width - 3, 2 * height); + let next_bands = packed_band_rects(next); + let first = adjusted(next_bands[0], clip); + if first.width() < 2 || first.height() < 2 { + continue; + } + let stored = packed_band_rects(first) + .into_iter() + .chain(next_bands[1..].iter().map(|rect| adjusted(*rect, clip))); + let levels: Vec<(Vec, IntRect)> = stored + .map(|rect| { + seed += 1; + let len = (rect.width() * rect.height()) as usize; + (pseudo_random_coefficients(len, seed, integral), rect) + }) + .collect(); + let source = |index: usize| { + let (samples, rect) = &levels[index]; + CoefficientSource::new(samples, *rect, rect.width()) + }; + let coarse = whole_window_level( + [source(0), source(1), source(2), source(3)], + first, + transform, + ); + let expected = whole_window_level( + [ + CoefficientSource::new(&coarse, first, first.width()), + source(4), + source(5), + source(6), + ], + next, + transform, + ); + let mut rows = StoredBands { bands: levels }; + let windows = [first, next]; + let mut ring = + vec![f32::NAN; ring_samples(windows, transform).expect("two-level ring")]; + let copy = adjusted(next, [-1, -2, -2, -1]); + let mut streamed = vec![f32::NAN; expected.len()]; + reconstruct_rows( + &mut rows, + &[[1, 2, 3], [4, 5, 6]], + &windows, + 0, + transform, + &mut ring, + copy, + &mut |y, samples| { + let start = ((y - next.y0) * next.width() + copy.x0 - next.x0) as usize; + streamed[start..][..samples.len()].copy_from_slice(samples); + Ok(()) + }, + &mut output_samples, + ) + .expect("streamed levels"); + for y in copy.y0..copy.y1 { + let start = ((y - next.y0) * next.width() + copy.x0 - next.x0) as usize; + let range = start..start + copy.width() as usize; + assert_eq!( + bits(&streamed[range.clone()]), + bits(&expected[range]), + "two levels {label} row {y}" + ); + } + } + } + } + } +} From b87178ff88c60a6a4b81599546f6f6f2d7818d41 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 02:17:25 -0400 Subject: [PATCH 04/94] perf(j2k-native): reconstruct region components on the pool, pack RGB8 in place A tile streaming rows from pool bands reconstructed every component on the calling thread, a row at a time, transformed each row and handed it to the sink, which packed it. The calling thread's IDWT and pack were the decode's critical path once Tier-1 ran on the pool. - Rows reconstruct in batches of up to 256 rows, 256 Ki samples per component (`streamed_batch_rows`), charged with the line buffer. - Each component's reconstruction advances through its own view of the bands, on the pool, when the tile is HTJ2K, or classic with 12 pool threads or more; with fewer, the classic readers would hold threads the bands need, and one view on the calling thread reads every component. Each view takes a workspace from the bank, so the route sends that many fewer workers. - A single classic view that waits for a band a worker is decoding first decodes the next band nobody has started, counted as an active worker while it does. - Three 8-bit components pack straight into the sink's interleaved RGB8 rows when the new rustdoc-hidden `ComponentRowSink::rgb8_region` offers them, which the facade's 8-bit sink does for tight RGB8 output without alpha. Other sinks take one `write_rows` call per batch. - HTJ2K tiles now stream rows from pool bands too. - A view that waits on a band whose worker failed returns the shared error and leaves it for the other views. Tests compare pool rows with component planes at 2, 4 and 12 threads, corrupt code-blocks through several views, and the RGB8 destination with the sink's own pack, byte for byte. Measured with region decodes to RGB8 (5 interleaved rounds, median of per-round ratios; output byte-identical at 1, 6 and 18 threads) on regions of an 18000 x 12000 RGB image: HTJ2K, classic with PLT markers, and classic without. Against the parent / against f33e40c6: - HTJ2K whole image at 1/4 scale: -35% / -78% at 6 threads, -32% / -78% at 18; at 1/16 scale: -26% / -63% at 6, -30% / -62% at 18. - HTJ2K 1024 x 1024: -16% / -47% at 6 threads, -20% / -48% at 18. - Classic at 18 threads (a reader per component): PLT 1024 x 1024 -9% to -18% / -46% to -54%; whole image at 1/16 scale -12% / -63%; PLT 2048 x 2048 at 1/2 scale +1% / -43%; whole image at 1/8 scale -1% / -56%. - Classic at 6 threads (one reader): PLT 1024 x 1024 -5% to +1% / -23% to -33%; whole image at 1/8 scale -10% / -19%; at 1/16 scale +5% / -27%. - One thread decodes serially, unchanged. - Peak heap of the parallel rows: -14% to -66% against f33e40c6 (no component planes); at one thread unchanged. --- crates/j2k-native/src/color/mod.rs | 2 +- crates/j2k-native/src/color/output_planes.rs | 11 + crates/j2k-native/src/color/packing.rs | 2 +- crates/j2k-native/src/j2c/decode.rs | 6 +- .../j2k-native/src/j2c/decode/banded_tier1.rs | 155 +++++++++++++ .../src/j2c/decode/parallel_bands.rs | 215 +++++++++++++----- .../src/j2c/decode/subband/tile_parallel.rs | 4 +- crates/j2k-native/src/j2c/idwt.rs | 4 +- crates/j2k-native/src/j2c/idwt/roi.rs | 26 ++- crates/j2k-native/src/tests/row_sink.rs | 171 +++++++++++++- crates/j2k/src/decode/output/u8.rs | 15 ++ ...able-api-1.0.implementation-public-api.txt | 1 + 12 files changed, 539 insertions(+), 73 deletions(-) diff --git a/crates/j2k-native/src/color/mod.rs b/crates/j2k-native/src/color/mod.rs index 7e494a75..cb7e3b54 100644 --- a/crates/j2k-native/src/color/mod.rs +++ b/crates/j2k-native/src/color/mod.rs @@ -26,7 +26,7 @@ pub use output_planes::{ #[cfg(all(feature = "simd", target_arch = "x86_64"))] pub(crate) use packing::store_rgb8_x8; pub(crate) use packing::{ - interleave_and_convert_after_mct, interleave_and_convert_region, + interleave_and_convert_after_mct, interleave_and_convert_region, pack_rgb8, validate_interleaved_output_buffer, }; pub(crate) use palette::resolve_palette_indices; diff --git a/crates/j2k-native/src/color/output_planes.rs b/crates/j2k-native/src/color/output_planes.rs index 78c79bd9..dde051e3 100644 --- a/crates/j2k-native/src/color/output_planes.rs +++ b/crates/j2k-native/src/color/output_planes.rs @@ -208,6 +208,17 @@ impl<'a> ComponentPlane<'a> { pub trait ComponentRowSink { /// Receive the samples of one rectangle of the region. fn write_rows(&mut self, x: u32, y: u32, planes: &[ComponentPlane<'_>]); + + /// The sink's destination for a `width` x `height` region of three 8-bit + /// components, when it is interleaved RGB8 rows of `3 * width` bytes back + /// to back that the sink would fill from such planes by rounding and + /// saturating each sample. A decode may then pack its rows straight into + /// this buffer instead of calling [`Self::write_rows`]. The default, + /// `None`, always uses `write_rows`. + fn rgb8_region(&mut self, width: u32, height: u32) -> Option<&mut [u8]> { + let _ = (width, height); + None + } } /// Borrowed decoded component planes for an image. diff --git a/crates/j2k-native/src/color/packing.rs b/crates/j2k-native/src/color/packing.rs index f3041429..02c9079b 100644 --- a/crates/j2k-native/src/color/packing.rs +++ b/crates/j2k-native/src/color/packing.rs @@ -301,7 +301,7 @@ pub(crate) fn interleave_and_convert( } /// Pack 8-bit RGB planes as interleaved bytes, rounded and saturated. -fn pack_rgb8(planes: [&[f32]; 3], output: &mut [u8]) { +pub(crate) fn pack_rgb8(planes: [&[f32]; 3], output: &mut [u8]) { #[cfg(all(feature = "simd", target_arch = "x86_64"))] { use fearless_simd::{dispatch, Level}; diff --git a/crates/j2k-native/src/j2c/decode.rs b/crates/j2k-native/src/j2c/decode.rs index b621632a..311705d0 100644 --- a/crates/j2k-native/src/j2c/decode.rs +++ b/crates/j2k-native/src/j2c/decode.rs @@ -944,8 +944,7 @@ fn tile_bands( } /// Whether `tile` decodes its bands on the worker pool ahead of this -/// decode's row-streamed IDWT (see [`parallel_bands::applies`]) and can -/// stream its rows from them (see [`parallel_bands::streams_rows`]). +/// decode's row-streamed IDWT (see [`parallel_bands::applies`]). fn pool_bands( header: &Header<'_>, tile: &Tile<'_>, @@ -954,8 +953,7 @@ fn pool_bands( parallelism: CpuDecodeParallelism, ) -> bool { #[cfg(feature = "parallel")] - return parallel_bands::applies(header, tile, tile_ctx, hooked, parallelism) - && parallel_bands::streams_rows(tile); + return parallel_bands::applies(header, tile, tile_ctx, hooked, parallelism); #[cfg(not(feature = "parallel"))] { let _ = (header, tile, tile_ctx, hooked, parallelism); diff --git a/crates/j2k-native/src/j2c/decode/banded_tier1.rs b/crates/j2k-native/src/j2c/decode/banded_tier1.rs index 581d4f3c..a8795aaf 100644 --- a/crates/j2k-native/src/j2c/decode/banded_tier1.rs +++ b/crates/j2k-native/src/j2c/decode/banded_tier1.rs @@ -394,6 +394,161 @@ impl<'o, 't> RowOutput<'o, 't> { } } +impl RowOutput<'_, '_> { + /// [`Self::write`] through one reader per component, which may run on + /// different threads, `batch` rows at a time. Each component's + /// reconstruction advances `batch` rows into its own rows of `scratch`, + /// on the pool when there is a reader per component, then the rows are + /// transformed and handed over, packed straight into the sink's RGB8 + /// destination when it has one for three 8-bit components. + pub(super) fn write_batches( + &mut self, + lines: &mut [LineReconstruction<'_>], + readers: &mut [&mut (dyn SubBandRows + Send)], + scratch: &mut [f32], + batch: usize, + ) -> Result<()> { + let (copy, region, components) = (self.copy, self.region, lines.len()); + let width = copy.width() as usize; + if lines.is_empty() || width == 0 || batch == 0 { + return Ok(()); + } + let batch_len = width + .checked_mul(batch) + .ok_or(ValidationError::ImageTooLarge)?; + let rgb8 = components == 3 && self.depths[..3].iter().all(|(depth, _)| *depth == 8); + let mut y = copy.y0; + while y < copy.y1 { + let rows = ((copy.y1 - y) as usize).min(batch); + let mut parts: [&mut [f32]; 4] = Default::default(); + for (part, chunk) in parts.iter_mut().zip(scratch.chunks_exact_mut(batch_len)) { + *part = &mut chunk[..rows * width]; + } + let parts = &mut parts[..components]; + if parts.iter().any(|part| part.len() != rows * width) { + return Err(DecodingError::CodeBlockDecodeFailure.into()); + } + reconstruct_batch(lines, readers, parts, y, width)?; + for row in 0..rows { + let mut outputs: [&mut [f32]; 4] = Default::default(); + for (output, part) in outputs.iter_mut().zip(parts.iter_mut()) { + *output = &mut part[row * width..][..width]; + } + store::transform_streamed_rows( + self.tile, + self.round_irreversible_output, + &mut outputs[..components], + )?; + } + let (x, row_in_region) = (copy.x0 - region.x0, y - region.y0); + let destination = rgb8 + .then(|| self.sink.rgb8_region(region.width(), region.height())) + .flatten(); + if let Some(out) = destination { + let stride = region.width() as usize * 3; + for row in 0..rows { + let start = (row_in_region as usize + row) * stride + x as usize * 3; + let pixels = out + .get_mut(start..start + width * 3) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let at = row * width..(row + 1) * width; + crate::color::pack_rgb8( + [&parts[0][at.clone()], &parts[1][at.clone()], &parts[2][at]], + pixels, + ); + } + } else { + let component_rows: [ComponentPlane<'_>; 4] = + core::array::from_fn(|index| ComponentPlane { + samples: parts.get(index).map_or(&[][..], |part| &**part), + dimensions: (copy.width(), rows as u32), + bit_depth: self.depths[index].0, + signed: self.depths[index].1, + sampling: (1, 1), + }); + self.sink + .write_rows(x, row_in_region, &component_rows[..components]); + } + self.streamed = self.streamed.saturating_add(rows * width * components); + y += rows as u32; + } + Ok(()) + } +} + +/// Reconstruct the next `parts[c].len() / width` rows of each component `c`, +/// from row `first_y`, into `parts[c]`: on the pool, one component per task, +/// when there is a reader per component, otherwise all through the first. +fn reconstruct_batch( + lines: &mut [LineReconstruction<'_>], + readers: &mut [&mut (dyn SubBandRows + Send)], + parts: &mut [&mut [f32]], + first_y: u32, + width: usize, +) -> Result<()> { + if readers.len() < lines.len() || lines.len() < 2 { + let reader = readers + .first_mut() + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + for (line, part) in lines.iter_mut().zip(parts.iter_mut()) { + reconstruct_rows_into(line, &mut **reader, part, first_y, width)?; + } + return Ok(()); + } + let mut results: [Result<()>; 4] = core::array::from_fn(|_| Ok(())); + #[cfg(feature = "parallel")] + rayon::in_place_scope(|scope| { + let mut tasks = lines + .iter_mut() + .zip(readers.iter_mut()) + .zip(parts.iter_mut()) + .zip(results.iter_mut()); + let first = tasks.next(); + for (((line, reader), part), result) in tasks { + scope.spawn(move |_| { + *result = reconstruct_rows_into(line, &mut **reader, part, first_y, width); + }); + } + if let Some((((line, reader), part), result)) = first { + *result = reconstruct_rows_into(line, &mut **reader, part, first_y, width); + } + }); + #[cfg(not(feature = "parallel"))] + for (((line, reader), part), result) in lines + .iter_mut() + .zip(readers.iter_mut()) + .zip(parts.iter_mut()) + .zip(results.iter_mut()) + { + *result = reconstruct_rows_into(line, &mut **reader, part, first_y, width); + } + results.into_iter().collect() +} + +/// The next rows of `line`, from row `first_y`, through `rows`, into `out`, +/// `width` samples each. +fn reconstruct_rows_into( + line: &mut LineReconstruction<'_>, + rows: &mut dyn SubBandRows, + out: &mut [f32], + first_y: u32, + width: usize, +) -> Result<()> { + for (index, chunk) in out.chunks_exact_mut(width).enumerate() { + let (row_y, row) = line + .next_row(rows)? + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let expected = u32::try_from(index) + .ok() + .and_then(|index| first_y.checked_add(index)); + if Some(row_y) != expected || row.len() != width { + return Err(DecodingError::CodeBlockDecodeFailure.into()); + } + chunk.copy_from_slice(row); + } + Ok(()) +} + /// When a lockstep decode fails, rerun Tier-1 over whole windows one /// component at a time, in the order the planes path decodes them, and /// return the first error. If the whole windows don't fit the cap, or the diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index 63df4f60..9e3cf19b 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -53,7 +53,9 @@ use super::{ use crate::error::{DecodeError, DecodingError, ValidationError}; use crate::j2c::build::{self, CodeBlockCoding}; use crate::j2c::codestream::WaveletTransform; -use crate::j2c::idwt::{ComponentLines, LinePlan, LineReconstruction, SubBandRows}; +use crate::j2c::idwt::{ + streamed_batch_rows, ComponentLines, LinePlan, LineReconstruction, SubBandRows, +}; use crate::j2c::rect::IntRect; use crate::profile; use crate::ComponentRowSink; @@ -67,22 +69,6 @@ const SUPPORT_ROWS: i64 = 4; /// to finish it. const STRANDED_CHECK: std::time::Duration = std::time::Duration::from_millis(50); -/// Whether a tile whose bands decode on the pool (see [`applies`]) streams -/// its rows to a row sink, every component advancing together: when its -/// code-blocks are classic. One component's bands at a time then leave the -/// pool idle at reduced resolutions, where each sub-band holds few bands. -/// HTJ2K bands decode fast enough that its component planes, whose inverse -/// transform runs on the pool, cost less. -pub(super) fn streams_rows(tile: &Tile<'_>) -> bool { - tile.component_infos.iter().all(|component| { - !component - .coding_style - .parameters - .code_block_style - .uses_high_throughput_block_coding() - }) -} - /// Whether a tile's components decode their bands on the worker pool: it /// feeds the streamed IDWT, in stored windows clipped to it, under `Auto` /// with two threads or more, every component has a level to reconstruct, and @@ -249,12 +235,15 @@ fn decode_route( let mut coefficients = core::mem::take(&mut storage.coefficients); let slots = split_slots(&rings, &bands, storage, &mut coefficients); let result = slots.and_then(|slots| { - let run = |rows: &mut dyn SubBandRows| -> Result<()> { - lines.as_mut().map_or(Ok(()), |lines| lines.run(rows)) + let run = |rows: &mut [&mut (dyn SubBandRows + Send)]| -> Result<()> { + match (lines.as_mut(), rows.first_mut()) { + (Some(lines), Some(rows)) => lines.run(&mut **rows), + _ => Ok(()), + } }; match route.coding() { CodeBlockCoding::Classic => run_bands( - route.workers(), + (route.workers(), 1, false), &rings, &bands, slots, @@ -268,7 +257,7 @@ fn decode_route( strand_once, ), CodeBlockCoding::HighThroughput => run_bands( - route.workers(), + (route.workers(), 1, false), &rings, &bands, slots, @@ -362,21 +351,41 @@ pub(super) fn decode_tile_rows<'a>( let (copy, region) = line_plans .first() .map_or((empty, empty), |plan| (plan.copy, plan.region)); + // Rows reconstruct in batches, charged by the region's width (see + // `roi_idwt_peak_elements`). + let batch = streamed_batch_rows(region.width() as usize); let scratch_len = (copy.width() as usize) .checked_mul(components) + .and_then(|samples| samples.checked_mul(batch)) .ok_or(ValidationError::ImageTooLarge)?; let (mut buffer, ring_lens) = banded_tier1::take_line_buffer(tile_ctx, storage, &line_plans, scratch_len)?; let mut output = RowOutput::new(tile, sink, tile_ctx, copy, region); - let workers = routes - .iter() - .map(|route| route.workers()) - .max() - .unwrap_or(0); // Every route plans the tile's one block coding (see `routes_tile`). let coding = routes .first() .map_or(CodeBlockCoding::Classic, |route| route.coding()); + // Each component's reconstruction reads through its own view of the + // bands, with a workspace of its own from the bank the routes prepared + // for their workers and the calling thread. + let bank = match coding { + CodeBlockCoding::Classic => tile_ctx.classic_task_workspaces.len(), + CodeBlockCoding::HighThroughput => tile_ctx.ht_task_workspaces.len(), + }; + let route_workers = routes + .iter() + .map(|route| route.workers()) + .max() + .unwrap_or(0); + let readers = if parallel_synthesis(components, coding, bank, route_workers) { + components + } else { + 1 + }; + let workers = route_workers.min(bank.saturating_sub(readers)); + // One reader for every component of a classic tile leaves the pool's + // threads to Tier-1, and decodes bands itself while it waits. + let helps = readers == 1 && coding == CodeBlockCoding::Classic; #[cfg(test)] let strand_once = tile_ctx.strand_pool_band; #[cfg(not(test))] @@ -397,14 +406,14 @@ pub(super) fn decode_tile_rows<'a>( line_rings = tail; lines.push(plan.reconstruction(ring)?); } - let run = |rows: &mut dyn SubBandRows| -> Result<()> { - let written = output.write(&mut lines, rows, scratch); + let run = |readers: &mut [&mut (dyn SubBandRows + Send)]| -> Result<()> { + let written = output.write_batches(&mut lines, readers, scratch, batch); output_samples = lines.iter().map(LineReconstruction::output_samples).sum(); written }; match coding { CodeBlockCoding::Classic => run_bands( - workers, + (workers, readers, helps), &rings, &bands, slots, @@ -419,7 +428,7 @@ pub(super) fn decode_tile_rows<'a>( strand_once, ), CodeBlockCoding::HighThroughput => run_bands( - workers, + (workers, readers, helps), &rings, &bands, slots, @@ -483,6 +492,28 @@ pub(super) fn decode_tile_rows<'a>( } } +/// Pool threads a classic tile needs before its components reconstruct on +/// the pool. A reader waiting for a band holds its thread, and classic +/// Tier-1 keeps readers waiting, so a smaller pool does better with every +/// thread decoding bands. +const MIN_CLASSIC_SYNTHESIS_THREADS: usize = 12; + +/// Whether a streamed tile's `components` reconstruct on the pool, one per +/// task, given its block coding, a workspace bank of `bank` and routes +/// planned for `route_workers` pool threads. +fn parallel_synthesis( + components: usize, + coding: CodeBlockCoding, + bank: usize, + route_workers: usize, +) -> bool { + components > 1 + && route_workers >= 2 + && bank > components + && (coding == CodeBlockCoding::HighThroughput + || rayon::current_num_threads() >= MIN_CLASSIC_SYNTHESIS_THREADS) +} + /// The route of a band's `strip`, counted over every component's route from /// `strip_starts`, and the strip in that route. fn route_strip<'r, 'a>( @@ -952,27 +983,30 @@ impl<'c, 'w, W> Shared<'_, 'c, 'w, W> { } /// Decode `bands` into `slots` on up to `workers` pool threads while `run` -/// drives the IDWT through them on this thread, then decode the bands it -/// didn't read. Returns `false` when a band was stranded. +/// drives the IDWT through `readers` views of them, then decode the bands it +/// didn't read. Each view reads its own sub-bands and decodes a band nobody +/// has started with a workspace of its own, so views may run on different +/// threads. Views that `help` decode other bands while they wait. Returns +/// `false` when a band was stranded. #[expect( clippy::too_many_arguments, reason = "the executor takes the plan, its storage and its two drivers explicitly" )] fn run_bands( - workers: usize, + (workers, readers, helps): (usize, usize, bool), rings: &[Ring], bands: &[Band], slots: Slots<'_>, bank: &mut [TaskWorkspace], decode: impl Fn(usize, &mut [f32], &mut W, &mut Vec) -> Result<()> + Sync, - run: impl FnOnce(&mut dyn SubBandRows) -> Result<()>, + run: impl FnOnce(&mut [&mut (dyn SubBandRows + Send)]) -> Result<()>, profile_enabled: bool, tier1_us: &mut u128, strand_once: bool, ) -> Result { #[cfg(not(test))] let _ = strand_once; - let workspaces_needed = workers + 1; + let workspaces_needed = workers + readers.max(1); let mut workspaces: Vec<&mut W> = Vec::new(); workspaces .try_reserve_exact(workspaces_needed) @@ -985,9 +1019,7 @@ fn run_bands( if workspaces.len() != workspaces_needed { return Err(DecodingError::CodeBlockDecodeFailure.into()); } - let own_workspace = workspaces - .pop() - .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let reader_workspaces = workspaces.split_off(workers); let mut state = State { status: try_filled(bands.len(), Status::Waiting)?, held: try_filled_with(bands.len(), || None)?, @@ -1030,20 +1062,45 @@ fn run_bands( let decode = &decode; let shared = &shared; let scoped = rayon::in_place_scope(|scope| -> Result<()> { - let mut rows = PoolRows { - shared, - scope, - decode, - workspace: own_workspace, - clipped_scratch: Vec::new(), - current: try_filled_with(rings.len(), || None)?, - next: try_collect(rings.iter().map(|ring| ring.bands.start), rings.len())?, - profile_enabled, - tier1_us: 0, - }; - rows.send_workers()?; - let result = run(&mut rows).and_then(|()| rows.drain()); - *tier1_us = rows.tier1_us; + let mut views = Vec::new(); + views + .try_reserve_exact(reader_workspaces.len()) + .map_err(|_| DecodingError::HostAllocationFailed)?; + for workspace in reader_workspaces { + views.push(PoolRows { + shared, + scope, + decode, + workspace, + clipped_scratch: Vec::new(), + current: try_filled_with(rings.len(), || None)?, + next: try_collect(rings.iter().map(|ring| ring.bands.start), rings.len())?, + helps, + profile_enabled, + tier1_us: 0, + }); + } + views + .first() + .ok_or(DecodingError::CodeBlockDecodeFailure)? + .send_workers()?; + let mut readers: Vec<&mut (dyn SubBandRows + Send)> = Vec::new(); + readers + .try_reserve_exact(views.len()) + .map_err(|_| DecodingError::HostAllocationFailed)?; + readers.extend( + views + .iter_mut() + .map(|view| view as &mut (dyn SubBandRows + Send)), + ); + let result = run(&mut readers).and_then(|()| { + // Every view hands its bands back before the first drains. + for view in &mut views[1..] { + view.release_all()?; + } + views[0].drain() + }); + *tier1_us = views.iter().map(|view| view.tier1_us).sum(); if result.is_err() { // Let the workers stop at their next band. if let Ok(mut state) = shared.lock() { @@ -1161,6 +1218,9 @@ struct PoolRows<'s, 'p, 'c, 'w, W, D> { current: Vec>, /// Each ring's next band to read. next: Vec, + /// Whether a wait for a running band first decodes the next band + /// nobody has started. + helps: bool, profile_enabled: bool, tier1_us: u128, } @@ -1211,7 +1271,9 @@ where fn acquire(&mut self, index: usize) -> Result<&'c mut [f32]> { let mut state = self.shared.lock()?; loop { - if let Some(error) = state.error.take() { + // The error stays for the other readers, whose bands it may + // have left running. + if let Some(error) = state.error { return Err(error); } match state.status[index] { @@ -1221,6 +1283,22 @@ where .ok_or(DecodingError::CodeBlockDecodeFailure.into()); } Status::Running => { + // A reader that helps decodes the next band nobody has + // started while this one runs. + if self.helps { + if let Some((other, slot)) = self.shared.start_next(&mut state)? { + // A reader decoding a band counts as an active + // worker for anyone waiting on it. + state.active += 1; + drop(state); + self.send_workers()?; + let decoded = self.decode_for_pool(other, slot); + state = self.shared.lock()?; + state.active -= 1; + decoded?; + continue; + } + } state = self .shared .decoded @@ -1247,6 +1325,21 @@ where } } + /// Decode band `index`, which another reader or the drain will take, + /// and hand it over as a worker does. + fn decode_for_pool(&mut self, index: usize, slot: &'c mut [f32]) -> Result<()> { + let start = profile::profile_now(self.profile_enabled); + slot.fill(0.0); + let result = (self.decode)( + self.shared.bands[index].strip, + slot, + self.workspace, + &mut self.clipped_scratch, + ); + self.tier1_us += profile::elapsed_us(start); + self.shared.finish(index, slot, result) + } + fn decode_here(&mut self, index: usize, slot: &mut [f32]) -> Result<()> { let start = profile::profile_now(self.profile_enabled); slot.fill(0.0); @@ -1264,15 +1357,23 @@ where Ok(()) } - /// After the IDWT, free its bands and decode every band it didn't read, - /// here and on the pool. - fn drain(&mut self) -> Result<()> { + /// Hand back every band this view is reading. + fn release_all(&mut self) -> Result<()> { for ring in 0..self.current.len() { self.release(ring)?; } + Ok(()) + } + + /// After the IDWT, free its bands and decode every band it didn't read, + /// here and on the pool. + fn drain(&mut self) -> Result<()> { + self.release_all()?; loop { let mut state = self.shared.lock()?; - if let Some(error) = state.error.take() { + // The error stays for the other readers, whose bands it may + // have left running. + if let Some(error) = state.error { return Err(error); } // Free every decoded band, which lets its ring's next band start. diff --git a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs index aaabfb16..360559c3 100644 --- a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs +++ b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs @@ -1371,7 +1371,9 @@ fn decode_strip( let (block_output, output_stride) = if whole { (&mut output[placement.target_x as usize..], strip.stride) } else { - if clipped_scratch.is_empty() { + // One worker's scratch serves the strips of every component's + // plan, so it grows to the largest clipped block of each. + if clipped_scratch.len() < plan.max_clipped_samples { try_resize_decode_elements(clipped_scratch, plan.max_clipped_samples, 0.0)?; } let block = clipped_scratch diff --git a/crates/j2k-native/src/j2c/idwt.rs b/crates/j2k-native/src/j2c/idwt.rs index 5e57feac..b77ee9af 100644 --- a/crates/j2k-native/src/j2c/idwt.rs +++ b/crates/j2k-native/src/j2c/idwt.rs @@ -27,7 +27,9 @@ pub(crate) use lines::{LineReconstruction, SubBandRows}; pub(crate) use model::idwt_buffer_size; pub(crate) use model::{IDWTOutput, IdwtPlacement}; pub(crate) use orchestrate::{apply, planned_workspace_elements}; -pub(crate) use roi::{last_level_fits_channels, roi_idwt_peak_elements, ComponentLines, LinePlan}; +pub(crate) use roi::{ + last_level_fits_channels, roi_idwt_peak_elements, streamed_batch_rows, ComponentLines, LinePlan, +}; // Decoded high-pass sub-bands include JPEG 2000's orientation gain. OpenJPEG's // synthesis constant is therefore halved at this internal codestream boundary. diff --git a/crates/j2k-native/src/j2c/idwt/roi.rs b/crates/j2k-native/src/j2c/idwt/roi.rs index ba19b451..c19baa8f 100644 --- a/crates/j2k-native/src/j2c/idwt/roi.rs +++ b/crates/j2k-native/src/j2c/idwt/roi.rs @@ -17,6 +17,16 @@ use crate::j2c::Header; use crate::math::{SimdBuffer, SIMD_WIDTH}; use crate::{checked_decode_usize_product2, try_resize_decode_elements, Result}; +/// Samples of each component's rows a tile streaming from pool bands +/// reconstructs at once, before they are transformed and handed over. +const STREAMED_BATCH_SAMPLES: usize = 256 * 1024; + +/// Rows of each component a tile streaming from pool bands reconstructs at +/// once, for rows of `width` samples. +pub(crate) fn streamed_batch_rows(width: usize) -> usize { + (STREAMED_BATCH_SAMPLES / width.max(1)).clamp(1, 256) +} + /// The most f32 samples [`apply_roi`] holds at once for any one component. /// /// Each level reads the previous level's window and writes a new one, so both @@ -29,8 +39,9 @@ use crate::{checked_decode_usize_product2, try_resize_decode_elements, Result}; /// at the 9/7 ring depth whatever the component's transform. With no active /// level it holds nothing, since it copies the LL window straight into the /// plane. When the tile streams its rows to a sink, every component's rings -/// are live at once, with a scratch row each as wide as its final window, so -/// the peak is their sum. +/// are live at once, with a scratch row each as wide as its final window, or +/// a batch of such rows from pool bands (see [`streamed_batch_rows`]), so the +/// peak is their sum. pub(crate) fn roi_idwt_peak_elements( storage: &DecompositionStorage<'_>, skipped_resolution_levels: u8, @@ -65,9 +76,14 @@ pub(crate) fn roi_idwt_peak_elements( .map_while(|decomposition_idx| roi_plan.idwt_window(decomposition_idx)); let rings = ring_samples(windows, WaveletTransform::Irreversible97)?; if storage.lockstep_lines { - let scratch_row = roi_plan - .final_window(component_idx) - .map_or(0, |window| window.width() as usize); + let scratch_row = roi_plan.final_window(component_idx).map_or(0, |window| { + let width = window.width() as usize; + if storage.parallel_bands { + width.saturating_mul(streamed_batch_rows(width)) + } else { + width + } + }); lockstep = lockstep .checked_add(rings) .and_then(|total| total.checked_add(scratch_row)) diff --git a/crates/j2k-native/src/tests/row_sink.rs b/crates/j2k-native/src/tests/row_sink.rs index 79ac7b83..cd86f709 100644 --- a/crates/j2k-native/src/tests/row_sink.rs +++ b/crates/j2k-native/src/tests/row_sink.rs @@ -310,9 +310,7 @@ fn pool_band_rows_match_component_planes_bit_for_bit() { pool.install(|| decode_rows(&image, roi, CpuDecodeParallelism::Auto)); assert_eq!(rows.planes, planes.planes, "{label}"); assert_eq!(rows.depths, planes.depths, "{label}"); - // HT tiles decode their bands on the pool into component - // planes (see `parallel_bands::streams_rows`). - if rows.pool_components > 0 && !case.options.use_ht_block_coding { + if rows.pool_components > 0 { assert!(rows.streamed, "{label}: streamed"); assert!(rows.channel_bytes <= 256, "{label}: no planes"); assert_eq!(rows.decoded_blocks, planes.decoded_blocks, "{label}"); @@ -334,6 +332,173 @@ fn pool_band_rows_match_component_planes_bit_for_bit() { } } +/// Each component's route plans its own clipped code-blocks, while a pool +/// worker or reader decodes the strips of every component with one scratch. +/// Components with detail only along the right edge code only the narrow +/// last blocks of each band, and a noisy one full-size blocks, so a worker +/// that served an edge component first must still fit the noisy component's +/// clipped blocks. +#[cfg(feature = "parallel")] +#[test] +fn pool_band_rows_fit_every_components_clipped_blocks() { + let noise = noisy_pixels(1); + let mut pixels = Vec::with_capacity(noise.len() * 3); + for (at, &sample) in noise.iter().enumerate() { + let edge = at % WIDTH as usize >= WIDTH as usize - 6; + let detail = if edge { sample } else { 128 }; + pixels.extend_from_slice(&[sample, detail, detail ^ 0x55]); + } + for (label, use_ht_block_coding) in [("classic", false), ("ht", true)] { + let bytes = encode( + &pixels, + WIDTH, + HEIGHT, + 3, + 8, + false, + &EncodeOptions { + num_decomposition_levels: 4, + code_block_width_exp: 4, + code_block_height_exp: 0, + use_ht_block_coding, + use_mct: false, + ..EncodeOptions::default() + }, + ) + .expect("encode clipped block fixture"); + let image = Image::new_with_reduction(&bytes, &DecodeSettings::default(), 0) + .expect("parse clipped block fixture"); + for threads in [2, 3, 6, 12] { + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("build pool"); + for roi in [(100, 17, 103, 120), (37, 3, 166, 9), (60, 41, 143, 101)] { + let planes = decode_planes(&image, roi); + for attempt in 0..4 { + let rows = + pool.install(|| decode_rows(&image, roi, CpuDecodeParallelism::Auto)); + assert_eq!( + rows.planes, planes.planes, + "{label} threads={threads} roi={roi:?} attempt={attempt}" + ); + } + } + } + } +} + +/// A sink with an RGB8 destination, which [`ComponentRowSink::write_rows`] +/// fills the way the destination's contract says: each 8-bit sample rounded +/// and saturated. +struct Rgb8Sink { + width: usize, + rgb: Vec, + offers_destination: bool, + rectangles: usize, +} + +impl ComponentRowSink for Rgb8Sink { + #[expect( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + reason = "samples are rounded and saturated to 8 bits, as the destination contract says" + )] + fn write_rows(&mut self, x: u32, y: u32, planes: &[ComponentPlane<'_>]) { + self.rectangles += 1; + let (width, height) = planes[0].dimensions(); + for row in 0..height as usize { + for column in 0..width as usize { + let pixel = (y as usize + row) * self.width + x as usize + column; + for (channel, plane) in planes.iter().enumerate() { + let sample = plane.samples()[row * width as usize + column]; + self.rgb[pixel * 3 + channel] = sample.round().clamp(0.0, 255.0) as u8; + } + } + } + } + + fn rgb8_region(&mut self, width: u32, height: u32) -> Option<&mut [u8]> { + let len = (width as usize * 3).checked_mul(height as usize)?; + self.offers_destination.then(|| &mut self.rgb[..len]) + } +} + +/// A streamed decode may pack three 8-bit components straight into a sink's +/// RGB8 destination. The bytes must be those the sink packs from the rows, +/// on one thread and on the pool, where each component's reconstruction runs +/// as its own task when the pool is large enough. +#[cfg(feature = "parallel")] +#[test] +fn rgb8_destination_matches_the_sink_pack_byte_for_byte() { + let pools: Vec = [2, 4, 12] + .into_iter() + .map(|threads| { + rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("build pool") + }) + .collect(); + for case in row_sink_cases() + .into_iter() + .filter(|case| case.components == 3) + { + let pixels = noisy_pixels(case.components); + let bytes = encode(&pixels, WIDTH, HEIGHT, 3, 8, false, &case.options) + .expect("encode RGB8 destination fixture"); + let mut packed_into_destination = 0; + for reduction in [0_u8, 1] { + let image = Image::new_with_reduction(&bytes, &DecodeSettings::default(), reduction) + .expect("parse RGB8 destination fixture"); + let (width, height) = (image.width(), image.height()); + for roi in [ + (0, 0, width, height), + ( + width / 20, + height / 20, + width - width / 10, + height - height / 10, + ), + (width / 3, height / 3, width / 7, height / 9), + ] { + let decode = |parallelism, offers_destination| { + let mut context = DecoderContext::default(); + context.set_cpu_decode_parallelism(parallelism); + let mut sink = Rgb8Sink { + width: roi.2 as usize, + rgb: alloc::vec![0; (roi.2 * roi.3 * 3) as usize], + offers_destination, + rectangles: 0, + }; + image + .decode_region_rows_for_integer_output_with_context( + roi, + &mut context, + &mut sink, + ) + .expect("decode RGB8 destination rows"); + sink + }; + let label = format!("{} reduction={reduction} roi={roi:?}", case.label); + let expected = decode(CpuDecodeParallelism::Serial, false); + for (pool, threads) in pools.iter().zip([2, 4, 12]) { + let packed = pool.install(|| decode(CpuDecodeParallelism::Auto, true)); + assert_eq!(packed.rgb, expected.rgb, "{label} threads={threads}"); + packed_into_destination += usize::from(packed.rectangles == 0); + let rows = pool.install(|| decode(CpuDecodeParallelism::Auto, false)); + assert_eq!(rows.rgb, expected.rgb, "{label} threads={threads} rows"); + } + } + } + assert!( + packed_into_destination > 0, + "{}: never packed into the destination", + case.label + ); + } +} + /// Outcome of a region decode, through the planes or the row sink: a hash /// of the samples, or the error. fn outcome( diff --git a/crates/j2k/src/decode/output/u8.rs b/crates/j2k/src/decode/output/u8.rs index 83c29acd..3a655c9e 100644 --- a/crates/j2k/src/decode/output/u8.rs +++ b/crates/j2k/src/decode/output/u8.rs @@ -145,6 +145,21 @@ impl j2k_native::ComponentRowSink for U8RowSink<'_> { }; self.error = result.err(); } + + fn rgb8_region(&mut self, width: u32, height: u32) -> Option<&mut [u8]> { + // `write_rows` packs three 8-bit planes as `write_rgb_component_rows_u8` + // does, so a tight RGB8 buffer can take them packed by the decoder. + let row_bytes = (width as usize).checked_mul(3)?; + let tight_rgb8 = self.error.is_none() + && matches!(self.color_space, ColorSpace::RGB) + && !self.has_alpha + && self.fmt == PixelFormat::Rgb8 + && self.stride == row_bytes; + if !tight_rgb8 { + return None; + } + self.out.get_mut(..row_bytes.checked_mul(height as usize)?) + } } pub(super) fn component_sample_count(dims: (u32, u32)) -> Result { diff --git a/xtask/api/stable-api-1.0.implementation-public-api.txt b/xtask/api/stable-api-1.0.implementation-public-api.txt index 30313943..0f439d90 100644 --- a/xtask/api/stable-api-1.0.implementation-public-api.txt +++ b/xtask/api/stable-api-1.0.implementation-public-api.txt @@ -2463,6 +2463,7 @@ pub enum j2k_native::Jp2ColorSpec pub enum j2k_native::Jp2ComponentMappingType pub enum j2k_native::Jp2FileKind pub fn j2k_native::ComponentPlane<'a>::into_parts(self) -> j2k_native::ComponentPlaneParts<'a> +pub fn j2k_native::ComponentRowSink::rgb8_region(&mut self, u32, u32) -> core::option::Option<&mut [u8]> pub fn j2k_native::ComponentRowSink::write_rows(&mut self, u32, u32, &[j2k_native::ComponentPlane<'_>]) pub fn j2k_native::DecodedComponents<'a>::into_parts(self) -> j2k_native::DecodedComponentsParts<'a> pub fn j2k_native::DecodedComponents<'a>::live_bytes(&self) -> usize From a421fdff680c26b0d600868f357b171e5a40c8b7 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 02:17:25 -0400 Subject: [PATCH 05/94] perf(j2k-native): finish region batches on every reader's thread With a reader per component, the calling thread transformed and packed each batch alone while the other readers' threads waited for the next batch. The batch's rows now split into one range per reader, the first finished here and the rest spawned on the pool, each transforming its rows one at a time as before and packing them into its own output rows. One reader keeps a single range. Measured with region decodes to RGB8 (5 interleaved rounds, median of per-round ratios; output byte-identical at 1, 6 and 18 threads) on regions of an 18000 x 12000 RGB image: HTJ2K, classic with PLT markers, and classic without. Against the parent / against f33e40c6: - HTJ2K whole image at 1/4 scale: -10% / -81% at 6 threads, -15% / -81% at 18; at 1/16 scale: -4% to -5% / -65% at 6 and 18. - HTJ2K 1024 x 1024: -2% / -48% at 6 threads, -4% / -49% at 18. - Classic at 18 threads: PLT 1024 x 1024 -3% to +2% / -47% to -57% (a 7-round recheck of the 1024 x 1024 rows: -1% to -2%); whole image at 1/8 and 1/16 scale -1% / -57% and -63%. - Classic at 6 threads, one reader: within +-1%. --- .../j2k-native/src/j2c/decode/banded_tier1.rs | 183 +++++++++++++++--- 1 file changed, 160 insertions(+), 23 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode/banded_tier1.rs b/crates/j2k-native/src/j2c/decode/banded_tier1.rs index a8795aaf..c9c7edfb 100644 --- a/crates/j2k-native/src/j2c/decode/banded_tier1.rs +++ b/crates/j2k-native/src/j2c/decode/banded_tier1.rs @@ -429,35 +429,37 @@ impl RowOutput<'_, '_> { return Err(DecodingError::CodeBlockDecodeFailure.into()); } reconstruct_batch(lines, readers, parts, y, width)?; - for row in 0..rows { - let mut outputs: [&mut [f32]; 4] = Default::default(); - for (output, part) in outputs.iter_mut().zip(parts.iter_mut()) { - *output = &mut part[row * width..][..width]; - } - store::transform_streamed_rows( - self.tile, - self.round_irreversible_output, - &mut outputs[..components], - )?; - } let (x, row_in_region) = (copy.x0 - region.x0, y - region.y0); let destination = rgb8 .then(|| self.sink.rgb8_region(region.width(), region.height())) .flatten(); - if let Some(out) = destination { - let stride = region.width() as usize * 3; - for row in 0..rows { - let start = (row_in_region as usize + row) * stride + x as usize * 3; - let pixels = out - .get_mut(start..start + width * 3) + let rgb8_rows = match destination { + Some(out) => { + let stride = region.width() as usize * 3; + let start = row_in_region as usize * stride; + let out = out + .get_mut(start..start + rows * stride) .ok_or(DecodingError::CodeBlockDecodeFailure)?; - let at = row * width..(row + 1) * width; - crate::color::pack_rgb8( - [&parts[0][at.clone()], &parts[1][at.clone()], &parts[2][at]], - pixels, - ); + Some(Rgb8Rows { + out, + stride, + x: x as usize * 3, + }) } - } else { + None => None, + }; + let packed = rgb8_rows.is_some(); + // With a reader per component the rows' transform and pack split + // across as many tasks, which the readers' threads are free for. + let tasks = if readers.len() > 1 { readers.len() } else { 1 }; + finish_rows( + (self.tile, self.round_irreversible_output), + parts, + width, + rgb8_rows, + tasks, + )?; + if !packed { let component_rows: [ComponentPlane<'_>; 4] = core::array::from_fn(|index| ComponentPlane { samples: parts.get(index).map_or(&[][..], |part| &**part), @@ -549,6 +551,141 @@ fn reconstruct_rows_into( Ok(()) } +/// Output rows that take three 8-bit components packed as RGB8. +struct Rgb8Rows<'o> { + /// Whole output rows, `stride` bytes each, one per reconstructed row. + out: &'o mut [u8], + stride: usize, + /// Where the reconstructed samples start in each output row, in bytes. + x: usize, +} + +/// One range of a batch's rows: each component's rows and their output. +struct BatchRows<'p, 'o> { + parts: [&'p mut [f32]; 4], + rgb8: Option>, +} + +impl BatchRows<'_, '_> { + /// Transform the rows one at a time, as [`RowOutput::write`] does, then + /// pack them into the RGB8 rows if there are any. + fn finish( + mut self, + (tile, round_irreversible_output): (&Tile<'_>, bool), + components: usize, + width: usize, + ) -> Result<()> { + let rows = self.parts[0].len().checked_div(width).unwrap_or(0); + for row in 0..rows { + let mut outputs: [&mut [f32]; 4] = Default::default(); + for (output, part) in outputs.iter_mut().zip(self.parts.iter_mut()) { + *output = part + .get_mut(row * width..(row + 1) * width) + .unwrap_or_default(); + } + store::transform_streamed_rows( + tile, + round_irreversible_output, + &mut outputs[..components], + )?; + } + let Some(rgb8) = self.rgb8 else { + return Ok(()); + }; + let [red, green, blue, _] = &self.parts; + let samples = red + .chunks_exact(width) + .zip(green.chunks_exact(width)) + .zip(blue.chunks_exact(width)); + for (line, ((red, green), blue)) in rgb8.out.chunks_exact_mut(rgb8.stride).zip(samples) { + let pixels = line + .get_mut(rgb8.x..rgb8.x + width * 3) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + crate::color::pack_rgb8([red, green, blue], pixels); + } + Ok(()) + } +} + +/// Transform each component's rows in `parts`, `width` samples each, and +/// pack them into `rgb8` when given, in up to `tasks` ranges of rows, all but +/// the first spawned on the pool. +fn finish_rows( + transform: (&Tile<'_>, bool), + parts: &mut [&mut [f32]], + width: usize, + mut rgb8: Option>, + tasks: usize, +) -> Result<()> { + let components = parts.len(); + let rows = parts + .first() + .map_or(0, |part| part.len().checked_div(width).unwrap_or(0)); + if components == 0 || components > 4 || rows == 0 { + return Ok(()); + } + let range_rows = rows.div_ceil(tasks.clamp(1, 4)); + let mut rest: [&mut [f32]; 4] = Default::default(); + for (rest, part) in rest.iter_mut().zip(parts.iter_mut()) { + *rest = &mut **part; + } + let mut ranges: [Option>; 4] = Default::default(); + for range in &mut ranges { + let take = range_rows.min(rest[0].len() / width); + if take == 0 { + break; + } + let mut range_parts: [&mut [f32]; 4] = Default::default(); + for (part, rest) in range_parts.iter_mut().zip(rest.iter_mut()) { + let samples = core::mem::take(rest); + let (head, tail) = samples.split_at_mut((take * width).min(samples.len())); + *part = head; + *rest = tail; + } + let range_rgb8 = match rgb8.take() { + Some(Rgb8Rows { out, stride, x }) => { + let (head, tail) = out.split_at_mut((take * stride).min(out.len())); + rgb8 = Some(Rgb8Rows { + out: tail, + stride, + x, + }); + Some(Rgb8Rows { + out: head, + stride, + x, + }) + } + None => None, + }; + *range = Some(BatchRows { + parts: range_parts, + rgb8: range_rgb8, + }); + } + let mut results: [Result<()>; 4] = core::array::from_fn(|_| Ok(())); + #[cfg(feature = "parallel")] + rayon::in_place_scope(|scope| { + let mut jobs = ranges.into_iter().zip(results.iter_mut()); + let first = jobs.next(); + for (range, result) in jobs { + if let Some(range) = range { + scope.spawn(move |_| *result = range.finish(transform, components, width)); + } + } + if let Some((Some(range), result)) = first { + *result = range.finish(transform, components, width); + } + }); + #[cfg(not(feature = "parallel"))] + for (range, result) in ranges.into_iter().zip(results.iter_mut()) { + if let Some(range) = range { + *result = range.finish(transform, components, width); + } + } + results.into_iter().collect() +} + /// When a lockstep decode fails, rerun Tier-1 over whole windows one /// component at a time, in the order the planes path decodes them, and /// return the first error. If the whole windows don't fit the cap, or the From 4e8354fd03b58f25c0672443adfa5e55f22e9258 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 02:17:26 -0400 Subject: [PATCH 06/94] perf(j2k-native): finish region batches while the next ones reconstruct Each batch of streamed rows ended in a join: the readers reconstructed it, then it was transformed and packed while every reader waited. With a reader per component, the tile now holds two batches: one is transformed and packed on the pool while the readers reconstruct the next into the other. Batches hold up to 2 Mi samples of each component and 512 rows, and at most a quarter of the tile's rows, so the two batches stay half of the rows' samples or less; fewer, larger batches spend less time in joins. One reader reconstructs a row of each component at a time within a batch, as the row-at-a-time path does, instead of a batch of one component after another, which left the other components' band rings full and the workers idle. Measured with region decodes to RGB8 (5 interleaved rounds, median of per-round ratios; output byte-identical at 1, 6 and 18 threads) on regions of an 18000 x 12000 RGB image: HTJ2K, classic with PLT markers, and classic without. Against the parent / against f33e40c6: - HTJ2K whole image at 1/4 scale: -11% / -83% at 6 threads, -29% / -87% at 18; at 1/8 scale: -6% / -79% at 6, -18% / -81% at 18. - HTJ2K 1024 x 1024, 4096 x 4096 at 1/4 scale and whole image at 1/16 scale: -4% to +0.1% / -48% to -69%. - Classic PLT whole image at 1/16 scale, one reader at 6 threads: -12% / -36%. - Other classic rows at 6 and 18 threads: -2% to +2% / -13% to -64%. - Peak heap against f33e40c6: -48% to +0.2% at 6 and 18 threads (the +0.2% on 4096 x 4096 at 1/4 scale), at most +0.1% at one thread. --- .../j2k-native/src/j2c/decode/banded_tier1.rs | 301 ++++++++++++++---- .../src/j2c/decode/parallel_bands.rs | 22 +- crates/j2k-native/src/j2c/idwt.rs | 6 +- crates/j2k-native/src/j2c/idwt/roi.rs | 29 +- crates/j2k-native/src/tests/row_sink.rs | 19 ++ 5 files changed, 298 insertions(+), 79 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode/banded_tier1.rs b/crates/j2k-native/src/j2c/decode/banded_tier1.rs index c9c7edfb..e192c884 100644 --- a/crates/j2k-native/src/j2c/decode/banded_tier1.rs +++ b/crates/j2k-native/src/j2c/decode/banded_tier1.rs @@ -394,13 +394,19 @@ impl<'o, 't> RowOutput<'o, 't> { } } +#[cfg_attr( + not(feature = "parallel"), + expect(dead_code, reason = "only the parallel band reader writes batches") +)] impl RowOutput<'_, '_> { /// [`Self::write`] through one reader per component, which may run on /// different threads, `batch` rows at a time. Each component's /// reconstruction advances `batch` rows into its own rows of `scratch`, /// on the pool when there is a reader per component, then the rows are /// transformed and handed over, packed straight into the sink's RGB8 - /// destination when it has one for three 8-bit components. + /// destination when it has one for three 8-bit components. With a + /// reader per component, a batch is transformed and packed on the pool + /// while the next one reconstructs, so `scratch` holds two batches. pub(super) fn write_batches( &mut self, lines: &mut [LineReconstruction<'_>], @@ -408,7 +414,7 @@ impl RowOutput<'_, '_> { scratch: &mut [f32], batch: usize, ) -> Result<()> { - let (copy, region, components) = (self.copy, self.region, lines.len()); + let (copy, components) = (self.copy, lines.len()); let width = copy.width() as usize; if lines.is_empty() || width == 0 || batch == 0 { return Ok(()); @@ -416,71 +422,219 @@ impl RowOutput<'_, '_> { let batch_len = width .checked_mul(batch) .ok_or(ValidationError::ImageTooLarge)?; - let rgb8 = components == 3 && self.depths[..3].iter().all(|(depth, _)| *depth == 8); + let half = batch_len + .checked_mul(components) + .ok_or(ValidationError::ImageTooLarge)?; + let transform = (self.tile, self.round_irreversible_output); + let tasks = if readers.len() > 1 { readers.len() } else { 1 }; + let (now, rest) = scratch.split_at_mut(half.min(scratch.len())); + // With a reader per component and room for two batches, a batch is + // transformed and handed over on the pool while the next one + // reconstructs in the other half of `scratch`. + let pipelined = readers.len() > 1 && rest.len() >= half; + let mut halves = [now, rest]; + // The rows, from a row, of the batch in `halves[1]` left to finish. + let mut pending = None; let mut y = copy.y0; - while y < copy.y1 { - let rows = ((copy.y1 - y) as usize).min(batch); - let mut parts: [&mut [f32]; 4] = Default::default(); - for (part, chunk) in parts.iter_mut().zip(scratch.chunks_exact_mut(batch_len)) { - *part = &mut chunk[..rows * width]; + loop { + let rows = (copy.y1.saturating_sub(y) as usize).min(batch); + let [now, before] = &mut halves; + let done = pending.take(); + if rows == 0 { + if let Some(done) = done { + self.finish_batch( + transform, + before, + done, + (batch_len, components, tasks), + None, + )?; + } + return Ok(()); } - let parts = &mut parts[..components]; - if parts.iter().any(|part| part.len() != rows * width) { - return Err(DecodingError::CodeBlockDecodeFailure.into()); + let mut parts = batch_parts(now, batch_len, rows * width, components)?; + let mut reconstruct = + || reconstruct_batch(lines, readers, &mut parts[..components], y, width); + match done { + Some(done) => self.finish_batch( + transform, + before, + done, + (batch_len, components, tasks), + Some(&mut reconstruct), + )?, + None => reconstruct()?, } - reconstruct_batch(lines, readers, parts, y, width)?; - let (x, row_in_region) = (copy.x0 - region.x0, y - region.y0); - let destination = rgb8 - .then(|| self.sink.rgb8_region(region.width(), region.height())) - .flatten(); - let rgb8_rows = match destination { - Some(out) => { - let stride = region.width() as usize * 3; - let start = row_in_region as usize * stride; - let out = out - .get_mut(start..start + rows * stride) - .ok_or(DecodingError::CodeBlockDecodeFailure)?; - Some(Rgb8Rows { - out, - stride, - x: x as usize * 3, - }) - } - None => None, - }; - let packed = rgb8_rows.is_some(); - // With a reader per component the rows' transform and pack split - // across as many tasks, which the readers' threads are free for. - let tasks = if readers.len() > 1 { readers.len() } else { 1 }; - finish_rows( - (self.tile, self.round_irreversible_output), - parts, - width, - rgb8_rows, - tasks, - )?; - if !packed { - let component_rows: [ComponentPlane<'_>; 4] = - core::array::from_fn(|index| ComponentPlane { - samples: parts.get(index).map_or(&[][..], |part| &**part), - dimensions: (copy.width(), rows as u32), - bit_depth: self.depths[index].0, - signed: self.depths[index].1, - sampling: (1, 1), - }); - self.sink - .write_rows(x, row_in_region, &component_rows[..components]); + if pipelined { + pending = Some((y, rows)); + halves.swap(0, 1); + } else { + let [now, _] = &mut halves; + self.finish_batch( + transform, + now, + (y, rows), + (batch_len, components, tasks), + None, + )?; } - self.streamed = self.streamed.saturating_add(rows * width * components); y += rows as u32; } + } + + /// Transform the `rows` rows of each component in `buffer`, `batch_len` + /// samples apart, from row `y`, in up to `tasks` ranges, and hand them + /// over: packed into the sink's RGB8 rows when it has them, else through + /// `write_rows`. With `alongside`, the rows finish on the pool while + /// `alongside` runs here. + fn finish_batch( + &mut self, + transform: (&Tile<'_>, bool), + buffer: &mut [f32], + (y, rows): (u32, usize), + (batch_len, components, tasks): (usize, usize, usize), + alongside: Option<&mut dyn FnMut() -> Result<()>>, + ) -> Result<()> { + let width = self.copy.width() as usize; + let mut parts = batch_parts(buffer, batch_len, rows * width, components)?; + let destination = self.rgb8_rows(y, rows)?; + let packed = destination.is_some(); + match alongside { + Some(alongside) => { + let mut finished = Ok(()); + let mut alongside_result = Ok(()); + overlap( + || { + finished = finish_rows( + transform, + &mut parts[..components], + width, + destination, + tasks, + ); + }, + || alongside_result = alongside(), + ); + alongside_result?; + finished?; + } + None => finish_rows( + transform, + &mut parts[..components], + width, + destination, + tasks, + )?, + } + if !packed { + self.write_component_rows(&parts[..components], y, rows); + } + self.streamed = self.streamed.saturating_add(rows * width * components); Ok(()) } + + /// The sink's RGB8 rows for `rows` rows of the tile from row `y`, when + /// it has a destination for three 8-bit components. + fn rgb8_rows(&mut self, y: u32, rows: usize) -> Result>> { + let (copy, region) = (self.copy, self.region); + let rgb8 = self.depths.len() >= 3 + && self.tile.component_infos.len() == 3 + && self.depths[..3].iter().all(|(depth, _)| *depth == 8); + if !rgb8 { + return Ok(None); + } + let Some(out) = self.sink.rgb8_region(region.width(), region.height()) else { + return Ok(None); + }; + let stride = region.width() as usize * 3; + let start = (y - region.y0) as usize * stride; + let out = out + .get_mut(start..start + rows * stride) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + Ok(Some(Rgb8Rows { + out, + stride, + x: (copy.x0 - region.x0) as usize * 3, + })) + } + + /// Hand `rows` rows of each component in `parts`, from row `y`, to the + /// sink. + fn write_component_rows(&mut self, parts: &[&mut [f32]], y: u32, rows: usize) { + let (copy, region) = (self.copy, self.region); + let component_rows: [ComponentPlane<'_>; 4] = + core::array::from_fn(|index| ComponentPlane { + samples: parts.get(index).map_or(&[][..], |part| &**part), + dimensions: (copy.width(), rows as u32), + bit_depth: self.depths[index].0, + signed: self.depths[index].1, + sampling: (1, 1), + }); + self.sink.write_rows( + copy.x0 - region.x0, + y - region.y0, + &component_rows[..parts.len()], + ); + } +} + +/// Each component's first `len` samples of its `batch_len` in `buffer`. +#[cfg_attr( + not(feature = "parallel"), + expect(dead_code, reason = "only the parallel band reader writes batches") +)] +fn batch_parts( + buffer: &mut [f32], + batch_len: usize, + len: usize, + components: usize, +) -> Result<[&mut [f32]; 4]> { + let mut parts: [&mut [f32]; 4] = Default::default(); + for (part, chunk) in parts + .iter_mut() + .zip(buffer.chunks_exact_mut(batch_len)) + .take(components) + { + *part = chunk + .get_mut(..len) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + } + if parts[..components.min(4)] + .iter() + .any(|part| part.len() != len) + { + return Err(DecodingError::CodeBlockDecodeFailure.into()); + } + Ok(parts) +} + +/// Run `spawned` on the pool while `inline` runs here, and return once both +/// have. +#[cfg_attr( + not(feature = "parallel"), + expect(dead_code, reason = "only the parallel band reader writes batches") +)] +fn overlap(spawned: impl FnOnce() + Send, inline: impl FnOnce()) { + #[cfg(feature = "parallel")] + rayon::in_place_scope(|scope| { + scope.spawn(move |_| spawned()); + inline(); + }); + #[cfg(not(feature = "parallel"))] + { + spawned(); + inline(); + } } /// Reconstruct the next `parts[c].len() / width` rows of each component `c`, /// from row `first_y`, into `parts[c]`: on the pool, one component per task, -/// when there is a reader per component, otherwise all through the first. +/// when there is a reader per component, otherwise all through the first, +/// a row of each component at a time. +#[cfg_attr( + not(feature = "parallel"), + expect(dead_code, reason = "only the parallel band reader writes batches") +)] fn reconstruct_batch( lines: &mut [LineReconstruction<'_>], readers: &mut [&mut (dyn SubBandRows + Send)], @@ -492,8 +646,21 @@ fn reconstruct_batch( let reader = readers .first_mut() .ok_or(DecodingError::CodeBlockDecodeFailure)?; - for (line, part) in lines.iter_mut().zip(parts.iter_mut()) { - reconstruct_rows_into(line, &mut **reader, part, first_y, width)?; + // A row of each component at a time, as `RowOutput::write` reads + // them, so the bands of every component are taken in turn and the + // pool keeps decoding all of them. + let rows = parts.first().map_or(0, |part| part.len() / width); + for row in 0..rows { + let y = u32::try_from(row) + .ok() + .and_then(|row| first_y.checked_add(row)) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + for (line, part) in lines.iter_mut().zip(parts.iter_mut()) { + let out = part + .get_mut(row * width..(row + 1) * width) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + reconstruct_rows_into(line, &mut **reader, out, y, width)?; + } } return Ok(()); } @@ -529,6 +696,10 @@ fn reconstruct_batch( /// The next rows of `line`, from row `first_y`, through `rows`, into `out`, /// `width` samples each. +#[cfg_attr( + not(feature = "parallel"), + expect(dead_code, reason = "only the parallel band reader writes batches") +)] fn reconstruct_rows_into( line: &mut LineReconstruction<'_>, rows: &mut dyn SubBandRows, @@ -552,6 +723,10 @@ fn reconstruct_rows_into( } /// Output rows that take three 8-bit components packed as RGB8. +#[cfg_attr( + not(feature = "parallel"), + expect(dead_code, reason = "only the parallel band reader writes batches") +)] struct Rgb8Rows<'o> { /// Whole output rows, `stride` bytes each, one per reconstructed row. out: &'o mut [u8], @@ -561,11 +736,19 @@ struct Rgb8Rows<'o> { } /// One range of a batch's rows: each component's rows and their output. +#[cfg_attr( + not(feature = "parallel"), + expect(dead_code, reason = "only the parallel band reader writes batches") +)] struct BatchRows<'p, 'o> { parts: [&'p mut [f32]; 4], rgb8: Option>, } +#[cfg_attr( + not(feature = "parallel"), + expect(dead_code, reason = "only the parallel band reader writes batches") +)] impl BatchRows<'_, '_> { /// Transform the rows one at a time, as [`RowOutput::write`] does, then /// pack them into the RGB8 rows if there are any. @@ -610,6 +793,10 @@ impl BatchRows<'_, '_> { /// Transform each component's rows in `parts`, `width` samples each, and /// pack them into `rgb8` when given, in up to `tasks` ranges of rows, all but /// the first spawned on the pool. +#[cfg_attr( + not(feature = "parallel"), + expect(dead_code, reason = "only the parallel band reader writes batches") +)] fn finish_rows( transform: (&Tile<'_>, bool), parts: &mut [&mut [f32]], diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index 9e3cf19b..05572e20 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -351,16 +351,6 @@ pub(super) fn decode_tile_rows<'a>( let (copy, region) = line_plans .first() .map_or((empty, empty), |plan| (plan.copy, plan.region)); - // Rows reconstruct in batches, charged by the region's width (see - // `roi_idwt_peak_elements`). - let batch = streamed_batch_rows(region.width() as usize); - let scratch_len = (copy.width() as usize) - .checked_mul(components) - .and_then(|samples| samples.checked_mul(batch)) - .ok_or(ValidationError::ImageTooLarge)?; - let (mut buffer, ring_lens) = - banded_tier1::take_line_buffer(tile_ctx, storage, &line_plans, scratch_len)?; - let mut output = RowOutput::new(tile, sink, tile_ctx, copy, region); // Every route plans the tile's one block coding (see `routes_tile`). let coding = routes .first() @@ -382,6 +372,18 @@ pub(super) fn decode_tile_rows<'a>( } else { 1 }; + // Rows reconstruct in batches, charged by the region's width (see + // `roi_idwt_peak_elements`): two of them with a reader per component, so + // one finishes while the next reconstructs. + let batch = streamed_batch_rows(region.width() as usize, copy.height() as usize); + let scratch_len = (copy.width() as usize) + .checked_mul(components) + .and_then(|samples| samples.checked_mul(batch)) + .and_then(|samples| samples.checked_mul(if readers > 1 { 2 } else { 1 })) + .ok_or(ValidationError::ImageTooLarge)?; + let (mut buffer, ring_lens) = + banded_tier1::take_line_buffer(tile_ctx, storage, &line_plans, scratch_len)?; + let mut output = RowOutput::new(tile, sink, tile_ctx, copy, region); let workers = route_workers.min(bank.saturating_sub(readers)); // One reader for every component of a classic tile leaves the pool's // threads to Tier-1, and decodes bands itself while it waits. diff --git a/crates/j2k-native/src/j2c/idwt.rs b/crates/j2k-native/src/j2c/idwt.rs index b77ee9af..f0d4a5f2 100644 --- a/crates/j2k-native/src/j2c/idwt.rs +++ b/crates/j2k-native/src/j2c/idwt.rs @@ -27,9 +27,9 @@ pub(crate) use lines::{LineReconstruction, SubBandRows}; pub(crate) use model::idwt_buffer_size; pub(crate) use model::{IDWTOutput, IdwtPlacement}; pub(crate) use orchestrate::{apply, planned_workspace_elements}; -pub(crate) use roi::{ - last_level_fits_channels, roi_idwt_peak_elements, streamed_batch_rows, ComponentLines, LinePlan, -}; +#[cfg(feature = "parallel")] +pub(crate) use roi::streamed_batch_rows; +pub(crate) use roi::{last_level_fits_channels, roi_idwt_peak_elements, ComponentLines, LinePlan}; // Decoded high-pass sub-bands include JPEG 2000's orientation gain. OpenJPEG's // synthesis constant is therefore halved at this internal codestream boundary. diff --git a/crates/j2k-native/src/j2c/idwt/roi.rs b/crates/j2k-native/src/j2c/idwt/roi.rs index c19baa8f..623acd20 100644 --- a/crates/j2k-native/src/j2c/idwt/roi.rs +++ b/crates/j2k-native/src/j2c/idwt/roi.rs @@ -18,13 +18,20 @@ use crate::math::{SimdBuffer, SIMD_WIDTH}; use crate::{checked_decode_usize_product2, try_resize_decode_elements, Result}; /// Samples of each component's rows a tile streaming from pool bands -/// reconstructs at once, before they are transformed and handed over. -const STREAMED_BATCH_SAMPLES: usize = 256 * 1024; +/// reconstructs at most at once, before they are transformed and handed over. +const STREAMED_BATCH_SAMPLES: usize = 2 * 1024 * 1024; + +/// Batches a tile streaming from pool bands splits its rows into at least, +/// so the two batches it holds at once stay at most half of the rows' +/// samples. +const STREAMED_MIN_BATCHES: usize = 4; /// Rows of each component a tile streaming from pool bands reconstructs at -/// once, for rows of `width` samples. -pub(crate) fn streamed_batch_rows(width: usize) -> usize { - (STREAMED_BATCH_SAMPLES / width.max(1)).clamp(1, 256) +/// once, for `height` rows of `width` samples. +pub(crate) fn streamed_batch_rows(width: usize, height: usize) -> usize { + (STREAMED_BATCH_SAMPLES / width.max(1)) + .min(height / STREAMED_MIN_BATCHES) + .clamp(1, 512) } /// The most f32 samples [`apply_roi`] holds at once for any one component. @@ -40,8 +47,8 @@ pub(crate) fn streamed_batch_rows(width: usize) -> usize { /// level it holds nothing, since it copies the LL window straight into the /// plane. When the tile streams its rows to a sink, every component's rings /// are live at once, with a scratch row each as wide as its final window, or -/// a batch of such rows from pool bands (see [`streamed_batch_rows`]), so the -/// peak is their sum. +/// two batches of such rows from pool bands (see [`streamed_batch_rows`]), so +/// the peak is their sum. pub(crate) fn roi_idwt_peak_elements( storage: &DecompositionStorage<'_>, skipped_resolution_levels: u8, @@ -77,9 +84,13 @@ pub(crate) fn roi_idwt_peak_elements( let rings = ring_samples(windows, WaveletTransform::Irreversible97)?; if storage.lockstep_lines { let scratch_row = roi_plan.final_window(component_idx).map_or(0, |window| { - let width = window.width() as usize; + let (width, height) = (window.width() as usize, window.height() as usize); if storage.parallel_bands { - width.saturating_mul(streamed_batch_rows(width)) + // Two batches, when each component reconstructs + // on its own reader. + width + .saturating_mul(streamed_batch_rows(width, height)) + .saturating_mul(2) } else { width } diff --git a/crates/j2k-native/src/tests/row_sink.rs b/crates/j2k-native/src/tests/row_sink.rs index cd86f709..0f1d4c9e 100644 --- a/crates/j2k-native/src/tests/row_sink.rs +++ b/crates/j2k-native/src/tests/row_sink.rs @@ -70,8 +70,20 @@ struct Decoded { streamed: bool, channel_bytes: usize, peak_bytes: usize, + #[cfg_attr( + not(feature = "parallel"), + expect(dead_code, reason = "only the pool band tests compare the counters") + )] pool_components: usize, + #[cfg_attr( + not(feature = "parallel"), + expect(dead_code, reason = "only the pool band tests compare the counters") + )] decoded_blocks: usize, + #[cfg_attr( + not(feature = "parallel"), + expect(dead_code, reason = "only the pool band tests compare the counters") + )] skipped_blocks: usize, } @@ -391,6 +403,13 @@ fn pool_band_rows_fit_every_components_clipped_blocks() { /// A sink with an RGB8 destination, which [`ComponentRowSink::write_rows`] /// fills the way the destination's contract says: each 8-bit sample rounded /// and saturated. +#[cfg_attr( + not(feature = "parallel"), + expect( + dead_code, + reason = "only the parallel band reader offers an RGB8 destination" + ) +)] struct Rgb8Sink { width: usize, rgb: Vec, From 37831ccf50c907bf6b89fac5f05605ff4ead9215 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 02:17:26 -0400 Subject: [PATCH 07/94] perf(j2k-native): deepen band rings for classic tiles read per component A classic tile whose components reconstruct on their own readers waits on classic Tier-1: at 18 threads its pool runs out of bands to start while the rings are full. Such a tile now holds one band more in every ring, and one batch of reconstructed rows instead of two, which more than pays for the bands. HTJ2K tiles keep the shallower rings and the second batch, which their readers use. Measured with region decodes to RGB8 (5 interleaved rounds, median of per-round ratios; output byte-identical at 1, 6 and 18 threads) on regions of an 18000 x 12000 RGB image: classic with PLT markers, classic without, and HTJ2K. Against the parent / against f33e40c6, at 18 threads: - Classic PLT 1024 x 1024: -4% to -6% / -49% to -56%. - Classic whole image at 1/16 and 1/8 scale: -4% / -65% and -57%. - Classic 2048 x 2048 at 1/2 and 4096 x 4096 at 1/4 scale, whole image at 1/4 scale, and 1024 x 1024 without PLT: -1% to +0.1% / -25% to -56%. - At 6 threads and for HTJ2K: within +-1%. - Peak heap: -2% to -11% against the parent on the classic rows at 18 threads; at most 0.98x of f33e40c6's on any row at 6 and 18 threads. --- crates/j2k-native/src/j2c/build/allocation.rs | 13 +++++++- crates/j2k-native/src/j2c/decode.rs | 5 ++++ .../src/j2c/decode/parallel_bands.rs | 30 +++++++++++++++++-- crates/j2k-native/src/j2c/decode/state.rs | 5 ++++ crates/j2k-native/src/j2c/idwt/roi.rs | 6 ++-- 5 files changed, 54 insertions(+), 5 deletions(-) diff --git a/crates/j2k-native/src/j2c/build/allocation.rs b/crates/j2k-native/src/j2c/build/allocation.rs index 804e7e94..a2956bd4 100644 --- a/crates/j2k-native/src/j2c/build/allocation.rs +++ b/crates/j2k-native/src/j2c/build/allocation.rs @@ -393,6 +393,11 @@ pub(crate) fn allocate_deferred_coefficients( pub(crate) const FINEST_LEVEL_BAND_SLOTS: u32 = 3; pub(crate) const OTHER_LEVEL_BAND_SLOTS: u32 = 2; +/// Band slots of each level with `deep_band_rings`: one more band of +/// lookahead for the pool's workers, held in place of a second batch of +/// reconstructed rows. +const DEEP_RING_EXTRA_SLOTS: u32 = 1; + /// Give each sub-band of a region decode the coefficient range for its /// stored rectangle: the required code-blocks, clipped to the IDWT windows /// unless `clip_to_idwt_windows` is false. With banded Tier-1, a range holds @@ -454,10 +459,15 @@ fn lay_out_region_coefficients( .sub_bands .contains(&sub_band_idx) }); - if finest { + let slots = if finest { FINEST_LEVEL_BAND_SLOTS } else { OTHER_LEVEL_BAND_SLOTS + }; + if storage.deep_band_rings { + slots + DEEP_RING_EXTRA_SLOTS + } else { + slots } } else { 1 @@ -505,6 +515,7 @@ pub(crate) fn lay_out_whole_region_coefficients( ) -> Result<()> { storage.banded_tier1 = false; storage.parallel_bands = false; + storage.deep_band_rings = false; let coefficient_count = lay_out_region_coefficients(storage, true, 0)?; let mut budget = reuse::ReallocationBudget::from_live_bytes(storage.structural_workspace_bytes)?; diff --git a/crates/j2k-native/src/j2c/decode.rs b/crates/j2k-native/src/j2c/decode.rs index 311705d0..ca977d2d 100644 --- a/crates/j2k-native/src/j2c/decode.rs +++ b/crates/j2k-native/src/j2c/decode.rs @@ -901,6 +901,11 @@ fn prepare_region_storage( } storage.lockstep_lines = (storage.banded_tier1 || storage.parallel_bands) && tile_ctx.stream_rows; + #[cfg(feature = "parallel")] + { + storage.deep_band_rings = + storage.parallel_bands && storage.lockstep_lines && parallel_bands::deep_rings(tile); + } build::allocate_deferred_coefficients( storage, header.skipped_resolution_levels, diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index 05572e20..adbae59e 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -374,12 +374,19 @@ pub(super) fn decode_tile_rows<'a>( }; // Rows reconstruct in batches, charged by the region's width (see // `roi_idwt_peak_elements`): two of them with a reader per component, so - // one finishes while the next reconstructs. + // one finishes while the next reconstructs, unless the band rings are + // deep instead. let batch = streamed_batch_rows(region.width() as usize, copy.height() as usize); let scratch_len = (copy.width() as usize) .checked_mul(components) .and_then(|samples| samples.checked_mul(batch)) - .and_then(|samples| samples.checked_mul(if readers > 1 { 2 } else { 1 })) + .and_then(|samples| { + samples.checked_mul(if readers > 1 && !storage.deep_band_rings { + 2 + } else { + 1 + }) + }) .ok_or(ValidationError::ImageTooLarge)?; let (mut buffer, ring_lens) = banded_tier1::take_line_buffer(tile_ctx, storage, &line_plans, scratch_len)?; @@ -494,6 +501,25 @@ pub(super) fn decode_tile_rows<'a>( } } +/// Whether a tile streaming its rows from pool bands holds deeper band rings +/// (see `DecompositionStorage::deep_band_rings`): a classic tile of several +/// components with the pool threads to reconstruct each on its own reader +/// (see [`parallel_synthesis`]). Its readers wait on classic Tier-1, which +/// more bands in flight keep busier; the memory comes from holding one batch +/// of rows instead of two. HTJ2K bands keep up with the shallower rings, +/// and its readers gain more from the second batch. +pub(super) fn deep_rings(tile: &Tile<'_>) -> bool { + tile.component_infos.len() > 1 + && rayon::current_num_threads() >= MIN_CLASSIC_SYNTHESIS_THREADS + && tile.component_infos.iter().all(|component| { + !component + .coding_style + .parameters + .code_block_style + .uses_high_throughput_block_coding() + }) +} + /// Pool threads a classic tile needs before its components reconstruct on /// the pool. A reader waiting for a band holds its thread, and classic /// Tier-1 keeps readers waiting, so a smaller pool does better with every diff --git a/crates/j2k-native/src/j2c/decode/state.rs b/crates/j2k-native/src/j2c/decode/state.rs index 948d376d..b311d78a 100644 --- a/crates/j2k-native/src/j2c/decode/state.rs +++ b/crates/j2k-native/src/j2c/decode/state.rs @@ -121,6 +121,10 @@ pub(crate) struct DecompositionStorage<'a> { /// IDWT on the worker pool (see [`parallel_bands`]). Each sub-band's /// `coefficients` then hold a ring of bands. pub(crate) parallel_bands: bool, + /// Whether the band rings of `parallel_bands` hold one band more at + /// every level, for a classic tile that streams its rows from a reader + /// per component (see [`parallel_bands::deep_rings`]). + pub(crate) deep_band_rings: bool, /// Whether every component's IDWT rings, and a scratch row each, are /// live at once, because the tile streams its rows to a sink. pub(crate) lockstep_lines: bool, @@ -162,6 +166,7 @@ impl DecompositionStorage<'_> { self.line_region_idwt = false; self.banded_tier1 = false; self.parallel_bands = false; + self.deep_band_rings = false; self.lockstep_lines = false; self.roi_idwt_charge_elements = 0; self.tier1_band = None; diff --git a/crates/j2k-native/src/j2c/idwt/roi.rs b/crates/j2k-native/src/j2c/idwt/roi.rs index 623acd20..5926c5f6 100644 --- a/crates/j2k-native/src/j2c/idwt/roi.rs +++ b/crates/j2k-native/src/j2c/idwt/roi.rs @@ -87,10 +87,12 @@ pub(crate) fn roi_idwt_peak_elements( let (width, height) = (window.width() as usize, window.height() as usize); if storage.parallel_bands { // Two batches, when each component reconstructs - // on its own reader. + // on its own reader, unless the band rings hold + // more bands instead. + let batches = if storage.deep_band_rings { 1 } else { 2 }; width .saturating_mul(streamed_batch_rows(width, height)) - .saturating_mul(2) + .saturating_mul(batches) } else { width } From d8ad4f5b02692f50187e6fdbc0346f944e898109 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 04:34:31 -0400 Subject: [PATCH 08/94] perf(j2k): pack 8-bit RGB region rows with the j2k-native RGB8 pack `write_rgb_component_rows_u8` rounded each sample with `f32::round`, which is a `roundf` call per sample on x86-64 and does not vectorize there. Rows whose three planes are 8-bit now go through `j2k_native::pack_rgb8` (exported `#[doc(hidden)]`), which packs at the CPU's SIMD level on x86-64; ARM64 keeps its auto-vectorized loop. Output is byte-identical. On x86-64 this commit was timed only together with the block-load commit and a region row-filter dispatch that was not kept, so its own share of the region and overview gains is not isolated. --- crates/j2k-native/src/color/mod.rs | 3 ++- crates/j2k-native/src/color/packing.rs | 2 +- crates/j2k-native/src/j2c/decode.rs | 3 ++- .../j2k-native/src/j2c/decode/banded_tier1.rs | 11 +++++++++- .../src/j2c/decode/parallel_bands.rs | 3 ++- .../src/j2c/decode/subband/tile_parallel.rs | 4 ++++ crates/j2k-native/src/j2c/idwt/lines.rs | 4 ++++ crates/j2k-native/src/lib.rs | 2 ++ crates/j2k-native/src/tests/row_sink.rs | 4 ++++ crates/j2k/src/decode/output/u8.rs | 21 +++++-------------- ...able-api-1.0.implementation-public-api.txt | 1 + 11 files changed, 37 insertions(+), 21 deletions(-) diff --git a/crates/j2k-native/src/color/mod.rs b/crates/j2k-native/src/color/mod.rs index cb7e3b54..058257a1 100644 --- a/crates/j2k-native/src/color/mod.rs +++ b/crates/j2k-native/src/color/mod.rs @@ -23,10 +23,11 @@ pub use output_planes::{ DecodedComponentsParts, DecodedNativeComponents, DecodedNativeComponentsParts, NativeComponentPlane, NativeComponentPlaneParts, }; +pub use packing::pack_rgb8; #[cfg(all(feature = "simd", target_arch = "x86_64"))] pub(crate) use packing::store_rgb8_x8; pub(crate) use packing::{ - interleave_and_convert_after_mct, interleave_and_convert_region, pack_rgb8, + interleave_and_convert_after_mct, interleave_and_convert_region, validate_interleaved_output_buffer, }; pub(crate) use palette::resolve_palette_indices; diff --git a/crates/j2k-native/src/color/packing.rs b/crates/j2k-native/src/color/packing.rs index 02c9079b..1797a292 100644 --- a/crates/j2k-native/src/color/packing.rs +++ b/crates/j2k-native/src/color/packing.rs @@ -301,7 +301,7 @@ pub(crate) fn interleave_and_convert( } /// Pack 8-bit RGB planes as interleaved bytes, rounded and saturated. -pub(crate) fn pack_rgb8(planes: [&[f32]; 3], output: &mut [u8]) { +pub fn pack_rgb8(planes: [&[f32]; 3], output: &mut [u8]) { #[cfg(all(feature = "simd", target_arch = "x86_64"))] { use fearless_simd::{dispatch, Level}; diff --git a/crates/j2k-native/src/j2c/decode.rs b/crates/j2k-native/src/j2c/decode.rs index ca977d2d..a9d974f4 100644 --- a/crates/j2k-native/src/j2c/decode.rs +++ b/crates/j2k-native/src/j2c/decode.rs @@ -986,7 +986,8 @@ enum TileOutput<'o, 'p> { /// planes, and with [`TileOutput::Sink`], the tile's rows go to the sink. #[expect( clippy::too_many_arguments, - reason = "the per-component stages share decode_tile's state, adapter, and profiling explicitly" + clippy::too_many_lines, + reason = "the per-component stages share decode_tile's state, adapter, and profiling explicitly, and the sink routes and the Tier-1, IDWT and store order stay in one place" )] fn decode_components<'a>( tile: &Tile<'a>, diff --git a/crates/j2k-native/src/j2c/decode/banded_tier1.rs b/crates/j2k-native/src/j2c/decode/banded_tier1.rs index e192c884..f5158e21 100644 --- a/crates/j2k-native/src/j2c/decode/banded_tier1.rs +++ b/crates/j2k-native/src/j2c/decode/banded_tier1.rs @@ -478,7 +478,12 @@ impl RowOutput<'_, '_> { None, )?; } - y += rows as u32; + #[expect( + clippy::cast_possible_truncation, + reason = "a batch holds at most the `copy.y1 - y` rows left, a u32" + )] + let rows = rows as u32; + y += rows; } } @@ -562,6 +567,10 @@ impl RowOutput<'_, '_> { /// sink. fn write_component_rows(&mut self, parts: &[&mut [f32]], y: u32, rows: usize) { let (copy, region) = (self.copy, self.region); + #[expect( + clippy::cast_possible_truncation, + reason = "`rows` is one batch of the copy rectangle's rows, at most its u32 height" + )] let component_rows: [ComponentPlane<'_>; 4] = core::array::from_fn(|index| ComponentPlane { samples: parts.get(index).map_or(&[][..], |part| &**part), diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index adbae59e..d9cc62e7 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -1018,7 +1018,8 @@ impl<'c, 'w, W> Shared<'_, 'c, 'w, W> { /// `false` when a band was stranded. #[expect( clippy::too_many_arguments, - reason = "the executor takes the plan, its storage and its two drivers explicitly" + clippy::too_many_lines, + reason = "the executor takes the plan, its storage and its two drivers explicitly, and sets up its shared state and runs its views in one scope" )] fn run_bands( (workers, readers, helps): (usize, usize, bool), diff --git a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs index 360559c3..5682cc24 100644 --- a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs +++ b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs @@ -1522,6 +1522,10 @@ pub(in crate::j2c::decode) fn plan_band_route<'a>( /// live beside it: the budget of the route planned last for the same /// executor, which holds every earlier route's charges. Without it, the /// route is charged to the decode's storage budget. +#[expect( + clippy::too_many_lines, + reason = "the route's coding, tile route, plan and cutoffs decide one band routing in order" +)] pub(in crate::j2c::decode) fn plan_band_route_beside<'a>( tile: &Tile<'a>, component_idx: usize, diff --git a/crates/j2k-native/src/j2c/idwt/lines.rs b/crates/j2k-native/src/j2c/idwt/lines.rs index 3ff112a8..88a7a9de 100644 --- a/crates/j2k-native/src/j2c/idwt/lines.rs +++ b/crates/j2k-native/src/j2c/idwt/lines.rs @@ -329,6 +329,10 @@ fn finish_row( } /// Interleave, filter horizontally and scale the next row of `levels[level]`. +#[expect( + clippy::too_many_lines, + reason = "one row's band reads, interleave, horizontal filter and scaling stay together for its whole-row and windowed paths" +)] fn feed_row(levels: &mut [LevelRows<'_>], level: usize, inputs: &mut Inputs<'_>) -> Result<()> { let (lower, upper) = levels.split_at_mut(level); let current = &mut upper[0]; diff --git a/crates/j2k-native/src/lib.rs b/crates/j2k-native/src/lib.rs index 92a2bb40..07ea9315 100644 --- a/crates/j2k-native/src/lib.rs +++ b/crates/j2k-native/src/lib.rs @@ -164,6 +164,8 @@ pub(crate) mod writer; use crate::math::{dispatch, Level}; #[cfg(test)] use color::cielab_to_rgb; +#[doc(hidden)] +pub use color::pack_rgb8; pub(crate) use color::{ convert_color_space, interleave_and_convert_after_mct, interleave_and_convert_region, resolve_alpha_and_color_space, resolve_palette_indices, validate_and_reorder_channels, diff --git a/crates/j2k-native/src/tests/row_sink.rs b/crates/j2k-native/src/tests/row_sink.rs index 0f1d4c9e..6ecba420 100644 --- a/crates/j2k-native/src/tests/row_sink.rs +++ b/crates/j2k-native/src/tests/row_sink.rs @@ -64,6 +64,10 @@ impl ComponentRowSink for CollectingSink { } } +#[expect( + clippy::struct_field_names, + reason = "decoded_blocks and skipped_blocks mirror the decoder's decoded and skipped code-block counters" +)] struct Decoded { planes: Vec>, depths: Vec<(u8, bool)>, diff --git a/crates/j2k/src/decode/output/u8.rs b/crates/j2k/src/decode/output/u8.rs index 3a655c9e..4cacb4d9 100644 --- a/crates/j2k/src/decode/output/u8.rs +++ b/crates/j2k/src/decode/output/u8.rs @@ -225,25 +225,14 @@ fn write_rgb_component_rows_u8( height: usize, ) { if planes[..3].iter().all(|plane| plane.bit_depth() == 8) { - // One indexed loop over whole pixels per row, which LLVM vectorizes. + // `sample_as_u8` at 8 bits is the RGB8 pack's rounding and saturation, + // which j2k-native runs at the CPU's SIMD level. for y in 0..height { let row = y * width..(y + 1) * width; - let (red, green, blue) = ( - &planes[0].samples()[row.clone()], - &planes[1].samples()[row.clone()], - &planes[2].samples()[row], + j2k_native::pack_rgb8( + [0, 1, 2].map(|channel| &planes[channel].samples()[row.clone()]), + &mut out[y * stride..y * stride + width * 3], ); - let pixels = out[y * stride..y * stride + width * 3] - .as_chunks_mut::<3>() - .0; - let len = pixels.len().min(red.len()).min(green.len()).min(blue.len()); - for x in 0..len { - pixels[x] = [ - sample_as_u8_depth8(red[x]), - sample_as_u8_depth8(green[x]), - sample_as_u8_depth8(blue[x]), - ]; - } } return; } diff --git a/xtask/api/stable-api-1.0.implementation-public-api.txt b/xtask/api/stable-api-1.0.implementation-public-api.txt index 0f439d90..cf5d0d7e 100644 --- a/xtask/api/stable-api-1.0.implementation-public-api.txt +++ b/xtask/api/stable-api-1.0.implementation-public-api.txt @@ -2629,6 +2629,7 @@ pub fn j2k_native::irreversible_quantization_profile_scale_ceiling(u8, &j2k_nati pub fn j2k_native::irreversible_quantization_scale_bounds(u8, &j2k_native::EncodeOptions) -> (f32, f32) pub fn j2k_native::irreversible_quantization_step_for_subband(u8, u8, f32, j2k_types::transform::quantization::IrreversibleQuantizationSubbandScales, j2k_types::tier1::classic::J2kSubBandType, u8, u8) -> j2k_types::transform::quantization::IrreversibleQuantizationStep pub fn j2k_native::pack_j2k_code_block_scalar_from_tier1_tokens(&[u8], &[j2k_native::J2kTier1TokenSegment], u8, u8) -> j2k_native::EncodeResult +pub fn j2k_native::pack_rgb8([&[f32]; 3], &mut [u8]) pub fn j2k_native::packet_math::HtSegmentLengthError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub fn j2k_native::packet_math::bits_for_ht_cleanup_length(u32, u8) -> u32 pub fn j2k_native::packet_math::bits_for_ht_refinement_only_length(u32, u8) -> u32 From 28365aabb588b0487e4abc59ecc26352b8e1a131 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 04:34:31 -0400 Subject: [PATCH 09/94] docs(changelog): note the region row speedups on the pool bands --- CHANGELOG.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2749a9e..5818dac3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -94,6 +94,45 @@ Staged workspace version: `0.13.2`. faster lossless at 1 and 6 threads (5% and 13% at 18), and 1024 x 1024 regions and reduced-resolution decodes of an 18000 x 12000 image 2 to 10% faster. Output and peak heap are unchanged. +- Parallel CPU region decodes of classic tiles whose rows stream to a row + sink run every component's bands on the pool at once, and the IDWT + advances the components together and hands the rows to the sink, instead + of decoding one component's bands, then its planes, at a time. At reduced + resolutions, where each sub-band holds few bands, one component's bands + left most of the pool idle. On an 18000 x 12000 RGB image, overviews at + 1/16 and 1/8 scale decode 53% and 47% faster at 18 threads and 1024 x 1024 + regions up to 5% faster, with 29 to 59% less peak heap. HTJ2K tiles are + unchanged. +- The row-streamed region IDWT synthesizes rows with the banded IDWT's SIMD + horizontal kernel, reading the stored band rows in place, instead of + interleaving each row and lifting it with the scalar filter. Output is + bit-identical. On an 18000 x 12000 RGB image, HTJ2K regions decode 27 to + 52% faster and classic 1024 x 1024 regions 16 to 36% faster at 6 and 18 + threads, and 3 to 35% faster at one thread. +- Parallel CPU region decodes streaming rows from pool bands reconstruct + their components in batches of rows, each component on its own pool task + for HTJ2K tiles and for classic tiles with 12 threads or more, and pack + three 8-bit components straight into interleaved RGB8 output. HTJ2K tiles + now stream rows from pool bands too. `ComponentRowSink` gains a + rustdoc-hidden `rgb8_region` method, `None` by default, through which a + sink offers such a destination. On an 18000 x 12000 RGB image the HTJ2K + overview at 1/4 scale decodes 32 to 35% faster, HTJ2K 1024 x 1024 regions + 16 to 20%, and classic 1024 x 1024 regions at 18 threads 9 to 18%. +- Parallel CPU region decodes with a reader per component transform and pack + each batch of rows on every reader's thread instead of the calling thread + alone. The HTJ2K overview at 1/4 scale of an 18000 x 12000 RGB image + decodes 10 to 15% faster. +- Parallel CPU region decodes with a reader per component transform and pack + one batch of rows on the pool while the next reconstructs, in batches of + up to 2 Mi samples per component and a quarter of the tile's rows; a + single reader takes a row of each component at a time. On an 18000 x 12000 + RGB image the HTJ2K overviews at 1/4 and 1/8 scale decode 6 to 29% faster, + and the classic overview at 1/16 scale 12% faster at 6 threads. +- Parallel CPU region decodes of classic tiles that reconstruct each + component on its own reader hold one band more in every ring and one + batch of rows instead of two. On an 18000 x 12000 RGB image, 1024 x 1024 + regions and overviews at 1/8 and 1/16 scale decode 4 to 6% faster at 18 + threads, with less peak heap. ## [0.13.1] - 2026-10-29 From d97dc21df02859f9c63a1859f109379d1fde4a87 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 23:30:34 -0400 Subject: [PATCH 10/94] fix(j2k): stream over-budget Gray8/RGB8 batch items instead of rejecting them A prepared item whose offset-plan execution exceeds the per-decode bound (the 5661 x 8287 RGB8 scan: 563 MB of planes; RGB from 4096 x 4096, gray from 6656 x 6656) failed with ImageTooLarge although the facade's full-output region route decodes it in bounded memory. Preparation now marks such Gray8/RGB8 items for streamed execution, a choice made from the plan before any decode work. They decode through the region rows route decode_region_into uses (whole output rectangle, at the requested scale) into a row sink that writes the batch's NCHW/NHWC samples with the same rounding as whole planes, and claim one worst-case native decode of admission while in flight (no planned figure for that route yet). Groups holding one never take the staged route. Other outputs keep the typed prepare rejection. Items that fit keep their plans, bytes and times. --- crates/j2k/src/owned_batch/cpu_execute.rs | 15 +- crates/j2k/src/owned_batch/cpu_materialize.rs | 151 +++++++++++++++++- .../prepare/cpu_preflight_tests.rs | 116 +++++++++++--- crates/j2k/src/owned_batch/prepare/image.rs | 28 +++- crates/j2k/src/owned_batch/prepared.rs | 14 +- 5 files changed, 290 insertions(+), 34 deletions(-) diff --git a/crates/j2k/src/owned_batch/cpu_execute.rs b/crates/j2k/src/owned_batch/cpu_execute.rs index 9674a0b1..bd24cbfa 100644 --- a/crates/j2k/src/owned_batch/cpu_execute.rs +++ b/crates/j2k/src/owned_batch/cpu_execute.rs @@ -10,9 +10,9 @@ use super::cpu_staged_execute::run_staged_typed_group; use super::{ decode_warnings_for_recovery, retained_inner_parallelism, run_retained_chunks, Arc, BatchDecodeOptions, BatchErrorStage, BatchGroupInfo, BatchInfrastructureError, BatchItemError, - BatchWorker, CpuBatchGroup, CpuBatchSamples, CpuDecodeParallelism, CpuGroupFastWorkspace, - CpuStagedWorkspace, IndexedBatchError, J2kError, NativeSampleType, NonZeroUsize, - PreparedBatchGroup, PreparedImage, TileBatchOptions, Vec, + BatchWorker, CpuBatchGroup, CpuBatchSamples, CpuDecodeParallelism, CpuExecution, + CpuGroupFastWorkspace, CpuStagedWorkspace, IndexedBatchError, J2kError, NativeSampleType, + NonZeroUsize, PreparedBatchGroup, PreparedImage, TileBatchOptions, Vec, }; use crate::batch::admission::{hold_at_least, BatchAllocationBudget}; use crate::batch::allocation::try_vec_with_capacity; @@ -37,7 +37,14 @@ pub(super) fn decode_cpu_group( // workers. When workers may parallelize inside one image, each prepared // plan decodes faster on its own: its code-block rows and wavelet bands // spread over every core, still without reparsing. - let has_flattened_payloads = fast_workspace.prepare_group(group)? + // Streamed images never hold whole-image scratch, so a group with one + // never enters the staged route. + let streams_rows = group + .images + .iter() + .any(|image| image.cpu_execution() == CpuExecution::StreamedRows); + let has_flattened_payloads = !streams_rows + && fast_workspace.prepare_group(group)? && retained_inner_parallelism(group.images.len(), tile_options)? == CpuDecodeParallelism::Serial; let samples_per_image = diff --git a/crates/j2k/src/owned_batch/cpu_materialize.rs b/crates/j2k/src/owned_batch/cpu_materialize.rs index f9dda639..6b5340f3 100644 --- a/crates/j2k/src/owned_batch/cpu_materialize.rs +++ b/crates/j2k/src/owned_batch/cpu_materialize.rs @@ -91,8 +91,20 @@ pub(super) fn decode_image_u8<'image>( worker: &mut BatchWorker, out: &mut [u8], ) -> Result<(), J2kError> { - if image.cpu_execution() == CpuExecution::ExceedsDecodeBound { - return Err(CpuExecution::decode_bound_error()); + match image.cpu_execution() { + CpuExecution::ExceedsDecodeBound => return Err(CpuExecution::decode_bound_error()), + CpuExecution::StreamedRows => { + return decode_streamed_samples( + image, + options, + info, + parallelism, + context, + out, + convert_u8, + ); + } + CpuExecution::OffsetPlan { .. } | CpuExecution::ComponentPlanes => {} } if decode_prepared_plan_samples(image, info, worker, out, |sample, precision| { u8::try_from(round_unsigned(sample, precision)).unwrap_or(u8::MAX) @@ -153,6 +165,141 @@ pub(super) fn decode_image_i16<'image>( Ok(()) } +/// Decode `image` through the bounded region decode, which hands over its +/// rows as they are finished, writing each sample exactly as +/// [`write_samples`] writes whole planes. Working memory stays bounded by the +/// streamed decode instead of growing with the image. +fn decode_streamed_samples<'image, T: Copy>( + image: &'image PreparedImage, + options: BatchDecodeOptions, + info: &BatchGroupInfo, + parallelism: CpuDecodeParallelism, + context: &mut j2k_native::DecoderContext<'image>, + out: &mut [T], + convert: fn(f32, u8) -> T, +) -> Result<(), J2kError> { + context.set_cpu_decode_parallelism(parallelism.to_native()); + let plan = image.plan(); + let decoded = backend::image_at_scale(image.bytes(), options.settings, plan.scale())?; + let output_rect = plan.output_rect(); + if (output_rect.w, output_rect.h) != info.dimensions { + return Err(J2kError::internal_backend( + "prepared batch output metadata changed during decode", + )); + } + let mut sink = SampleRowSink::new(info, out, convert)?; + decoded + .decode_region_rows_for_integer_output_with_context( + (output_rect.x, output_rect.y, output_rect.w, output_rect.h), + context, + &mut sink, + ) + .map_err(J2kError::from_native_decode_error)?; + sink.finish() +} + +/// Writes streamed region rectangles into one image's dense batch samples. +struct SampleRowSink<'out, 'info, T> { + info: &'info BatchGroupInfo, + out: &'out mut [T], + convert: fn(f32, u8) -> T, + pixels: usize, + written_pixels: usize, + malformed: bool, +} + +impl<'out, 'info, T: Copy> SampleRowSink<'out, 'info, T> { + fn new( + info: &'info BatchGroupInfo, + out: &'out mut [T], + convert: fn(f32, u8) -> T, + ) -> Result { + let pixels = (info.dimensions.0 as usize) + .checked_mul(info.dimensions.1 as usize) + .ok_or_else(|| J2kError::internal_backend("prepared batch sample count overflow"))?; + if pixels.checked_mul(info.color.channels()) != Some(out.len()) { + return Err(J2kError::internal_backend( + "prepared batch output slot does not match its image", + )); + } + Ok(Self { + info, + out, + convert, + pixels, + written_pixels: 0, + malformed: false, + }) + } + + /// Every pixel arrived exactly once (rectangles never overlap). + fn finish(self) -> Result<(), J2kError> { + if self.malformed || self.written_pixels != self.pixels { + return Err(J2kError::internal_backend( + "streamed batch decode did not cover its output exactly", + )); + } + Ok(()) + } + + fn write_rectangle(&mut self, x: u32, y: u32, planes: &[j2k_native::ComponentPlane<'_>]) { + let channels = self.info.color.channels(); + let width = self.info.dimensions.0 as usize; + let Some((rect_width, rect_height)) = planes.first().map(|plane| plane.dimensions()) else { + self.malformed = true; + return; + }; + let (x, y, rect_width, rect_height) = ( + x as usize, + y as usize, + rect_width as usize, + rect_height as usize, + ); + let rect_pixels = rect_width * rect_height; + if planes.len() != channels + || x + rect_width > width + || y + rect_height > self.info.dimensions.1 as usize + || planes.iter().any(|plane| { + plane.dimensions() != (rect_width as u32, rect_height as u32) + || plane.samples().len() < rect_pixels + }) + { + self.malformed = true; + return; + } + let precision = self.info.precision; + let planar = self.info.layout == BatchLayout::Nchw || channels == 1; + for (channel, plane) in planes.iter().enumerate() { + for (row, samples) in plane.samples()[..rect_pixels] + .chunks_exact(rect_width) + .enumerate() + { + let pixel = (y + row) * width + x; + if planar { + let destination = &mut self.out[channel * self.pixels + pixel..][..rect_width]; + for (destination, &sample) in destination.iter_mut().zip(samples) { + *destination = (self.convert)(sample, precision); + } + } else { + let destination = &mut self.out[pixel * channels..][..rect_width * channels]; + for (pixel, &sample) in destination.chunks_exact_mut(channels).zip(samples) { + pixel[channel] = (self.convert)(sample, precision); + } + } + } + } + self.written_pixels += rect_pixels; + } +} + +impl j2k_native::ComponentRowSink for SampleRowSink<'_, '_, T> { + fn write_rows(&mut self, x: u32, y: u32, planes: &[j2k_native::ComponentPlane<'_>]) { + if !self.malformed { + self.write_rectangle(x, y, planes); + } + } +} + fn validate_decoded_components( components: &j2k_native::DecodedComponents<'_>, info: &BatchGroupInfo, diff --git a/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs b/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs index 0395bb08..26677765 100644 --- a/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs +++ b/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs @@ -4,26 +4,45 @@ use alloc::{sync::Arc, vec, vec::Vec}; use core::num::NonZeroUsize; use super::super::{ - prepare_batch, BatchDecodeOptions, BatchErrorStage, BatchItemError, CpuBatchDecoder, - CpuBatchSamples, CpuExecution, EncodedImage, IndexedBatchError, PreparedBatch, + prepare_batch, BatchDecodeOptions, BatchErrorStage, BatchItemError, BatchLayout, + CpuBatchDecoder, CpuBatchSamples, CpuExecution, DecodeRequest, EncodedImage, IndexedBatchError, + PreparedBatch, }; use super::image::PLANNED_DECODE_BOUND; +use j2k_core::Downscale; -fn classic_rgb8(width: u32, height: u32) -> Arc<[u8]> { - let samples = (0..width * height * 3) - .map(|index| u8::try_from((index * 7 + index / 97) & 0xff).expect("masked sample")) - .collect::>(); +fn classic_fixture(width: u32, height: u32, components: u16, precision: u8) -> Arc<[u8]> { + let count = width * height * u32::from(components); + let samples = if precision <= 8 { + (0..count) + .map(|index| u8::try_from((index * 7 + index / 97) & 0xff).expect("masked sample")) + .collect::>() + } else { + (0..count) + .flat_map(|index| { + u16::try_from((index * 977 + 31) % (1 << precision)) + .expect("masked sample") + .to_le_bytes() + }) + .collect() + }; let options = j2k_native::EncodeOptions { reversible: true, num_decomposition_levels: 3, ..j2k_native::EncodeOptions::default() }; Arc::from( - j2k_native::encode(&samples, width, height, 3, 8, false, &options) - .expect("encode classic RGB8 fixture"), + j2k_native::encode( + &samples, width, height, components, precision, false, &options, + ) + .expect("encode classic fixture"), ) } +fn classic_rgb8(width: u32, height: u32) -> Arc<[u8]> { + classic_fixture(width, height, 3, 8) +} + /// One worker keeps preparation and decode on this thread, where the /// planned-bound override applies. fn single_worker_options() -> BatchDecodeOptions { @@ -79,25 +98,87 @@ fn is_decode_bound_error(error: &IndexedBatchError, index: usize, stage: BatchEr ) } +fn single_worker_options_with(layout: BatchLayout) -> BatchDecodeOptions { + let mut options = single_worker_options(); + options.layout = layout; + options +} + +fn all_u8_samples(result: &super::super::CpuBatchDecodeResult) -> Vec<(Vec, Vec)> { + assert!(result.errors().is_empty(), "{:?}", result.errors()); + let mut groups = result + .groups() + .iter() + .map(|group| { + let CpuBatchSamples::U8(samples) = group.samples() else { + panic!("8-bit decodes to u8 samples"); + }; + (group.source_indices().to_vec(), samples.clone()) + }) + .collect::>(); + groups.sort(); + groups +} + #[test] -fn cpu_prepare_rejects_plans_over_the_decode_bound_before_decoding() { +fn cpu_batch_streams_rgb8_and_gray8_plans_over_the_decode_bound() { let small = classic_rgb8(16, 16); let large = classic_rgb8(96, 64); + let gray = classic_fixture(80, 72, 1, 8); let inputs = || { vec![ EncodedImage::full(Arc::clone(&small)), EncodedImage::full(Arc::clone(&large)), + EncodedImage::full(Arc::clone(&gray)), + EncodedImage::new( + Arc::clone(&large), + DecodeRequest::Reduced { + scale: Downscale::Half, + }, + ), + ] + }; + for layout in [BatchLayout::Nchw, BatchLayout::Nhwc] { + let mut decoder = CpuBatchDecoder::new(single_worker_options_with(layout)); + let unbounded = decoder.prepare(inputs()).expect("prepare without a bound"); + let planned = planned_bytes(&unbounded); + let CpuExecution::OffsetPlan { bytes: small_bytes } = planned[0].1 else { + panic!("the small image keeps its offset plan: {planned:?}"); + }; + let reference = all_u8_samples( + &decoder + .decode_prepared(&unbounded) + .expect("decode without a bound"), + ); + + let _bound = PlannedBound::hold(small_bytes); + let bounded = decoder.prepare(inputs()).expect("prepare under the bound"); + assert!(bounded.errors().is_empty(), "{:?}", bounded.errors()); + let planned = planned_bytes(&bounded); + assert!(matches!(planned[0].1, CpuExecution::OffsetPlan { .. })); + for (index, execution) in &planned[1..] { + assert_eq!(*execution, CpuExecution::StreamedRows, "input {index}"); + } + let streamed = all_u8_samples(&decoder.decode_prepared(&bounded).expect("decode")); + assert_eq!(streamed, reference, "{layout:?}"); + } +} + +#[test] +fn cpu_prepare_rejects_plans_over_the_bound_without_a_streamed_route() { + let small = classic_rgb8(16, 16); + let gray12 = classic_fixture(96, 64, 1, 12); + let inputs = || { + vec![ + EncodedImage::full(Arc::clone(&small)), + EncodedImage::full(Arc::clone(&gray12)), ] }; let mut decoder = CpuBatchDecoder::new(single_worker_options()); let unbounded = decoder.prepare(inputs()).expect("prepare without a bound"); - let planned = planned_bytes(&unbounded); - let [(0, CpuExecution::OffsetPlan { bytes: small_bytes }), (1, CpuExecution::OffsetPlan { bytes: large_bytes })] = - planned[..] - else { - panic!("both images keep offset plans: {planned:?}"); + let CpuExecution::OffsetPlan { bytes: small_bytes } = planned_bytes(&unbounded)[0].1 else { + panic!("the small image keeps its offset plan"); }; - assert!(small_bytes < large_bytes, "{small_bytes} < {large_bytes}"); let reference = decoder .decode_prepared(&unbounded) .expect("decode without a bound"); @@ -111,7 +192,6 @@ fn cpu_prepare_rejects_plans_over_the_decode_bound_before_decoding() { 1, BatchErrorStage::Prepare )); - assert_eq!(planned_bytes(&rejected).len(), 1); let output = decoder .decode_prepared(&rejected) .expect("decode the admitted image"); @@ -137,8 +217,4 @@ fn cpu_prepare_rejects_plans_over_the_decode_bound_before_decoding() { 1, BatchErrorStage::Decode )); - assert_eq!( - u8_samples(output.groups(), 0), - u8_samples(reference.groups(), 0) - ); } diff --git a/crates/j2k/src/owned_batch/prepare/image.rs b/crates/j2k/src/owned_batch/prepare/image.rs index 3bb05b39..d92360e8 100644 --- a/crates/j2k/src/owned_batch/prepare/image.rs +++ b/crates/j2k/src/owned_batch/prepare/image.rs @@ -54,7 +54,7 @@ pub(super) fn prepare_image( .map_or(PreparedCodecPlan::MetadataOnly, PreparedCodecPlan::Htj2k), }; reconcile_codec_plan_metadata(&mut info, &codec_plan)?; - let cpu_execution = choose_cpu_execution(&codec_plan); + let cpu_execution = choose_cpu_execution(&codec_plan, &info); let preparation_depth = codec_plan.preparation_depth(); let execution_shape = batch_execution_shape(&support, plan, preparation_depth); let image = PreparedImage { @@ -77,9 +77,13 @@ pub(super) fn prepare_image( /// Choose the CPU execution from the planned working memory, before any /// decode work, with the same accounting the offset plan's execution /// enforces. Only an image whose serial execution already exceeds the -/// per-decode bound is [`CpuExecution::ExceedsDecodeBound`], so the choice -/// rejects nothing execution could decode; the claim is the parallel peak. -pub(super) fn choose_cpu_execution(codec_plan: &PreparedCodecPlan) -> CpuExecution { +/// per-decode bound leaves its offset plan, so every image the plan can +/// execute keeps it; such an image streams its rows when its output has a +/// streamed route and is rejected otherwise. The claim is the parallel peak. +pub(super) fn choose_cpu_execution( + codec_plan: &PreparedCodecPlan, + info: &BatchGroupInfo, +) -> CpuExecution { let planned = |parallel| match codec_plan { PreparedCodecPlan::Classic(plan) => { j2k_native::referenced_classic_plan_execution_bytes(plan.cpu_plan(), parallel) @@ -92,12 +96,17 @@ pub(super) fn choose_cpu_execution(codec_plan: &PreparedCodecPlan) -> CpuExecuti if matches!(codec_plan, PreparedCodecPlan::MetadataOnly) { return CpuExecution::ComponentPlanes; } + let exceeds_bound = if streams_rows(info) { + CpuExecution::StreamedRows + } else { + CpuExecution::ExceedsDecodeBound + }; match (planned(false), planned(true)) { ( Err(j2k_native::DecodeError::Validation(j2k_native::ValidationError::ImageTooLarge)), _, - ) => CpuExecution::ExceedsDecodeBound, - (Ok(serial), _) if serial > planned_decode_bound() => CpuExecution::ExceedsDecodeBound, + ) => exceeds_bound, + (Ok(serial), _) if serial > planned_decode_bound() => exceeds_bound, (Ok(_), Ok(bytes)) => CpuExecution::OffsetPlan { bytes }, // A plan that fits only serially, or a malformed plan, keeps the // worst-case claim and reports any error from execution, as before. @@ -107,6 +116,13 @@ pub(super) fn choose_cpu_execution(codec_plan: &PreparedCodecPlan) -> CpuExecuti } } +/// Gray8 and RGB8 outputs have a streamed route: the region decode hands +/// over finished rows of 8-bit components. Other outputs keep the bound. +fn streams_rows(info: &BatchGroupInfo) -> bool { + info.sample_type == NativeSampleType::U8 + && matches!(info.color, BatchColor::Gray | BatchColor::Rgb) +} + #[cfg(test)] std::thread_local! { /// Test-only lower bound for [`choose_cpu_execution`] on this thread. diff --git a/crates/j2k/src/owned_batch/prepared.rs b/crates/j2k/src/owned_batch/prepared.rs index cb97967f..ede71129 100644 --- a/crates/j2k/src/owned_batch/prepared.rs +++ b/crates/j2k/src/owned_batch/prepared.rs @@ -43,8 +43,13 @@ pub(super) enum CpuExecution { /// Reparse and decode whole component planes; no offset plan exists. ComponentPlanes, /// The planned offset-plan execution exceeds the per-decode memory - /// bound, so the CPU decoder rejects the image before any decode work. + /// bound and the output has no streamed route (not Gray8 or RGB8), so the + /// CPU decoder rejects the image before any decode work. ExceedsDecodeBound, + /// The planned offset-plan execution exceeds the per-decode memory + /// bound, so the image streams rows from the full-output region decode, + /// whose working memory does not grow with the image. + StreamedRows, } impl CpuExecution { @@ -53,7 +58,12 @@ impl CpuExecution { pub(super) const fn claim_bytes(self) -> usize { match self { Self::OffsetPlan { bytes } => bytes, - Self::ComponentPlanes => crate::batch::allocation::GENERIC_WORKER_CLAIM_BYTES, + // The streamed route has no planned peak yet; it is claimed as one + // worst-case native decode (measured: 95 MiB for a 5661 x 8287 + // RGB8 scan at default threads). + Self::ComponentPlanes | Self::StreamedRows => { + crate::batch::allocation::GENERIC_WORKER_CLAIM_BYTES + } Self::ExceedsDecodeBound => 0, } } From 6b30a0c7f548da5c70dd9b8f5114588f878b5a73 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 23:30:34 -0400 Subject: [PATCH 11/94] fix(j2k): split owned batch groups whose output exceeds one allocation A homogeneous group's dense output larger than the per-allocation cap (eight 5792^2 RGB8 images, 805 MB) failed the whole decode call with AllocationTooLarge. Such a group now decodes as several groups with the same info and consecutive images, each buffer within the cap. Groups that fit are unchanged; an image whose own output exceeds the cap still fails. --- crates/j2k/src/owned_batch/cpu.rs | 119 +++++++++++++++++++++++++++--- 1 file changed, 107 insertions(+), 12 deletions(-) diff --git a/crates/j2k/src/owned_batch/cpu.rs b/crates/j2k/src/owned_batch/cpu.rs index 81115c49..73e84638 100644 --- a/crates/j2k/src/owned_batch/cpu.rs +++ b/crates/j2k/src/owned_batch/cpu.rs @@ -7,7 +7,7 @@ use super::{ prepare_batch_with_workers, BatchDecodeOptions, BatchDecoder, BatchGroupInfo, BatchInfrastructureError, BatchWorker, CpuBatchWorkspaceStats, CpuGroupFastWorkspace, CpuStagedWorkspace, EncodedImage, IndexedBatchError, J2kDecodeWarning, Mutex, NativeSampleType, - NonZeroUsize, PrepareFor, PreparedBatch, PreparedImage, Rect, Vec, + NonZeroUsize, PrepareFor, PreparedBatch, PreparedBatchGroup, PreparedImage, Rect, Vec, }; use crate::batch::admission::BatchAllocationBudget; use crate::batch::allocation::try_vec_with_capacity; @@ -392,17 +392,57 @@ impl CpuBatchDecoder { // plan, under the same aggregate cap as the tile-batch facade. let admission = BatchAllocationBudget::with_baseline(0)?; for group in prepared.groups() { - if let Some(decoded) = decode_cpu_group( - workers, - &mut self.fast_workspace, - &mut self.staged_workspace, - group, - prepared.options, - self.options.workers, - &admission, - &mut errors, - )? { - groups.push(decoded); + let images_per_group = output_images_per_group(group); + if images_per_group >= group.images.len() { + if let Some(decoded) = decode_cpu_group( + workers, + &mut self.fast_workspace, + &mut self.staged_workspace, + group, + prepared.options, + self.options.workers, + &admission, + &mut errors, + )? { + push_group(&mut groups, decoded)?; + } + continue; + } + // One dense buffer for the whole group would exceed the + // per-allocation cap: decode consecutive images as several + // groups with the same info, each buffer within the cap. + for (images, source_indices) in group + .images + .chunks(images_per_group) + .zip(group.source_indices.chunks(images_per_group)) + { + let mut part_images = + try_vec_with_capacity(images.len(), "J2K owned batch output group images")?; + part_images.extend(images.iter().cloned()); + let mut part_indices = try_vec_with_capacity( + source_indices.len(), + "J2K owned batch output group source indices", + )?; + part_indices.extend_from_slice(source_indices); + let part = PreparedBatchGroup { + info: group.info.clone(), + options: group.options, + execution_shape: group.execution_shape, + images: part_images, + source_indices: part_indices, + }; + if let Some(decoded) = decode_cpu_group( + workers, + &mut self.fast_workspace, + &mut self.staged_workspace, + &part, + prepared.options, + self.options.workers, + &admission, + &mut errors, + )? { + push_group(&mut groups, decoded)?; + } } } errors.sort_by_key(|error| error.index); @@ -410,6 +450,44 @@ impl CpuBatchDecoder { } } +/// Images whose dense outputs fit one per-allocation-capped buffer together +/// (at least one). An image whose own output exceeds the cap still fails in +/// its own group, as before. +fn output_images_per_group(group: &PreparedBatchGroup) -> usize { + let sample_bytes = match group.info.sample_type { + NativeSampleType::U8 => 1, + NativeSampleType::U16 | NativeSampleType::I16 => 2, + _ => return group.images.len(), + }; + group + .info + .samples_per_image() + .and_then(|samples| samples.checked_mul(sample_bytes)) + .map_or(group.images.len(), |image_bytes| { + images_within_cap(image_bytes, j2k_core::DEFAULT_MAX_HOST_ALLOCATION_BYTES) + }) +} + +/// How many images of `image_bytes` one buffer of at most `cap` bytes holds, +/// never fewer than one. +fn images_within_cap(image_bytes: usize, cap: usize) -> usize { + cap.checked_div(image_bytes).unwrap_or(usize::MAX).max(1) +} + +fn push_group( + groups: &mut Vec, + group: CpuBatchGroup, +) -> Result<(), BatchInfrastructureError> { + groups + .try_reserve(1) + .map_err(|_| BatchInfrastructureError::HostAllocationFailed { + what: "J2K owned batch result groups", + bytes: core::mem::size_of::(), + })?; + groups.push(group); + Ok(()) +} + impl BatchDecoder for CpuBatchDecoder { type Output = CpuBatchDecodeResult; type Error = BatchInfrastructureError; @@ -433,3 +511,20 @@ impl BatchDecoder for CpuBatchDecoder { Self::decode_prepared(self, prepared) } } + +#[cfg(test)] +mod output_group_tests { + use super::images_within_cap; + + #[test] + fn groups_above_the_cap_split_into_buffers_within_it() { + const MIB: usize = 1024 * 1024; + // Eight 5792² RGB8 images (96 MiB each) under a 512 MiB cap. + assert_eq!(images_within_cap(96 * MIB, 512 * MIB), 5); + assert_eq!(images_within_cap(512 * MIB, 512 * MIB), 1); + // An image above the cap keeps a group of its own. + assert_eq!(images_within_cap(768 * MIB, 512 * MIB), 1); + assert_eq!(images_within_cap(3 * MIB, 512 * MIB), 170); + assert_eq!(images_within_cap(0, 512 * MIB), usize::MAX); + } +} From ba58f1184c17b33f994fe2da81e020d438227ded Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 04:39:07 -0400 Subject: [PATCH 12/94] perf(j2k-native): decode multi-layer classic plans' code-block rows in parallel Parallel referenced classic dispatch borrowed every payload as one input slice and fell back to serial code-block decode when a block's bytes were split over fragments. Multi-layer codestreams (one fragment per contributing layer) therefore decoded every block on one thread; only the wavelet steps ran in parallel. Row jobs now carry their payload's fragment ranges, and each row task joins fragmented payloads in its own buffer: the main task in the scratch's combine buffer, every extra task in a buffer kept beside its code-block workspace. Each task buffer is planned and reserved at the plan's largest fragmented payload (none when every payload is one input slice), so planning, admission and the retained-scratch accounting see it. Every block decodes exactly as on the serial path; samples are bit-identical. --- crates/j2k-native/src/direct_cpu.rs | 22 +- .../j2k-native/src/direct_cpu/allocation.rs | 50 +++-- .../src/direct_cpu/allocation/referenced.rs | 10 +- .../direct_cpu/allocation/referenced/view.rs | 29 +-- .../src/direct_cpu/referenced/component.rs | 9 +- .../src/direct_cpu/referenced/rows.rs | 59 +++--- .../src/direct_cpu/referenced_classic.rs | 188 +++++++++++------- .../direct_cpu/referenced_classic/tests.rs | 139 +++++++++---- 8 files changed, 328 insertions(+), 178 deletions(-) diff --git a/crates/j2k-native/src/direct_cpu.rs b/crates/j2k-native/src/direct_cpu.rs index f3b8fd14..83edd3e1 100644 --- a/crates/j2k-native/src/direct_cpu.rs +++ b/crates/j2k-native/src/direct_cpu.rs @@ -95,12 +95,21 @@ pub struct J2kDirectCpuScratch { ht_task_workspaces: Vec, /// Extra classic workspaces for parallel referenced classic execution, /// one per additional Rayon worker. - classic_task_workspaces: Vec, + classic_task_workspaces: Vec, /// Whether referenced execution may spread work over the Rayon pool. parallel: bool, staged_state: Option, } +/// One extra Rayon worker's state for parallel referenced classic execution: +/// its code-block workspace and the buffer it joins a code block's payload +/// fragments into when a multi-layer codestream splits them. +#[derive(Debug, Default)] +struct ClassicTaskWorkspace { + decode: J2kCodeBlockDecodeWorkspace, + payload: Vec, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum StagedDirectRoute { Classic, @@ -170,12 +179,21 @@ impl J2kDirectCpuScratch { .iter() .try_fold( self.classic_workspace.allocated_bytes().ok(), - |bytes, workspace| Some(bytes?.checked_add(workspace.allocated_bytes().ok()?)), + |bytes, task| Some(bytes?.checked_add(task.decode.allocated_bytes().ok()?)), ) .flatten() .unwrap_or(usize::MAX) } + /// Length and capacity of each extra classic task's payload buffer. + #[cfg(test)] + fn classic_task_payloads(&self) -> Vec<(usize, usize)> { + self.classic_task_workspaces + .iter() + .map(|task| (task.payload.len(), task.payload.capacity())) + .collect() + } + /// Number of coefficient/IDWT buffer owners retained by this scratch. /// /// Referenced multi-tile execution retains the maximum live tile shape, diff --git a/crates/j2k-native/src/direct_cpu/allocation.rs b/crates/j2k-native/src/direct_cpu/allocation.rs index 6f7c22fa..656942eb 100644 --- a/crates/j2k-native/src/direct_cpu/allocation.rs +++ b/crates/j2k-native/src/direct_cpu/allocation.rs @@ -452,8 +452,9 @@ fn parallel_ht_task_workspace_count( } /// Largest code block and most code-block rows of any sub-band a parallel -/// referenced classic execution decodes. One dispatch decodes at most one -/// row per Rayon worker, so it never needs more workspaces than rows. +/// referenced classic execution decodes, and its largest payload split over +/// several input fragments. One dispatch decodes at most one row per Rayon +/// worker, so it never needs more workspaces than rows. #[derive(Debug, Clone, Copy)] struct ClassicTaskShape { dimensions: (u32, u32), @@ -462,6 +463,9 @@ struct ClassicTaskShape { expect(dead_code, reason = "rows bound parallel dispatch only") )] rows: usize, + /// Bytes each task joins one fragmented payload into; zero when every + /// payload is one input slice. + fragmented_payload_bytes: usize, } /// Extra classic workspaces beside the main one: one per additional Rayon @@ -478,36 +482,44 @@ fn classic_task_workspace_count(parallel: bool, classic_tasks: Option, scratch: &J2kDirectCpuScratch, ) -> Result<()> { let target_count = classic_task_workspace_count(scratch.parallel, classic_tasks); - budget.include_capacity::( + budget.include_capacity::( scratch.classic_task_workspaces.capacity().max(target_count), )?; - let target_bytes = classic_tasks.map_or(Ok(0), |shape| { + let target_workspace_bytes = classic_tasks.map_or(Ok(0), |shape| { bitplane::classic_decode_workspace_bytes(shape.dimensions.0, shape.dimensions.1) })?; + let target_payload_bytes = classic_tasks.map_or(0, |shape| shape.fragmented_payload_bytes); for index in 0..scratch.classic_task_workspaces.len().max(target_count) { - let retained = scratch + let (retained_workspace, retained_payload) = scratch .classic_task_workspaces .get(index) - .map_or(Ok(0), crate::J2kCodeBlockDecodeWorkspace::allocated_bytes)?; - let target = if index < target_count { - target_bytes + .map_or(Ok((0, 0)), |task| { + task.decode + .allocated_bytes() + .map(|bytes| (bytes, task.payload.capacity())) + })?; + let (target_workspace, target_payload) = if index < target_count { + (target_workspace_bytes, target_payload_bytes) } else { - 0 + (0, 0) }; - budget.include_bytes(retained.max(target))?; + budget.include_bytes(retained_workspace.max(target_workspace))?; + budget.include_bytes(retained_payload.max(target_payload))?; } Ok(()) } /// Keep exactly the extra classic workspaces parallel referenced classic -/// execution uses, each prepared for the plan's largest code block. +/// execution uses, each prepared for the plan's largest code block and +/// holding a buffer for its largest fragmented payload. fn reserve_classic_task_workspaces( classic_tasks: Option, scratch: &mut J2kDirectCpuScratch, @@ -518,15 +530,23 @@ fn reserve_classic_task_workspaces( while scratch.classic_task_workspaces.len() < count { scratch .classic_task_workspaces - .push(crate::J2kCodeBlockDecodeWorkspace::default()); + .push(super::ClassicTaskWorkspace::default()); } if let Some(ClassicTaskShape { dimensions: (width, height), + fragmented_payload_bytes, .. }) = classic_tasks { - for workspace in &mut scratch.classic_task_workspaces { - workspace.prepare(width, height)?; + for task in &mut scratch.classic_task_workspaces { + task.decode.prepare(width, height)?; + task.payload.clear(); + if task.payload.capacity() < fragmented_payload_bytes { + // Release before growing so the old and new buffers never + // coexist beyond the charged maximum. + task.payload = Vec::new(); + } + try_reserve_decode_elements(&mut task.payload, fragmented_payload_bytes)?; } } Ok(()) diff --git a/crates/j2k-native/src/direct_cpu/allocation/referenced.rs b/crates/j2k-native/src/direct_cpu/allocation/referenced.rs index 491a767c..e40c2f44 100644 --- a/crates/j2k-native/src/direct_cpu/allocation/referenced.rs +++ b/crates/j2k-native/src/direct_cpu/allocation/referenced.rs @@ -15,7 +15,7 @@ use super::{ use budget::validate_referenced_aggregate_plan; use storage::{normalize_referenced_scratch, reserve_referenced_scratch}; use view::{ - classic_plan_has_fragmented_payloads, max_referenced_classic_payload_bytes, + max_referenced_classic_fragmented_payload_bytes, max_referenced_classic_payload_bytes, max_referenced_classic_rows, max_referenced_dimensions, max_referenced_payload_bytes, ReferencedPlanView, }; @@ -82,16 +82,12 @@ pub(in super::super) fn planned_referenced_classic_execution_bytes( .map(DirectWorkspaceBudget::peak_bytes) } -/// Shape of the extra classic workspaces parallel execution of `plan` uses: -/// none for plans whose payloads are split over fragments, which parallel -/// dispatch cannot borrow as one slice. +/// Shape of the extra classic workspaces parallel execution of `plan` uses. fn classic_task_shape(plan: &J2kReferencedClassicPlan) -> Option { - if classic_plan_has_fragmented_payloads(plan) { - return None; - } Some(ClassicTaskShape { dimensions: max_referenced_classic_dimensions(plan)?, rows: max_referenced_classic_rows(plan), + fragmented_payload_bytes: max_referenced_classic_fragmented_payload_bytes(plan), }) } diff --git a/crates/j2k-native/src/direct_cpu/allocation/referenced/view.rs b/crates/j2k-native/src/direct_cpu/allocation/referenced/view.rs index 73e1425a..c37a08ca 100644 --- a/crates/j2k-native/src/direct_cpu/allocation/referenced/view.rs +++ b/crates/j2k-native/src/direct_cpu/allocation/referenced/view.rs @@ -221,10 +221,7 @@ fn referenced_parallel_classic_dispatch_bytes( parallel: bool, ) -> Result { let worker_count = rayon::current_num_threads(); - let ReferencedPlanView::Classic(classic) = plan else { - return Ok(0); - }; - if !parallel || worker_count <= 1 || classic_plan_has_fragmented_payloads(classic) { + if !matches!(plan, ReferencedPlanView::Classic(_)) || !parallel || worker_count <= 1 { return Ok(0); } let mut maximum_jobs = 0usize; @@ -240,10 +237,11 @@ fn referenced_parallel_classic_dispatch_bytes( if maximum_jobs == 0 { return Ok(0); } - super::super::super::referenced::rows::row_dispatch_transient_bytes::< - crate::J2kOwnedCodeBlockBatchJob, - crate::J2kCodeBlockDecodeWorkspace, - >(maximum_jobs, maximum_jobs, worker_count) + super::super::super::referenced_classic::parallel_classic_decode_transient_bytes( + maximum_jobs, + maximum_jobs, + worker_count, + ) } #[cfg(not(feature = "parallel"))] @@ -333,13 +331,18 @@ pub(in super::super::super) fn max_referenced_classic_dimensions( max_referenced_dimensions(ReferencedPlanView::Classic(plan), false) } -/// Whether some classic payload is split over several input fragments (a -/// multi-layer codestream). Parallel classic dispatch borrows each payload -/// as one input slice, so such plans decode their blocks serially. -pub(super) fn classic_plan_has_fragmented_payloads(plan: &J2kReferencedClassicPlan) -> bool { +/// Largest classic payload split over several input fragments (a +/// multi-layer codestream), which a parallel task joins into its own buffer; +/// zero when every payload is one input slice. +pub(super) fn max_referenced_classic_fragmented_payload_bytes( + plan: &J2kReferencedClassicPlan, +) -> usize { plan.payloads() .iter() - .any(|payload| payload.range_count > 1) + .filter(|payload| payload.range_count > 1) + .map(|payload| payload.combined_length) + .max() + .unwrap_or(0) } /// Most runs of code-block rows (consecutive jobs sharing `output_y`) in any diff --git a/crates/j2k-native/src/direct_cpu/referenced/component.rs b/crates/j2k-native/src/direct_cpu/referenced/component.rs index e33293a5..0e3ba295 100644 --- a/crates/j2k-native/src/direct_cpu/referenced/component.rs +++ b/crates/j2k-native/src/direct_cpu/referenced/component.rs @@ -206,7 +206,7 @@ fn execute_ht_sub_band_referenced( } #[cfg(feature = "parallel")] -type RowJob<'p, 'i> = super::rows::RowJob<'p, 'i, crate::HtOwnedCodeBlockBatchJob>; +type RowJob<'p, 'i> = super::rows::RowJob<'p, crate::HtOwnedCodeBlockBatchJob, &'i [u8]>; /// Bytes owned together while a referenced HT sub-band is dispatched in /// parallel. Planning and execution share this helper so the private dispatch @@ -219,7 +219,8 @@ pub(in crate::direct_cpu) fn parallel_ht_decode_transient_bytes( ) -> Result { super::rows::row_dispatch_transient_bytes::< crate::HtOwnedCodeBlockBatchJob, - HtCodeBlockDecodeWorkspace, + &[u8], + &mut HtCodeBlockDecodeWorkspace, >(entry_capacity, row_capacity, workspace_capacity) } @@ -244,7 +245,7 @@ fn decode_ht_sub_band_rows_parallel( }; entries.sort_unstable_by_key(|entry| (entry.job.output_y, entry.job.output_x)); let entry_capacity = entries.capacity(); - let Some(rows) = super::rows::carve_rows::<_, HtCodeBlockDecodeWorkspace>( + let Some(rows) = super::rows::carve_rows::<_, _, &mut HtCodeBlockDecodeWorkspace>( &entries, output, sub_band_width, @@ -260,7 +261,7 @@ fn decode_ht_sub_band_rows_parallel( rows, entry_capacity, &mut *ht.main, - ht.tasks, + ht.tasks.iter_mut(), ht.budget, |entry, output, workspace| { let job = entry.job; diff --git a/crates/j2k-native/src/direct_cpu/referenced/rows.rs b/crates/j2k-native/src/direct_cpu/referenced/rows.rs index 7910af28..7c008730 100644 --- a/crates/j2k-native/src/direct_cpu/referenced/rows.rs +++ b/crates/j2k-native/src/direct_cpu/referenced/rows.rs @@ -2,7 +2,7 @@ //! Row-parallel code-block dispatch shared by referenced HT and classic //! execution: borrowed jobs are grouped into disjoint rows of one sub-band -//! and decoded on the Rayon pool, each row with a pooled workspace. +//! and decoded on the Rayon pool, each row with a pooled task workspace. use crate::error::{DecodingError, Result, ValidationError}; use crate::{HtOwnedCodeBlockBatchJob, J2kOwnedCodeBlockBatchJob}; @@ -47,37 +47,39 @@ impl CodeBlockRowGeometry for J2kOwnedCodeBlockBatchJob { } } -/// One code block of a parallel sub-band decode with its borrowed payload and -/// output range in the sub-band. -pub(in crate::direct_cpu) struct RowJob<'p, 'i, J> { +/// One code block of a parallel sub-band decode with its payload `D` and +/// output range in the sub-band. HT borrows each payload as one input slice; +/// classic carries the payload's input fragments for its task to join. +pub(in crate::direct_cpu) struct RowJob<'p, J, D> { pub(in crate::direct_cpu) job: &'p J, - pub(in crate::direct_cpu) data: &'i [u8], + pub(in crate::direct_cpu) data: D, pub(in crate::direct_cpu) start: usize, pub(in crate::direct_cpu) end: usize, } /// A row of code blocks: its output rows, its jobs, and its first sample. -pub(in crate::direct_cpu) type CodeBlockRow<'o, 'j, 'p, 'i, J> = - (&'o mut [f32], &'j [RowJob<'p, 'i, J>], usize); +pub(in crate::direct_cpu) type CodeBlockRow<'o, 'j, 'p, J, D> = + (&'o mut [f32], &'j [RowJob<'p, J, D>], usize); -/// Bytes owned together while one sub-band is dispatched in parallel. -/// Planning and execution share this helper so the private dispatch types -/// cannot drift from their accounting. -pub(in crate::direct_cpu) fn row_dispatch_transient_bytes( +/// Bytes owned together while one sub-band is dispatched in parallel, with +/// `W` the pooled per-task workspace handle. Planning and execution share +/// this helper so the private dispatch types cannot drift from their +/// accounting. +pub(in crate::direct_cpu) fn row_dispatch_transient_bytes( entry_capacity: usize, row_capacity: usize, workspace_capacity: usize, ) -> Result { entry_capacity - .checked_mul(core::mem::size_of::>()) + .checked_mul(core::mem::size_of::>()) .and_then(|bytes| { row_capacity - .checked_mul(core::mem::size_of::>()) + .checked_mul(core::mem::size_of::>()) .and_then(|rows| bytes.checked_add(rows)) }) .and_then(|bytes| { workspace_capacity - .checked_mul(core::mem::size_of::<&mut W>()) + .checked_mul(core::mem::size_of::()) .and_then(|workspaces| bytes.checked_add(workspaces)) }) .ok_or_else(|| ValidationError::ImageTooLarge.into()) @@ -85,16 +87,16 @@ pub(in crate::direct_cpu) fn row_dispatch_transient_bytes( /// Split `output` into one slice per row of code blocks, or `None` when the /// jobs, sorted by position, do not form disjoint rows of equal-height blocks. -pub(in crate::direct_cpu) fn carve_rows<'o, 'j, 'p, 'i, J: CodeBlockRowGeometry, W>( - entries: &'j [RowJob<'p, 'i, J>], +pub(in crate::direct_cpu) fn carve_rows<'o, 'j, 'p, J: CodeBlockRowGeometry, D, W>( + entries: &'j [RowJob<'p, J, D>], output: &'o mut [f32], sub_band_width: usize, entry_capacity: usize, budget: DirectWorkspaceBudget, -) -> Result>>> { +) -> Result>>> { let mut rows = alloc::vec::Vec::new(); crate::try_reserve_decode_elements(&mut rows, entries.len())?; - budget.validate_workspace(row_dispatch_transient_bytes::( + budget.validate_workspace(row_dispatch_transient_bytes::( entry_capacity, rows.capacity(), 0, @@ -131,20 +133,21 @@ pub(in crate::direct_cpu) fn carve_rows<'o, 'j, 'p, 'i, J: CodeBlockRowGeometry, Ok(Some(rows)) } -/// Decode rows of code blocks on the Rayon pool, each with a pooled +/// Decode rows of code blocks on the Rayon pool, each with a pooled task /// workspace taken from `main` and `tasks`. -pub(in crate::direct_cpu) fn decode_rows( - rows: alloc::vec::Vec>, +pub(in crate::direct_cpu) fn decode_rows( + rows: alloc::vec::Vec>, entry_capacity: usize, - main: &mut W, - tasks: &mut [W], + main: W, + tasks: impl ExactSizeIterator, budget: DirectWorkspaceBudget, decode: F, ) -> Result<()> where J: Sync, + D: Sync, W: Send, - F: Fn(&RowJob<'_, '_, J>, &mut [f32], &mut W) -> Result<()> + Sync, + F: Fn(&RowJob<'_, J, D>, &mut [f32], &mut W) -> Result<()> + Sync, { use rayon::prelude::*; use std::sync::Mutex; @@ -157,8 +160,8 @@ where let mut workspaces = alloc::vec::Vec::new(); crate::try_reserve_decode_elements(&mut workspaces, workspace_count)?; workspaces.push(main); - workspaces.extend(tasks.iter_mut()); - budget.validate_workspace(row_dispatch_transient_bytes::( + workspaces.extend(tasks); + budget.validate_workspace(row_dispatch_transient_bytes::( entry_capacity, row_capacity, workspaces.capacity(), @@ -173,10 +176,10 @@ where .lock() .map_err(|_| DecodingError::CodeBlockDecodeFailure)? .pop(); - let workspace = pooled.ok_or(DecodingError::CodeBlockDecodeFailure)?; + let mut workspace = pooled.ok_or(DecodingError::CodeBlockDecodeFailure)?; let result = jobs.iter().try_for_each(|entry| { let output = &mut row[entry.start - row_start..entry.end - row_start]; - decode(entry, output, workspace) + decode(entry, output, &mut workspace) }); pool.lock() .map_err(|_| DecodingError::CodeBlockDecodeFailure)? diff --git a/crates/j2k-native/src/direct_cpu/referenced_classic.rs b/crates/j2k-native/src/direct_cpu/referenced_classic.rs index d0d2928d..8ad63306 100644 --- a/crates/j2k-native/src/direct_cpu/referenced_classic.rs +++ b/crates/j2k-native/src/direct_cpu/referenced_classic.rs @@ -14,9 +14,9 @@ use super::allocation::prepare_referenced_classic_scratch; use super::referenced::{decoded_color_components, decoded_plane, payload_slice}; use super::{ apply_inverse_mct_region, checked_sub_band_job_output_range, execute_idwt_step, - prepare_sub_band_output, rounds_at_store, store_component, DirectComponentBandScratch, - DirectComponentPlane, DirectWorkspaceBudget, J2kDirectCpuScratch, J2kDirectDecodedComponents, - SubBandJobOutputRange, + prepare_sub_band_output, rounds_at_store, store_component, ClassicTaskWorkspace, + DirectComponentBandScratch, DirectComponentPlane, DirectWorkspaceBudget, J2kDirectCpuScratch, + J2kDirectDecodedComponents, SubBandJobOutputRange, }; /// Execute retained per-tile Gray/RGB/RGBA classic JPEG 2000 geometry @@ -281,7 +281,7 @@ struct ClassicWorkspaces<'a> { not(feature = "parallel"), expect(dead_code, reason = "extra workspaces serve parallel execution only") )] - tasks: &'a mut [J2kCodeBlockDecodeWorkspace], + tasks: &'a mut [ClassicTaskWorkspace], parallel: bool, #[cfg_attr( not(feature = "parallel"), @@ -310,6 +310,7 @@ fn execute_classic_sub_band_with_workspaces( output, sub_band_width, payloads, + compressed_payload, workspaces, )? { @@ -395,23 +396,51 @@ fn decode_classic_sub_band_jobs( #[cfg(feature = "parallel")] type ClassicRowJob<'p, 'i> = - super::referenced::rows::RowJob<'p, 'i, crate::J2kOwnedCodeBlockBatchJob>; + super::referenced::rows::RowJob<'p, crate::J2kOwnedCodeBlockBatchJob, ClassicFragments<'i>>; + +/// One row task's classic state: its code-block workspace and the buffer it +/// joins fragmented payloads into. +#[cfg(feature = "parallel")] +struct ClassicRowTask<'a> { + workspace: &'a mut J2kCodeBlockDecodeWorkspace, + payload: &'a mut alloc::vec::Vec, +} + +/// Bytes owned together while a referenced classic sub-band is dispatched in +/// parallel. Planning and execution share this helper so the private +/// dispatch types cannot drift from their accounting. +#[cfg(feature = "parallel")] +pub(in crate::direct_cpu) fn parallel_classic_decode_transient_bytes( + entry_capacity: usize, + row_capacity: usize, + workspace_capacity: usize, +) -> Result { + super::referenced::rows::row_dispatch_transient_bytes::< + crate::J2kOwnedCodeBlockBatchJob, + ClassicFragments<'_>, + ClassicRowTask<'_>, + >(entry_capacity, row_capacity, workspace_capacity) +} /// Decode a classic sub-band's rows of code blocks in parallel, each writing -/// its own rows of `output`. Returns `false` without consuming payloads when a -/// block's bytes are not one contiguous input slice or the blocks do not tile -/// the sub-band in rows; the serial path then decodes the sub-band. Every -/// block decodes exactly as on the serial path, so the samples are identical. +/// its own rows of `output`. Each row task joins fragmented (multi-layer) +/// payloads in its own buffer: the main task in `compressed_payload`, every +/// extra one in its task workspace. Returns `false` without consuming +/// payloads when the blocks do not tile the sub-band in rows; the serial path +/// then decodes the sub-band. Every block decodes exactly as on the serial +/// path, so the samples are identical. #[cfg(feature = "parallel")] fn decode_classic_sub_band_rows_parallel( plan: &J2kOwnedSubBandPlan, output: &mut [f32], sub_band_width: usize, payloads: &mut ClassicPayloadCursor<'_>, + compressed_payload: &mut alloc::vec::Vec, workspaces: &mut ClassicWorkspaces<'_>, ) -> Result { use super::referenced::rows; + let encoded_input = payloads.encoded_input; let checkpoint = payloads.position(); let Some(mut entries) = collect_classic_row_jobs( plan, @@ -426,7 +455,7 @@ fn decode_classic_sub_band_rows_parallel( }; entries.sort_unstable_by_key(|entry| (entry.job.output_y, entry.job.output_x)); let entry_capacity = entries.capacity(); - let Some(rows) = rows::carve_rows::<_, J2kCodeBlockDecodeWorkspace>( + let Some(rows) = rows::carve_rows::<_, _, ClassicRowTask<'_>>( &entries, output, sub_band_width, @@ -438,16 +467,25 @@ fn decode_classic_sub_band_rows_parallel( return Ok(false); }; let irreversible_midpoint = plan.irreversible_midpoint; + let main = ClassicRowTask { + workspace: &mut *workspaces.main, + payload: compressed_payload, + }; + let tasks = workspaces.tasks.iter_mut().map(|task| ClassicRowTask { + workspace: &mut task.decode, + payload: &mut task.payload, + }); rows::decode_rows( rows, entry_capacity, - &mut *workspaces.main, - workspaces.tasks, + main, + tasks, workspaces.budget, - |entry, output, workspace| { + |entry, output, task| { + let ClassicRowTask { workspace, payload } = task; let job = entry.job; let code_block = J2kCodeBlockDecodeJob { - data: entry.data, + data: entry.data.join(encoded_input, payload)?, segments: &job.segments, width: job.width, height: job.height, @@ -471,8 +509,8 @@ fn decode_classic_sub_band_rows_parallel( Ok(true) } -/// Validate every job's output range and borrow its payload, or `None` when -/// a payload is split over several input fragments. +/// Validate every job's output range and borrow its payload fragments, or +/// `None` when a job's output stride is not the sub-band width. #[cfg(feature = "parallel")] fn collect_classic_row_jobs<'p, 'i>( plan: &'p J2kOwnedSubBandPlan, @@ -483,10 +521,11 @@ fn collect_classic_row_jobs<'p, 'i>( ) -> Result>>> { let mut entries = alloc::vec::Vec::new(); try_reserve_decode_elements(&mut entries, plan.jobs.len())?; - budget.validate_workspace(super::referenced::rows::row_dispatch_transient_bytes::< - crate::J2kOwnedCodeBlockBatchJob, - J2kCodeBlockDecodeWorkspace, - >(entries.capacity(), 0, 0)?)?; + budget.validate_workspace(parallel_classic_decode_transient_bytes( + entries.capacity(), + 0, + 0, + )?)?; for job in &plan.jobs { if !job.data.is_empty() { bail!(DecodingError::CodeBlockDecodeFailure); @@ -505,12 +544,9 @@ fn collect_classic_row_jobs<'p, 'i>( if job.output_stride != sub_band_width { return Ok(None); } - let Some(data) = payloads.next_contiguous()? else { - return Ok(None); - }; entries.push(ClassicRowJob { job, - data, + data: payloads.next_fragments()?, start: range.start, end: range.end, }); @@ -552,6 +588,45 @@ pub(super) fn validate_payload_ranges( Ok(()) } +/// One code block's compressed payload as ordered input fragments; a +/// multi-layer codestream splits it over one fragment per contributing +/// layer. +#[derive(Clone, Copy)] +struct ClassicFragments<'plan> { + ranges: &'plan [crate::J2kCodestreamRange], + combined_length: usize, +} + +impl<'plan> ClassicFragments<'plan> { + /// The payload bytes: the input slice itself for one fragment, else the + /// fragments joined in `combined`. + fn join<'scratch>( + self, + encoded_input: &'plan [u8], + combined: &'scratch mut alloc::vec::Vec, + ) -> Result<&'scratch [u8]> + where + 'plan: 'scratch, + { + if let [range] = self.ranges { + if range.length != self.combined_length { + bail!(DecodingError::CodeBlockDecodeFailure); + } + return payload_slice(encoded_input, *range); + } + + combined.clear(); + try_reserve_decode_elements(combined, self.combined_length)?; + for range in self.ranges { + combined.extend_from_slice(payload_slice(encoded_input, *range)?); + } + if combined.len() != self.combined_length { + bail!(DecodingError::CodeBlockDecodeFailure); + } + Ok(combined) + } +} + pub(super) struct ClassicPayloadCursor<'plan> { encoded_input: &'plan [u8], payloads: &'plan [crate::J2kClassicCodeBlockPayload], @@ -574,9 +649,17 @@ impl<'plan> ClassicPayloadCursor<'plan> { } fn next_data<'scratch>( - &'scratch mut self, + &mut self, combined: &'scratch mut alloc::vec::Vec, - ) -> Result<&'scratch [u8]> { + ) -> Result<&'scratch [u8]> + where + 'plan: 'scratch, + { + self.next_fragments()?.join(self.encoded_input, combined) + } + + /// The next payload's input fragments. + fn next_fragments(&mut self) -> Result> { let payload = self .payloads .get(self.next) @@ -588,27 +671,14 @@ impl<'plan> ClassicPayloadCursor<'plan> { let end_range = payload .end_range() .ok_or(DecodingError::CodeBlockDecodeFailure)?; - let fragments = self + let ranges = self .ranges .get(payload.first_range..end_range) .ok_or(DecodingError::CodeBlockDecodeFailure)?; - - if let [range] = fragments { - if range.length != payload.combined_length { - bail!(DecodingError::CodeBlockDecodeFailure); - } - return payload_slice(self.encoded_input, *range); - } - - combined.clear(); - try_reserve_decode_elements(combined, payload.combined_length)?; - for range in fragments { - combined.extend_from_slice(payload_slice(self.encoded_input, *range)?); - } - if combined.len() != payload.combined_length { - bail!(DecodingError::CodeBlockDecodeFailure); - } - Ok(combined) + Ok(ClassicFragments { + ranges, + combined_length: payload.combined_length, + }) } /// Position for [`Self::rewind`] when a parallel attempt falls back. @@ -622,36 +692,6 @@ impl<'plan> ClassicPayloadCursor<'plan> { self.next = position; } - /// The next payload borrowed from the input when it is one contiguous - /// fragment (or empty), else `None` without consuming it. - #[cfg(feature = "parallel")] - fn next_contiguous(&mut self) -> Result> { - let payload = self - .payloads - .get(self.next) - .ok_or(DecodingError::CodeBlockDecodeFailure)?; - let end_range = payload - .end_range() - .ok_or(DecodingError::CodeBlockDecodeFailure)?; - let fragments = self - .ranges - .get(payload.first_range..end_range) - .ok_or(DecodingError::CodeBlockDecodeFailure)?; - let data = match fragments { - [] if payload.combined_length == 0 => &[][..], - [range] if range.length == payload.combined_length => { - payload_slice(self.encoded_input, *range)? - } - [_] => bail!(DecodingError::CodeBlockDecodeFailure), - _ => return Ok(None), - }; - self.next = self - .next - .checked_add(1) - .ok_or(DecodingError::CodeBlockDecodeFailure)?; - Ok(Some(data)) - } - pub(super) fn ensure_exhausted(&self) -> Result<()> { if self.next == self.payloads.len() { Ok(()) diff --git a/crates/j2k-native/src/direct_cpu/referenced_classic/tests.rs b/crates/j2k-native/src/direct_cpu/referenced_classic/tests.rs index 823825f7..e015c0cd 100644 --- a/crates/j2k-native/src/direct_cpu/referenced_classic/tests.rs +++ b/crates/j2k-native/src/direct_cpu/referenced_classic/tests.rs @@ -7,6 +7,7 @@ use crate::{ J2kDirectCpuScratch, }; +#[cfg(feature = "parallel")] fn rgb_fixture(width: u32, height: u32, reversible: bool) -> Vec { rgb_fixture_with_layers(width, height, reversible, 1) } @@ -29,14 +30,41 @@ fn rgb_fixture_with_layers(width: u32, height: u32, reversible: bool, num_layers encode(&samples, width, height, 3, 8, false, &options).expect("encode classic RGB fixture") } -/// Every reconstructed plane as raw bits, plus the retained classic -/// workspace bytes after the execution. -fn execute_whole_plan(bytes: &[u8], parallel: bool) -> (Vec>, usize) { +/// Reconstructed planes as raw bits, the retained classic workspace bytes, +/// and each extra task's payload buffer (length, capacity) after one +/// execution. +struct Execution { + planes: Vec>, + workspace_bytes: usize, + task_payloads: Vec<(usize, usize)>, +} + +type Region = (u32, u32, u32, u32); + +fn whole_image(bytes: &[u8]) -> Region { + let image = Image::new(bytes, &DecodeSettings::default()).expect("classic image"); + (0, 0, image.width(), image.height()) +} + +/// The largest payload split over several input fragments, if any. +#[cfg(feature = "parallel")] +fn largest_fragmented_payload(bytes: &[u8], region: Region) -> Option { + let image = Image::new(bytes, &DecodeSettings::default()).expect("classic image"); + let plan = image + .build_referenced_classic_plan_region_with_context(&mut DecoderContext::default(), region) + .expect("classic region plan"); + plan.payloads() + .iter() + .filter(|payload| payload.range_count > 1) + .map(|payload| payload.combined_length) + .max() +} + +fn execute_region(bytes: &[u8], region: Region, parallel: bool) -> Execution { let image = Image::new(bytes, &DecodeSettings::default()).expect("classic image"); - let region = (0, 0, image.width(), image.height()); let plan = image .build_referenced_classic_plan_region_with_context(&mut DecoderContext::default(), region) - .expect("classic whole-image plan"); + .expect("classic region plan"); let mut scratch = J2kDirectCpuScratch::new(); scratch.set_parallel_decode(parallel); let decoded = execute_referenced_classic_plan(&plan, bytes, false, &mut scratch) @@ -52,7 +80,11 @@ fn execute_whole_plan(bytes: &[u8], parallel: bool) -> (Vec>, usize) { .collect() }) .collect(); - (planes, scratch.retained_classic_workspace_bytes()) + Execution { + planes, + workspace_bytes: scratch.retained_classic_workspace_bytes(), + task_payloads: scratch.classic_task_payloads(), + } } #[cfg(feature = "parallel")] @@ -65,16 +97,24 @@ fn parallel_referenced_classic_execution_matches_serial_bits() { for reversible in [true, false] { // Several rows of 64 x 64 code blocks per high-resolution sub-band. let bytes = rgb_fixture(300, 260, reversible); - let (serial, serial_workspace_bytes) = execute_whole_plan(&bytes, false); - let (parallel, parallel_workspace_bytes) = - pool.install(|| execute_whole_plan(&bytes, true)); - assert_eq!(serial, parallel, "reversible={reversible}"); + let region = whole_image(&bytes); + assert_eq!(largest_fragmented_payload(&bytes, region), None); + let serial = execute_region(&bytes, region, false); + let parallel = pool.install(|| execute_region(&bytes, region, true)); + assert_eq!(serial.planes, parallel.planes, "reversible={reversible}"); // The tallest sub-band (130 rows) holds three rows of 64-high // blocks, so of the four workers only two extra ones can take a row // and keep their own prepared workspace. assert_eq!( - parallel_workspace_bytes, - serial_workspace_bytes * 3, + parallel.workspace_bytes, + serial.workspace_bytes * 3, + "reversible={reversible}" + ); + // Single-fragment payloads are borrowed from the input, so the extra + // tasks keep no payload buffer. + assert_eq!( + parallel.task_payloads, + [(0, 0); 2], "reversible={reversible}" ); } @@ -82,35 +122,64 @@ fn parallel_referenced_classic_execution_matches_serial_bits() { #[cfg(feature = "parallel")] #[test] -fn fragmented_payloads_decode_serially_without_extra_workspaces() { - let bytes = rgb_fixture_with_layers(300, 260, false, 4); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("classic image"); - let plan = image - .build_referenced_classic_plan_region_with_context( - &mut DecoderContext::default(), - (0, 0, image.width(), image.height()), - ) - .expect("classic whole-image plan"); - assert!( - plan.payloads() - .iter() - .any(|payload| payload.range_count > 1), - "four quality layers split code-block payloads over fragments" - ); +fn fragmented_payloads_decode_in_parallel_with_per_task_buffers() { let pool = rayon::ThreadPoolBuilder::new() .num_threads(4) .build() .expect("four-thread pool"); - let (serial, serial_workspace_bytes) = execute_whole_plan(&bytes, false); - let (parallel, parallel_workspace_bytes) = pool.install(|| execute_whole_plan(&bytes, true)); - assert_eq!(serial, parallel); - assert_eq!(parallel_workspace_bytes, serial_workspace_bytes); + for (reversible, layers) in [(false, 4), (true, 8), (false, 8)] { + let bytes = rgb_fixture_with_layers(300, 260, reversible, layers); + let crop = (37, 21, 230, 211); + for region in [whole_image(&bytes), crop] { + let case = format!("reversible={reversible} layers={layers} region={region:?}"); + let largest = largest_fragmented_payload(&bytes, region) + .unwrap_or_else(|| panic!("{case}: layers split payloads over fragments")); + let serial = execute_region(&bytes, region, false); + let parallel = pool.install(|| execute_region(&bytes, region, true)); + assert_eq!(serial.planes, parallel.planes, "{case}"); + // One prepared workspace per extra task, each beside its buffer. + let tasks = parallel.task_payloads.len(); + assert!(tasks >= 1, "{case}: no extra task"); + assert_eq!( + parallel.workspace_bytes, + serial.workspace_bytes * (tasks + 1), + "{case}" + ); + for &(_, capacity) in ¶llel.task_payloads { + assert!( + capacity >= largest, + "{case}: task buffer {capacity} < {largest}" + ); + } + // The pool hands out the extra tasks first, so the parallel rows + // joined fragments in their own buffers. + assert!( + parallel.task_payloads.iter().any(|&(len, _)| len > 0), + "{case}: no extra task joined a fragmented payload" + ); + } + } } #[test] fn parallel_request_on_the_global_pool_matches_serial_bits() { - let bytes = rgb_fixture(200, 136, false); - let (serial, _) = execute_whole_plan(&bytes, false); - let (parallel, _) = execute_whole_plan(&bytes, true); - assert_eq!(serial, parallel); + for layers in [1, 6] { + let bytes = rgb_fixture_with_layers(200, 136, false, layers); + let region = whole_image(&bytes); + let serial = execute_region(&bytes, region, false); + let parallel = execute_region(&bytes, region, true); + assert_eq!(serial.planes, parallel.planes, "layers={layers}"); + assert_eq!( + serial.task_payloads, + Vec::<(usize, usize)>::new(), + "layers={layers}" + ); + // However many workers the global pool has, each extra task keeps + // one prepared workspace. + assert_eq!( + parallel.workspace_bytes, + serial.workspace_bytes * (parallel.task_payloads.len() + 1), + "layers={layers}" + ); + } } From 85c10ac595d82c5c675d80aa7a7319b0b4ed8d5a Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 05:14:13 -0400 Subject: [PATCH 13/94] fix(j2k): stream large Gray8/RGB8 batch items instead of holding whole planes Every Gray8/RGB8 image on the CPU batch decoder's offset-plan route held whole-image f32 planes and coefficients, about 14 times its 8-bit output, until its decode ended: eight 3584^2 RGB8 images in flight peaked at 4.3 GB for 294 MB of output, with one worker as with the pool (the staged route keeps every window image's scratch). Preparation now marks a Gray8/RGB8 image whose planned execution exceeds 4 MiB for the streamed region rows route that over-budget images already use, which holds a band of rows instead of planes; its output is bit-identical. Groups with such an image skip the staged route, so a batch's peak is the streamed working sets of the images in flight plus the outputs. Images below the threshold, and outputs without a streamed route (16-bit, RGBA), keep their parse-free plans. The one exception is a lone Gray8 image (the only image of its group, hence the whole in-flight set) whose plan fits the per-decode bound: it keeps decoding its plan, which spreads one component's code-block rows over every core; streamed, such images decoded 26-66% slower. RGB8 images stream faster than their plans decoded (whole image, reduced scales, regions, tiles). An explicit single worker over several 8-bit images loses the staged route's cross-image windows: about 30% slower there for a tenth of the memory. --- crates/j2k/src/owned_batch/cpu_execute.rs | 19 ++- crates/j2k/src/owned_batch/cpu_materialize.rs | 61 ++++++-- .../prepare/cpu_preflight_tests.rs | 148 +++++++++++++++++- crates/j2k/src/owned_batch/prepare/image.rs | 43 ++++- crates/j2k/src/owned_batch/prepared.rs | 22 ++- 5 files changed, 257 insertions(+), 36 deletions(-) diff --git a/crates/j2k/src/owned_batch/cpu_execute.rs b/crates/j2k/src/owned_batch/cpu_execute.rs index bd24cbfa..9cb45bb8 100644 --- a/crates/j2k/src/owned_batch/cpu_execute.rs +++ b/crates/j2k/src/owned_batch/cpu_execute.rs @@ -4,15 +4,15 @@ use super::cpu_materialize::{ convert_u16, convert_u8, decode_image_i16, decode_image_u16, decode_image_u8, - ensure_batch_output_within_cap, round_signed, try_zeroed_vec, + decode_lone_gray_u8, ensure_batch_output_within_cap, round_signed, try_zeroed_vec, }; use super::cpu_staged_execute::run_staged_typed_group; use super::{ - decode_warnings_for_recovery, retained_inner_parallelism, run_retained_chunks, Arc, + decode_warnings_for_recovery, retained_inner_parallelism, run_retained_chunks, Arc, BatchColor, BatchDecodeOptions, BatchErrorStage, BatchGroupInfo, BatchInfrastructureError, BatchItemError, - BatchWorker, CpuBatchGroup, CpuBatchSamples, CpuDecodeParallelism, CpuExecution, - CpuGroupFastWorkspace, CpuStagedWorkspace, IndexedBatchError, J2kError, NativeSampleType, - NonZeroUsize, PreparedBatchGroup, PreparedImage, TileBatchOptions, Vec, + BatchWorker, CpuBatchGroup, CpuBatchSamples, CpuDecodeParallelism, CpuGroupFastWorkspace, + CpuStagedWorkspace, IndexedBatchError, J2kError, NativeSampleType, NonZeroUsize, + PreparedBatchGroup, PreparedImage, TileBatchOptions, Vec, }; use crate::batch::admission::{hold_at_least, BatchAllocationBudget}; use crate::batch::allocation::try_vec_with_capacity; @@ -42,7 +42,7 @@ pub(super) fn decode_cpu_group( let streams_rows = group .images .iter() - .any(|image| image.cpu_execution() == CpuExecution::StreamedRows); + .any(|image| image.cpu_execution().streams()); let has_flattened_payloads = !streams_rows && fast_workspace.prepare_group(group)? && retained_inner_parallelism(group.images.len(), tile_options)? @@ -78,7 +78,12 @@ pub(super) fn decode_cpu_group( has_flattened_payloads.then_some(&mut *fast_workspace), staged_workspace, admission, - decode_image_u8, + // The only image of a Gray8 group is the whole in-flight set. + if group.images.len() == 1 && group.info.color == BatchColor::Gray { + decode_lone_gray_u8 + } else { + decode_image_u8 + }, convert_u8, errors, )?; diff --git a/crates/j2k/src/owned_batch/cpu_materialize.rs b/crates/j2k/src/owned_batch/cpu_materialize.rs index 6b5340f3..2b64f1e8 100644 --- a/crates/j2k/src/owned_batch/cpu_materialize.rs +++ b/crates/j2k/src/owned_batch/cpu_materialize.rs @@ -92,20 +92,47 @@ pub(super) fn decode_image_u8<'image>( out: &mut [u8], ) -> Result<(), J2kError> { match image.cpu_execution() { - CpuExecution::ExceedsDecodeBound => return Err(CpuExecution::decode_bound_error()), - CpuExecution::StreamedRows => { - return decode_streamed_samples( - image, - options, - info, - parallelism, - context, - out, - convert_u8, - ); + CpuExecution::ExceedsDecodeBound => Err(CpuExecution::decode_bound_error()), + CpuExecution::StreamedRows | CpuExecution::StreamedRowsOrPlan { .. } => { + decode_streamed_samples(image, options, info, parallelism, context, out, convert_u8) + } + CpuExecution::OffsetPlan { .. } | CpuExecution::ComponentPlanes => { + decode_planned_u8(image, options, info, parallelism, context, worker, out) } - CpuExecution::OffsetPlan { .. } | CpuExecution::ComponentPlanes => {} } +} + +/// [`decode_image_u8`] for the only image of a Gray8 group: an image that may +/// stream or decode its plan decodes its plan, which spreads one component's +/// code-block rows over every core while nothing else is in flight. +pub(super) fn decode_lone_gray_u8<'image>( + image: &'image PreparedImage, + options: BatchDecodeOptions, + info: &BatchGroupInfo, + parallelism: CpuDecodeParallelism, + context: &mut j2k_native::DecoderContext<'image>, + worker: &mut BatchWorker, + out: &mut [u8], +) -> Result<(), J2kError> { + if matches!( + image.cpu_execution(), + CpuExecution::StreamedRowsOrPlan { .. } + ) { + decode_planned_u8(image, options, info, parallelism, context, worker, out) + } else { + decode_image_u8(image, options, info, parallelism, context, worker, out) + } +} + +fn decode_planned_u8<'image>( + image: &'image PreparedImage, + options: BatchDecodeOptions, + info: &BatchGroupInfo, + parallelism: CpuDecodeParallelism, + context: &mut j2k_native::DecoderContext<'image>, + worker: &mut BatchWorker, + out: &mut [u8], +) -> Result<(), J2kError> { if decode_prepared_plan_samples(image, info, worker, out, |sample, precision| { u8::try_from(round_unsigned(sample, precision)).unwrap_or(u8::MAX) })? { @@ -245,23 +272,23 @@ impl<'out, 'info, T: Copy> SampleRowSink<'out, 'info, T> { fn write_rectangle(&mut self, x: u32, y: u32, planes: &[j2k_native::ComponentPlane<'_>]) { let channels = self.info.color.channels(); let width = self.info.dimensions.0 as usize; - let Some((rect_width, rect_height)) = planes.first().map(|plane| plane.dimensions()) else { + let Some(rect_dimensions) = planes.first().map(j2k_native::ComponentPlane::dimensions) + else { self.malformed = true; return; }; let (x, y, rect_width, rect_height) = ( x as usize, y as usize, - rect_width as usize, - rect_height as usize, + rect_dimensions.0 as usize, + rect_dimensions.1 as usize, ); let rect_pixels = rect_width * rect_height; if planes.len() != channels || x + rect_width > width || y + rect_height > self.info.dimensions.1 as usize || planes.iter().any(|plane| { - plane.dimensions() != (rect_width as u32, rect_height as u32) - || plane.samples().len() < rect_pixels + plane.dimensions() != rect_dimensions || plane.samples().len() < rect_pixels }) { self.malformed = true; diff --git a/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs b/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs index 26677765..7bd84e44 100644 --- a/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs +++ b/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs @@ -8,7 +8,7 @@ use super::super::{ CpuBatchDecoder, CpuBatchSamples, CpuExecution, DecodeRequest, EncodedImage, IndexedBatchError, PreparedBatch, }; -use super::image::PLANNED_DECODE_BOUND; +use super::image::{PLANNED_DECODE_BOUND, STREAM_ABOVE_OVERRIDE, STREAM_ABOVE_PLANNED_BYTES}; use j2k_core::Downscale; fn classic_fixture(width: u32, height: u32, components: u16, precision: u8) -> Arc<[u8]> { @@ -67,6 +67,21 @@ impl Drop for PlannedBound { } } +struct StreamAbove; + +impl StreamAbove { + fn hold(bytes: usize) -> Self { + STREAM_ABOVE_OVERRIDE.with(|threshold| threshold.set(Some(bytes))); + Self + } +} + +impl Drop for StreamAbove { + fn drop(&mut self) { + STREAM_ABOVE_OVERRIDE.with(|threshold| threshold.set(None)); + } +} + fn planned_bytes(prepared: &PreparedBatch) -> Vec<(usize, CpuExecution)> { let mut planned = prepared .groups() @@ -218,3 +233,134 @@ fn cpu_prepare_rejects_plans_over_the_bound_without_a_streamed_route() { BatchErrorStage::Decode )); } + +#[test] +fn cpu_batch_streams_large_rgb8_and_gray8_plans_below_the_decode_bound() { + let small = classic_rgb8(16, 16); + let large = classic_rgb8(96, 64); + let gray = classic_fixture(80, 72, 1, 8); + let gray12 = classic_fixture(96, 64, 1, 12); + let inputs = || { + vec![ + EncodedImage::full(Arc::clone(&small)), + EncodedImage::full(Arc::clone(&large)), + EncodedImage::full(Arc::clone(&gray)), + EncodedImage::new( + Arc::clone(&large), + DecodeRequest::Region { + roi: j2k_core::Rect { + x: 13, + y: 7, + w: 61, + h: 50, + }, + }, + ), + // Two Gray8 images in flight stream; a lone one decodes its plan. + EncodedImage::full(Arc::clone(&gray)), + EncodedImage::full(Arc::clone(&gray12)), + ] + }; + for layout in [BatchLayout::Nchw, BatchLayout::Nhwc] { + let mut decoder = CpuBatchDecoder::new(single_worker_options_with(layout)); + let planned_route = decoder.prepare(inputs()).expect("prepare small plans"); + let planned = planned_bytes(&planned_route); + assert!( + planned + .iter() + .all(|(_, execution)| matches!(execution, CpuExecution::OffsetPlan { .. })), + "fixtures this small keep their plans: {planned:?}" + ); + let CpuExecution::OffsetPlan { bytes: small_bytes } = planned[0].1 else { + unreachable!("checked above"); + }; + let reference = decoder + .decode_prepared(&planned_route) + .expect("decode the plans"); + + let _threshold = StreamAbove::hold(small_bytes); + let streamed_route = decoder + .prepare(inputs()) + .expect("prepare under the threshold"); + assert_eq!(streamed_route.errors(), &[] as &[IndexedBatchError]); + let streamed = planned_bytes(&streamed_route); + assert!(matches!(streamed[0].1, CpuExecution::OffsetPlan { .. })); + for (index, execution) in &streamed[1..5] { + assert!( + matches!(execution, CpuExecution::StreamedRowsOrPlan { .. }), + "input {index}: {execution:?}" + ); + } + // 12-bit output has no streamed route and stays within the bound. + assert_eq!(streamed[5], planned[5], "{layout:?}"); + let output = decoder + .decode_prepared(&streamed_route) + .expect("decode streamed items"); + assert_eq!(output.errors(), &[] as &[IndexedBatchError]); + assert_eq!(output.groups(), reference.groups(), "{layout:?}"); + } +} + +#[test] +fn cpu_batch_streams_rgb8_images_whose_plans_pass_the_default_threshold() { + // About 14 bytes of plan per 8-bit sample: 1.4 MiB of RGB8 output + // plans several MiB, over the default threshold. + let rgb = classic_rgb8(704, 704); + let rgb12 = classic_fixture(704, 704, 3, 12); + let inputs = || { + vec![ + EncodedImage::full(Arc::clone(&rgb)), + EncodedImage::full(Arc::clone(&rgb12)), + ] + }; + let mut decoder = CpuBatchDecoder::new(single_worker_options()); + let prepared = decoder.prepare(inputs()).expect("prepare"); + let planned = planned_bytes(&prepared); + assert!( + matches!(planned[0].1, CpuExecution::StreamedRowsOrPlan { .. }), + "{planned:?}" + ); + let CpuExecution::OffsetPlan { bytes } = planned[1].1 else { + panic!("12-bit output keeps its plan: {planned:?}"); + }; + assert!(bytes > STREAM_ABOVE_PLANNED_BYTES, "{bytes}"); + let output = decoder.decode_prepared(&prepared).expect("decode"); + assert_eq!(output.errors(), &[] as &[IndexedBatchError]); + + // The streamed samples equal the plan route's. + let _threshold = StreamAbove::hold(usize::MAX); + let planned_route = decoder.prepare(inputs()).expect("prepare plans"); + assert!(matches!( + planned_bytes(&planned_route)[0].1, + CpuExecution::OffsetPlan { .. } + )); + let reference = decoder + .decode_prepared(&planned_route) + .expect("decode plans"); + assert_eq!(output.groups(), reference.groups()); +} + +#[test] +fn a_lone_gray8_image_decodes_its_plan_and_gray8_images_in_flight_stream() { + let gray = classic_fixture(80, 72, 1, 8); + let rgb = classic_rgb8(80, 72); + let decode = |copies: usize, image: &Arc<[u8]>| { + let inputs = vec![EncodedImage::full(Arc::clone(image)); copies]; + let mut decoder = CpuBatchDecoder::new(single_worker_options()); + let reference = decoder + .decode_prepared(&decoder.prepare(inputs.clone()).expect("prepare plans")) + .expect("decode plans"); + let plan_calls = decoder.workspace_stats().prepared_plan_decode_calls(); + let _threshold = StreamAbove::hold(0); + let prepared = decoder.prepare(inputs).expect("prepare"); + assert!(planned_bytes(&prepared) + .iter() + .all(|(_, execution)| matches!(execution, CpuExecution::StreamedRowsOrPlan { .. }))); + let output = decoder.decode_prepared(&prepared).expect("decode"); + assert_eq!(output.groups(), reference.groups(), "{copies} copies"); + decoder.workspace_stats().prepared_plan_decode_calls() - plan_calls + }; + assert_eq!(decode(1, &gray), 1, "a lone Gray8 image decodes its plan"); + assert_eq!(decode(2, &gray), 0, "Gray8 images in flight stream"); + assert_eq!(decode(1, &rgb), 0, "a lone RGB8 image streams"); +} diff --git a/crates/j2k/src/owned_batch/prepare/image.rs b/crates/j2k/src/owned_batch/prepare/image.rs index d92360e8..aa932a6a 100644 --- a/crates/j2k/src/owned_batch/prepare/image.rs +++ b/crates/j2k/src/owned_batch/prepare/image.rs @@ -76,10 +76,17 @@ pub(super) fn prepare_image( /// Choose the CPU execution from the planned working memory, before any /// decode work, with the same accounting the offset plan's execution -/// enforces. Only an image whose serial execution already exceeds the -/// per-decode bound leaves its offset plan, so every image the plan can -/// execute keeps it; such an image streams its rows when its output has a -/// streamed route and is rejected otherwise. The claim is the parallel peak. +/// enforces. +/// +/// A Gray8 or RGB8 image whose planned execution exceeds +/// [`STREAM_ABOVE_PLANNED_BYTES`] streams its rows: the offset plan holds +/// whole-image f32 planes and coefficients (about 14 times the 8-bit +/// output), which every image in flight kept until its decode ended, while +/// the streamed route holds a band of rows. While its plan fits the +/// per-decode bound, the image keeps it for decoding alone in a Gray8 group +/// ([`CpuExecution::StreamedRowsOrPlan`]). Other outputs keep their offset +/// plan unless its serial execution exceeds the per-decode bound, which +/// rejects them. The claim of an offset plan is its parallel peak. pub(super) fn choose_cpu_execution( codec_plan: &PreparedCodecPlan, info: &BatchGroupInfo, @@ -107,9 +114,14 @@ pub(super) fn choose_cpu_execution( _, ) => exceeds_bound, (Ok(serial), _) if serial > planned_decode_bound() => exceeds_bound, - (Ok(_), Ok(bytes)) => CpuExecution::OffsetPlan { bytes }, - // A plan that fits only serially, or a malformed plan, keeps the - // worst-case claim and reports any error from execution, as before. + (Ok(_), Ok(bytes)) if !streams_rows(info) || bytes <= stream_above_planned_bytes() => { + CpuExecution::OffsetPlan { bytes } + } + (Ok(_), Ok(bytes)) => CpuExecution::StreamedRowsOrPlan { bytes }, + // An 8-bit plan that fits only serially streams. + (Ok(_), _) if streams_rows(info) => CpuExecution::StreamedRows, + // Any other plan that fits only serially, or a malformed plan, keeps + // the worst-case claim and reports any error from execution. _ => CpuExecution::OffsetPlan { bytes: crate::batch::allocation::GENERIC_WORKER_CLAIM_BYTES, }, @@ -123,11 +135,28 @@ fn streams_rows(info: &BatchGroupInfo) -> bool { && matches!(info.color, BatchColor::Gray | BatchColor::Rgb) } +/// Planned offset-plan execution above which Gray8/RGB8 images stream +/// their rows (see [`choose_cpu_execution`]). Smaller images keep their +/// parse-free plan, whose working memory is negligible at that size. +pub(super) const STREAM_ABOVE_PLANNED_BYTES: usize = 4 << 20; + #[cfg(test)] std::thread_local! { /// Test-only lower bound for [`choose_cpu_execution`] on this thread. pub(super) static PLANNED_DECODE_BOUND: core::cell::Cell> = const { core::cell::Cell::new(None) }; + /// Test-only streaming threshold for [`choose_cpu_execution`] on this + /// thread. + pub(super) static STREAM_ABOVE_OVERRIDE: core::cell::Cell> = + const { core::cell::Cell::new(None) }; +} + +fn stream_above_planned_bytes() -> usize { + #[cfg(test)] + if let Some(bytes) = STREAM_ABOVE_OVERRIDE.with(core::cell::Cell::get) { + return bytes; + } + STREAM_ABOVE_PLANNED_BYTES } /// The per-decode bound planned executions are held to. Native planning diff --git a/crates/j2k/src/owned_batch/prepared.rs b/crates/j2k/src/owned_batch/prepared.rs index ede71129..aecf4fba 100644 --- a/crates/j2k/src/owned_batch/prepared.rs +++ b/crates/j2k/src/owned_batch/prepared.rs @@ -46,10 +46,16 @@ pub(super) enum CpuExecution { /// bound and the output has no streamed route (not Gray8 or RGB8), so the /// CPU decoder rejects the image before any decode work. ExceedsDecodeBound, - /// The planned offset-plan execution exceeds the per-decode memory - /// bound, so the image streams rows from the full-output region decode, - /// whose working memory does not grow with the image. + /// A Gray8/RGB8 image whose offset plan exceeds the per-decode bound + /// streams rows from the full-output region decode, whose working memory + /// does not grow with the image. StreamedRows, + /// A Gray8/RGB8 image whose offset plan fits the bound (`bytes` is its + /// parallel peak) but exceeds the streaming threshold. It streams rows + /// like [`Self::StreamedRows`], except as the only image of a Gray8 + /// group: alone in flight, its plan spreads one component's code-block + /// rows over every core, which the streamed route does not match. + StreamedRowsOrPlan { bytes: usize }, } impl CpuExecution { @@ -61,13 +67,21 @@ impl CpuExecution { // The streamed route has no planned peak yet; it is claimed as one // worst-case native decode (measured: 95 MiB for a 5661 x 8287 // RGB8 scan at default threads). - Self::ComponentPlanes | Self::StreamedRows => { + // An optional plan fits the bound, so its peak is within this + // claim too. + Self::ComponentPlanes | Self::StreamedRows | Self::StreamedRowsOrPlan { .. } => { crate::batch::allocation::GENERIC_WORKER_CLAIM_BYTES } Self::ExceedsDecodeBound => 0, } } + /// Whether the CPU decoder may stream this image's rows; groups holding + /// such an image never take the staged route. + pub(super) const fn streams(self) -> bool { + matches!(self, Self::StreamedRows | Self::StreamedRowsOrPlan { .. }) + } + /// The error the offset-plan execution would report, raised up front. pub(super) fn decode_bound_error() -> crate::J2kError { crate::J2kError::from_native_decode_error(j2k_native::ValidationError::ImageTooLarge.into()) From 02f332a6b336e839f0376b9d2abf8ab1e4d7d5d2 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 20:53:04 -0400 Subject: [PATCH 14/94] fix(j2k-native): reconstruct region tiles whose coarse levels are empty A tile whose coarsest resolutions hold no samples while finer ones do (a one-sample tile row or column at an odd position, two or more levels) has no IDWT window for those levels. Both region IDWTs took any level without a window as nothing to reconstruct and gave the component up: the whole-window path cleared it (mid-gray planes for 16-bit and component-plane regions) and the streamed line plan returned no rows (IncompletePlaneStore or CodeBlockDecodeFailure for 8-bit regions). They now skip the leading levels whose decomposition rect is empty and start at the first level with a window, from an empty LL band, as the full-image IDWT does. Changes output where decodes were wrong: over 632 region cases of tiles with empty coarse levels (8- and 16-bit), the decoder gave 187 wrong-pixel results and 316 IncompletePlaneStore/CodeBlockDecodeFailure errors; now all decode and pass the region-vs-full-crop check, or return the Unsupported colour-space error they returned before. Full-image decodes are unchanged. --- crates/j2k-native/src/j2c/idwt/roi.rs | 12 +++ .../fixtures/empty_coarse_levels/README.md | 31 +++++++ .../openjpeg_gray_8x6_tile8x5_53.j2k | Bin 0 -> 175 bytes .../openjpeg_rgb_6x6_tile5x5_97.j2k | Bin 0 -> 317 bytes crates/j2k/tests/region_rows.rs | 80 ++++++++++++++++++ 5 files changed, 123 insertions(+) create mode 100644 crates/j2k-test-support/fixtures/empty_coarse_levels/README.md create mode 100644 crates/j2k-test-support/fixtures/empty_coarse_levels/openjpeg_gray_8x6_tile8x5_53.j2k create mode 100644 crates/j2k-test-support/fixtures/empty_coarse_levels/openjpeg_rgb_6x6_tile5x5_97.j2k diff --git a/crates/j2k-native/src/j2c/idwt/roi.rs b/crates/j2k-native/src/j2c/idwt/roi.rs index 5926c5f6..ba9b9991 100644 --- a/crates/j2k-native/src/j2c/idwt/roi.rs +++ b/crates/j2k-native/src/j2c/idwt/roi.rs @@ -80,6 +80,10 @@ pub(crate) fn roi_idwt_peak_elements( .decompositions .clone() .take(active_len) + .skip_while(|decomposition_idx| { + roi_plan.idwt_window(*decomposition_idx).is_none() + && storage.decompositions[*decomposition_idx].rect.is_empty() + }) .map_while(|decomposition_idx| roi_plan.idwt_window(decomposition_idx)); let rings = ring_samples(windows, WaveletTransform::Irreversible97)?; if storage.lockstep_lines { @@ -228,6 +232,10 @@ pub(super) fn apply_roi( for (local_idx, decomposition) in decompositions.iter().take(active_len).enumerate() { let decomposition_idx = tile_decompositions.decompositions.start + local_idx; let Some(output_window) = roi_plan.idwt_window(decomposition_idx) else { + if !have_current && decomposition.rect.is_empty() { + // A coarse level with no samples: the next level's LL input is empty. + continue; + } output.coefficients.clear(); output.rect = IntRect::from_xywh(0, 0, 0, 0); return Ok(()); @@ -375,6 +383,10 @@ impl LinePlan { // Without a window the level has nothing to reconstruct, as in the // whole-window path. let Some(window) = roi_plan.idwt_window(decomposition_idx) else { + if windows.is_empty() && storage.decompositions[decomposition_idx].rect.is_empty() { + // A coarse level with no samples: the next level's LL input is empty. + continue; + } return Ok(None); }; windows.push(window); diff --git a/crates/j2k-test-support/fixtures/empty_coarse_levels/README.md b/crates/j2k-test-support/fixtures/empty_coarse_levels/README.md new file mode 100644 index 00000000..7d6f564e --- /dev/null +++ b/crates/j2k-test-support/fixtures/empty_coarse_levels/README.md @@ -0,0 +1,31 @@ +# Empty coarse levels fixtures + +Two small images with two decomposition levels whose last tile row (and, in +the RGB image, column) is one sample at an odd position. That tile's two +coarse resolutions are empty (rows `ceil(5/2)` to `ceil(6/2)` and `ceil(5/4)` +to `ceil(6/4)`) while its full resolution is not. A region decode of the tile +must reconstruct it from the first level that has samples, as the full-image +decode does. + +The unsigned 8-bit sample of component `c` at `(x, y)` is: + +```text +(13*x + 29*y + 37*c + 7) mod 251 +``` + +Create a binary PGM (`P5`, 8 x 6, component 0) and PPM (`P6`, 6 x 6, +components 0 to 2) with those row-major samples, then run OpenJPEG 2.5.4: + +```sh +opj_compress -i gray8x6.pgm -o openjpeg_gray_8x6_tile8x5_53.j2k -n 3 -t 8,5 +opj_compress -i rgb6x6.ppm -o openjpeg_rgb_6x6_tile5x5_97.j2k -n 3 -t 5,5 -I +``` + +| File | Contents | SHA-256 | +| --- | --- | --- | +| `openjpeg_gray_8x6_tile8x5_53.j2k` | Gray, 8 x 5 tiles, reversible 5/3; decodes to the samples exactly. | `09b737d7300ea4ddfc81a29740083105594e77c9029a2635c77bd05c98fb4d85` | +| `openjpeg_rgb_6x6_tile5x5_97.j2k` | RGB, 5 x 5 tiles, irreversible 9/7 with the colour transform. | `ea6d998d5d77cc63f86dbda5f9924177848eac1ea25c17cb5ea162ba687561e7` | + +The samples are generated for j2k and contain no externally sourced image +content. OpenJPEG is an independent fixture encoder, not a production codec +dependency. diff --git a/crates/j2k-test-support/fixtures/empty_coarse_levels/openjpeg_gray_8x6_tile8x5_53.j2k b/crates/j2k-test-support/fixtures/empty_coarse_levels/openjpeg_gray_8x6_tile8x5_53.j2k new file mode 100644 index 0000000000000000000000000000000000000000..3c8ab14e5ae5960af190dba072aca25081f41eff GIT binary patch literal 175 zcmezG|38pHlK})cfS3);f)F4+E1b{B&dB&bh=B(vz{tSF!otA#KZb$J!NJ2L0Eqsl zFsL#xIv1rTmZYXABvmT-7o_HS1-QB^l%*CGXXfWA80nennf#x?zy&r@1*m=U`7I*s uiv^R{9sJzP!1L|zfuD>G((h!!Y8Zj4q`_(qh_RpJ;Q6LDMVmkO|4jg08Z4Rs literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/empty_coarse_levels/openjpeg_rgb_6x6_tile5x5_97.j2k b/crates/j2k-test-support/fixtures/empty_coarse_levels/openjpeg_rgb_6x6_tile5x5_97.j2k new file mode 100644 index 0000000000000000000000000000000000000000..9c2fc37758b4a74c2134976966111e248358f886 GIT binary patch literal 317 zcmezG|38pHp8*8epcu>n60A@R;WM){GD6V*AO;?w7$YMS3kw6o{}={Ar}&@%Rv>f_ zzYK&)|5F%L85o_5QWHy3QxuXa75ocQ^SlCF-4)7Gi;6Sz^AwEqO!Z9uPhj8zn{CO! z_8XFTlNpPGD_eDGUS1LN5Qt9lTFqvuILdP76Q0a-4_4{eOU7#i#w8o(MEfm%&L z8V`PDa 50, "{decoded} decodes succeeded"); } + +/// The image decoded into a tight buffer, and its row length. +fn decode_full( + bytes: &[u8], + parallelism: CpuDecodeParallelism, + fmt: PixelFormat, +) -> (Vec, usize) { + let mut decoder = J2kDecoder::new(bytes).expect("decoder"); + decoder.set_cpu_decode_parallelism(parallelism); + let (width, height) = decoder.info().dimensions; + let stride = width as usize * fmt.bytes_per_pixel(); + let mut out = vec![0; stride * height as usize]; + decoder + .decode_into(&mut out, stride, fmt) + .expect("full decode"); + (out, stride) +} + +/// A region of a tile whose coarse resolutions are empty (a one-sample tile +/// row or column at an odd position, with two levels) decodes the full +/// image's pixels, on the streamed rows and on the planes, at both bit depths. +#[test] +fn region_of_tile_with_empty_coarse_levels_matches_full_decode() { + let gray: &[u8] = include_bytes!( + "../../j2k-test-support/fixtures/empty_coarse_levels/openjpeg_gray_8x6_tile8x5_53.j2k" + ); + let rgb: &[u8] = include_bytes!( + "../../j2k-test-support/fixtures/empty_coarse_levels/openjpeg_rgb_6x6_tile5x5_97.j2k" + ); + let fixtures = [ + ( + "gray 5/3", + gray, + [PixelFormat::Gray8, PixelFormat::Gray16], + [(0, 5, 8, 1), (3, 5, 1, 1), (2, 3, 5, 3), (0, 0, 8, 6)], + ), + ( + "rgb 9/7", + rgb, + [PixelFormat::Rgb8, PixelFormat::Rgb16], + [(5, 0, 1, 6), (5, 5, 1, 1), (0, 5, 6, 1), (3, 2, 3, 4)], + ), + ]; + for (label, bytes, formats, regions) in fixtures { + for fmt in formats { + let bytes_per_pixel = fmt.bytes_per_pixel(); + for parallelism in [CpuDecodeParallelism::Auto, CpuDecodeParallelism::Serial] { + let (full, full_stride) = decode_full(bytes, parallelism, fmt); + for (x, y, w, h) in regions { + let roi = Rect { x, y, w, h }; + let stride = w as usize * bytes_per_pixel; + let mut out = vec![0; stride * h as usize]; + let mut decoder = J2kDecoder::new(bytes).expect("decoder"); + decoder.set_cpu_decode_parallelism(parallelism); + decoder + .decode_region_into(&mut J2kScratchPool::new(), &mut out, stride, fmt, roi) + .unwrap_or_else(|error| { + panic!("{label} {fmt:?} {parallelism:?} {roi:?}: {error:?}") + }); + let crop: Vec = full + .chunks(full_stride) + .skip(y as usize) + .take(h as usize) + .flat_map(|row| &row[x as usize * bytes_per_pixel..][..stride]) + .copied() + .collect(); + assert_eq!(out, crop, "{label} {fmt:?} {parallelism:?} {roi:?}"); + } + } + } + } + // The reversible image decodes losslessly. + let expected: Vec = (0..6_u32) + .flat_map(|y| (0..8_u32).map(move |x| ((13 * x + 29 * y + 7) % 251) as u8)) + .collect(); + assert_eq!( + decode_full(gray, CpuDecodeParallelism::Auto, PixelFormat::Gray8).0, + expected + ); +} From a18efc1dd6165362ba8d60cb3a6e64356c2a3e86 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 03:53:34 -0400 Subject: [PATCH 15/94] perf(j2k-native): gather multi-segment block payloads as the blocks decode The tile and band routes copied the payload of every block whose segments span several packets into one arena per route before any block decoded, serially on the calling thread. A full-image streamed decode routes every block of the tile, and with several layers that is every payload, so the arenas held the tile's coded bytes for the whole decode: 17.4 MiB on a 5661x8287 RGB image with 8 layers, 57.6 MiB on an 18000x12000 one. Each block now gathers its segments from the tile's layer and segment records into its thread's scratch just before it decodes, through a borrowed `PayloadSource`; the scratch grows to the largest gathered payload and is charged per thread instead of the arena. Full-ROI streamed decode, 18 threads, heap above the output: 102.8 -> 85.6 MiB (5661x8287, 8 layers), 286.7 -> 229.2 MiB (18000x12000); at 1/2 scale 55.9 -> 44.4 MiB. Time unchanged within noise (110.6 -> 109.3 ms at 18 threads, 1365 -> 1364 ms at one thread). Output is identical. --- .../src/j2c/decode/parallel_bands.rs | 38 ++- crates/j2k-native/src/j2c/decode/subband.rs | 3 +- .../src/j2c/decode/subband/tile_parallel.rs | 280 +++++++++++------- 3 files changed, 193 insertions(+), 128 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index d9cc62e7..b4d1bd67 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -43,7 +43,8 @@ use std::sync::{Condvar, Mutex, MutexGuard}; use super::banded_tier1::{self, RowOutput}; use super::subband::{ - plan_band_route, plan_band_route_beside, routes_tile, BandRoute, BandRouting, + plan_band_route, plan_band_route_beside, routes_tile, BandRoute, BandRouting, BlockScratch, + PayloadSource, }; use super::workspace::{TaskScratch, TaskWorkspace}; use super::{ @@ -234,6 +235,7 @@ fn decode_route( let strand_once = false; let mut coefficients = core::mem::take(&mut storage.coefficients); let slots = split_slots(&rings, &bands, storage, &mut coefficients); + let source = PayloadSource::new(storage); let result = slots.and_then(|slots| { let run = |rows: &mut [&mut (dyn SubBandRows + Send)]| -> Result<()> { match (lines.as_mut(), rows.first_mut()) { @@ -249,7 +251,7 @@ fn decode_route( slots, &mut tile_ctx.classic_task_workspaces, |index, output, workspace, scratch| { - route.decode_classic(index, output, workspace, scratch) + route.decode_classic(index, output, workspace, (scratch, source)) }, run, profile_enabled, @@ -263,7 +265,7 @@ fn decode_route( slots, &mut tile_ctx.ht_task_workspaces, |index, output, workspace, scratch| { - route.decode_ht(index, output, workspace, scratch) + route.decode_ht(index, output, workspace, (scratch, source)) }, run, profile_enabled, @@ -403,6 +405,7 @@ pub(super) fn decode_tile_rows<'a>( let mut output_samples = 0usize; let mut coefficients = core::mem::take(&mut storage.coefficients); let slots = split_slots(&rings, &bands, storage, &mut coefficients); + let source = PayloadSource::new(storage); let result = slots.and_then(|slots| { let rings_len = buffer.len() - scratch_len; let (mut line_rings, scratch) = buffer.split_at_mut(rings_len); @@ -429,7 +432,7 @@ pub(super) fn decode_tile_rows<'a>( &mut tile_ctx.classic_task_workspaces, |strip, output, workspace, scratch| { let (route, strip) = route_strip(&routes, &strip_starts, strip)?; - route.decode_classic(strip, output, workspace, scratch) + route.decode_classic(strip, output, workspace, (scratch, source)) }, run, profile_enabled, @@ -444,7 +447,7 @@ pub(super) fn decode_tile_rows<'a>( &mut tile_ctx.ht_task_workspaces, |strip, output, workspace, scratch| { let (route, strip) = route_strip(&routes, &strip_starts, strip)?; - route.decode_ht(strip, output, workspace, scratch) + route.decode_ht(strip, output, workspace, (scratch, source)) }, run, profile_enabled, @@ -1027,7 +1030,7 @@ fn run_bands( bands: &[Band], slots: Slots<'_>, bank: &mut [TaskWorkspace], - decode: impl Fn(usize, &mut [f32], &mut W, &mut Vec) -> Result<()> + Sync, + decode: impl Fn(usize, &mut [f32], &mut W, &mut BlockScratch) -> Result<()> + Sync, run: impl FnOnce(&mut [&mut (dyn SubBandRows + Send)]) -> Result<()>, profile_enabled: bool, tier1_us: &mut u128, @@ -1101,7 +1104,7 @@ fn run_bands( scope, decode, workspace, - clipped_scratch: Vec::new(), + scratch: BlockScratch::default(), current: try_filled_with(rings.len(), || None)?, next: try_collect(rings.iter().map(|ring| ring.bands.start), rings.len())?, helps, @@ -1178,7 +1181,7 @@ fn try_filled_with(len: usize, value: impl FnMut() -> T) -> Result> { /// A pool worker: decode ready bands until none is left to start. fn work( shared: &Shared<'_, '_, '_, W>, - decode: &(impl Fn(usize, &mut [f32], &mut W, &mut Vec) -> Result<()> + Sync), + decode: &(impl Fn(usize, &mut [f32], &mut W, &mut BlockScratch) -> Result<()> + Sync), ) { let Ok(mut state) = shared.lock() else { return; @@ -1187,7 +1190,7 @@ fn work( state.active -= 1; return; }; - let mut clipped_scratch = Vec::new(); + let mut scratch = BlockScratch::default(); loop { let next = if state.error.is_some() { Ok(None) @@ -1215,12 +1218,7 @@ fn work( return; } slot.fill(0.0); - let result = decode( - shared.bands[index].strip, - slot, - workspace, - &mut clipped_scratch, - ); + let result = decode(shared.bands[index].strip, slot, workspace, &mut scratch); if shared.finish(index, slot, result).is_err() { return; } @@ -1242,7 +1240,7 @@ struct PoolRows<'s, 'p, 'c, 'w, W, D> { scope: &'s rayon::Scope<'p>, decode: &'p D, workspace: &'w mut W, - clipped_scratch: Vec, + scratch: BlockScratch, /// Each ring's band being read, with its slot. current: Vec>, /// Each ring's next band to read. @@ -1257,7 +1255,7 @@ struct PoolRows<'s, 'p, 'c, 'w, W, D> { impl<'p, 'c, W, D> PoolRows<'_, 'p, 'c, '_, W, D> where W: TaskScratch + 'p, - D: Fn(usize, &mut [f32], &mut W, &mut Vec) -> Result<()> + Sync, + D: Fn(usize, &mut [f32], &mut W, &mut BlockScratch) -> Result<()> + Sync, 'c: 'p, { /// Send pool workers, up to the plan's count, while bands are ready. @@ -1363,7 +1361,7 @@ where self.shared.bands[index].strip, slot, self.workspace, - &mut self.clipped_scratch, + &mut self.scratch, ); self.tier1_us += profile::elapsed_us(start); self.shared.finish(index, slot, result) @@ -1376,7 +1374,7 @@ where self.shared.bands[index].strip, slot, self.workspace, - &mut self.clipped_scratch, + &mut self.scratch, ); self.tier1_us += profile::elapsed_us(start); result?; @@ -1453,7 +1451,7 @@ where impl<'p, 'c, W, D> SubBandRows for PoolRows<'_, 'p, 'c, '_, W, D> where W: TaskScratch + 'p, - D: Fn(usize, &mut [f32], &mut W, &mut Vec) -> Result<()> + Sync, + D: Fn(usize, &mut [f32], &mut W, &mut BlockScratch) -> Result<()> + Sync, 'c: 'p, { fn prepare_row(&mut self, sub_band: usize, y: u32) -> Result<()> { diff --git a/crates/j2k-native/src/j2c/decode/subband.rs b/crates/j2k-native/src/j2c/decode/subband.rs index 6aea6dd7..5e80a706 100644 --- a/crates/j2k-native/src/j2c/decode/subband.rs +++ b/crates/j2k-native/src/j2c/decode/subband.rs @@ -44,7 +44,8 @@ pub(crate) use self::placement::BlockPlacement; pub(crate) use self::tile_parallel::MIN_BAND_WORK; #[cfg(feature = "parallel")] pub(in crate::j2c::decode) use self::tile_parallel::{ - plan_band_route, plan_band_route_beside, routes_tile, BandRoute, BandRouting, + plan_band_route, plan_band_route_beside, routes_tile, BandRoute, BandRouting, BlockScratch, + PayloadSource, }; /// Decode the code-blocks of the tile's `components` into their sub-bands. diff --git a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs index 5682cc24..ce24006d 100644 --- a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs +++ b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs @@ -8,9 +8,11 @@ //! blocks owns a contiguous range of its sub-band's coefficients, so tasks //! write their output in place through disjoint slices and Rayon balances //! them with work stealing. HT cleanup-only and single-segment classic -//! payloads are borrowed from the codestream; only payloads that span several -//! segments are gathered. Classic blocks take this route when each decodes as -//! one arithmetic segment: no selective bypass or per-pass termination. +//! payloads are borrowed from the codestream; a payload that spans several +//! segments is gathered from the block's segment records into its worker's +//! scratch just before the block decodes. Classic blocks take this route when +//! each decodes as one arithmetic segment: no selective bypass or per-pass +//! termination. use alloc::boxed::Box; use alloc::vec::Vec; @@ -28,7 +30,7 @@ use super::{ J2kCodeBlockDecodeWorkspace, Result, Tile, TileDecodeContext, MAX_BITPLANE_COUNT, }; use crate::error::{DecodingError, ValidationError}; -use crate::j2c::build::{CodeBlock, CodeBlockCoding}; +use crate::j2c::build::{CodeBlock, CodeBlockCoding, Layer, Segment}; use crate::j2c::codestream::CodeBlockStyle; use crate::j2c::decode::direct_plan::classic_allocation_counts; use crate::j2c::decode::peak::WorkspaceBytes; @@ -126,8 +128,36 @@ struct BandParameters { enum Payload<'a> { /// A single segment borrowed from the codestream. Borrowed(&'a [u8]), - /// Gathered segments starting at this offset of the payload arena. - Gathered(usize), + /// Segments gathered from the block's records into the decoding + /// worker's scratch. + Gathered, +} + +/// The records a gathered payload is read from while blocks decode: every +/// code-block, layer and segment of the tile, borrowed from its storage. +#[derive(Clone, Copy)] +pub(in crate::j2c::decode) struct PayloadSource<'s, 'a> { + code_blocks: &'s [CodeBlock], + layers: &'s [Layer], + segments: &'s [Segment<'a>], +} + +impl<'s, 'a> PayloadSource<'s, 'a> { + pub(in crate::j2c::decode) fn new(storage: &'s DecompositionStorage<'a>) -> Self { + Self { + code_blocks: &storage.code_blocks, + layers: &storage.layers, + segments: &storage.segments, + } + } +} + +/// A worker's scratch for the blocks it decodes: a block its sub-band stores +/// only part of, and a payload gathered from several segments. +#[derive(Default)] +pub(in crate::j2c::decode) struct BlockScratch { + clipped: Vec, + payload: Vec, } struct BlockJob<'a> { @@ -163,9 +193,9 @@ struct TilePlan<'a> { strict: bool, jobs: Vec>, strips: Vec, - arena: Vec, - /// Bytes of the payloads to gather into `arena` once the route is taken. - arena_len: usize, + /// Bytes of the largest payload gathered from several segments. Each + /// worker gathers such payloads into scratch of this size. + max_gathered_len: usize, skipped: usize, coded_bytes: usize, max_width: u32, @@ -228,7 +258,7 @@ pub(in crate::j2c::decode) fn plan_tile_route<'a>( return Ok(route); } let mut budget = DecodeAllocationBudget::for_storage(storage)?; - let mut plan = match plan_tile( + let plan = match plan_tile( tile, components, storage, @@ -259,8 +289,8 @@ pub(in crate::j2c::decode) fn plan_tile_route<'a>( let workers = rayon::current_num_threads().min(plan.strips.len()); let counts = TaskCounts::new(&plan, workers, storage.stale_coefficients)?; let route_budget = (|| -> Result<()> { - // Payloads are gathered only once the tile takes the route. - gather_payloads(&mut plan, storage, &mut budget)?; + // Each worker gathers payloads into scratch of its own as it decodes. + include_payload_scratch(&mut budget, &plan, workers)?; // `plan_tile` charged one task per strip. budget.include_elements::>(counts.tasks - plan.strips.len())?; budget.include_elements::>(counts.shared)?; @@ -317,8 +347,8 @@ fn decode_with_retained_bank<'a>( &mut tile_ctx.classic_task_workspaces, storage, budget, - |job, output, stride, workspace| { - decode_classic_strip_block(plan, job, output, stride, workspace) + |job, payload, output, stride, workspace| { + decode_classic_strip_block(plan, job, payload, output, stride, workspace) }, ), CodeBlockCoding::HighThroughput => decode_with_bank( @@ -328,8 +358,8 @@ fn decode_with_retained_bank<'a>( &mut tile_ctx.ht_task_workspaces, storage, budget, - |job, output, stride, workspace| { - decode_strip_block(plan, job, output, stride, workspace) + |job, payload, output, stride, workspace| { + decode_strip_block(plan, job, payload, output, stride, workspace) }, ), } @@ -385,7 +415,7 @@ fn record_route_peak( plan.jobs.capacity() * size_of::>(), plan.strips.capacity() * size_of::(), plan.band_sub_bands.capacity() * size_of::(), - plan.arena.capacity(), + workers.saturating_mul(plan.max_gathered_len), counts.tasks * size_of::>(), counts.shared * size_of::>(), workers @@ -410,7 +440,7 @@ fn decode_with_bank<'a, W: TaskScratch>( bank: &mut Vec>, storage: &mut DecompositionStorage<'a>, budget: &mut DecodeAllocationBudget, - decode_block: impl Fn(&BlockJob<'_>, &mut [f32], usize, &mut W) -> Result<()> + Sync, + decode_block: impl Fn(&BlockJob<'_>, &[u8], &mut [f32], usize, &mut W) -> Result<()> + Sync, ) -> Result> { let Ok(growths) = prepare_uniform_task_workspaces( bank, @@ -430,7 +460,10 @@ fn decode_with_bank<'a, W: TaskScratch>( stale, &mut bank[..workers], counts, - |job, output, stride, slot| decode_block(job, output, stride, &mut slot.workspace), + PayloadSource::new(storage), + |job, payload, output, stride, slot| { + decode_block(job, payload, output, stride, &mut slot.workspace) + }, ); storage.coefficients = coefficients; result?; @@ -488,8 +521,7 @@ fn plan_tile<'a>( strict: header.strict, jobs: Vec::new(), strips: Vec::new(), - arena: Vec::new(), - arena_len: 0, + max_gathered_len: 0, skipped: 0, coded_bytes: 0, max_width: 0, @@ -507,8 +539,6 @@ fn plan_tile<'a>( } else { budget.reserve_new(&mut plan.bands, storage.sub_bands.len())?; } - let mut arena_len = 0usize; - let component_infos = tile .component_infos .get(components.clone()) @@ -592,9 +622,12 @@ fn plan_tile<'a>( stored_placement(code_block, sub_band), band, storage, - &mut arena_len, classic_style, )?; + if matches!(job.payload, Payload::Gathered) { + plan.max_gathered_len = + plan.max_gathered_len.max(job.payload_length as usize); + } if !job.placement.is_whole(job.width, job.height) { let samples = (job.width as usize) .checked_mul(job.height as usize) @@ -634,7 +667,6 @@ fn plan_tile<'a>( if !banded && !strips_are_disjoint(&mut plan.strips, storage.coefficients.len()) { return Ok(None); } - plan.arena_len = arena_len; budget.include_elements::>(plan.strips.len())?; Ok(Some(plan)) } @@ -709,29 +741,20 @@ fn required_code_blocks( } /// A block's job. A payload in one codestream segment is borrowed; others -/// get room in the arena. +/// are gathered when the block decodes. fn plan_block<'a>( code_block: &CodeBlock, code_block_idx: usize, placement: BlockPlacement, band: usize, storage: &DecompositionStorage<'a>, - arena_len: &mut usize, classic_style: Option, ) -> Result> { let (payload_length, refinement_length, single) = match classic_style { Some(style) => classic_payload(code_block, style, storage)?, None => ht_payload(code_block, storage)?, }; - let payload = if let Some(data) = single { - Payload::Borrowed(data) - } else { - let start = *arena_len; - *arena_len = start - .checked_add(payload_length) - .ok_or(ValidationError::ImageTooLarge)?; - Payload::Gathered(start) - }; + let payload = single.map_or(Payload::Gathered, Payload::Borrowed); Ok(BlockJob { payload, code_block: code_block_idx, @@ -903,56 +926,75 @@ fn strips_are_disjoint(strips: &mut [Strip], coefficient_len: usize) -> bool { previous_end <= coefficient_len } -fn gather_payloads<'a>( - plan: &mut TilePlan<'a>, - storage: &DecompositionStorage<'a>, +/// Charge the payload scratch each of `threads` threads grows to as it +/// gathers the plan's payloads. +fn include_payload_scratch( budget: &mut DecodeAllocationBudget, + plan: &TilePlan<'_>, + threads: usize, ) -> Result<()> { - if plan.arena_len == 0 { - return Ok(()); - } - budget.reserve_new(&mut plan.arena, plan.arena_len)?; - for job in &mut plan.jobs { - let Payload::Gathered(_) = job.payload else { + budget.include_bytes( + threads + .checked_mul(plan.max_gathered_len) + .ok_or(ValidationError::ImageTooLarge)?, + ) +} + +/// Gather the payload of `job`, a block whose payload spans several +/// segments, into `scratch`: for a classic block every segment, for an HT +/// block the selected set's cleanup and refinement segments. +fn gather_payload<'s>( + coding: CodeBlockCoding, + job: &BlockJob<'_>, + source: PayloadSource<'_, '_>, + scratch: &'s mut Vec, +) -> Result<&'s [u8]> { + let expected = job.payload_length as usize; + scratch.clear(); + scratch + .try_reserve(expected) + .map_err(|_| DecodingError::HostAllocationFailed)?; + let code_block = source + .code_blocks + .get(job.code_block) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let cleanup_index = if coding == CodeBlockCoding::Classic { + None + } else { + let selected_set = code_block + .ht_selected_set + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + Some( + selected_set + .checked_mul(2) + .ok_or(DecodingError::CodeBlockDecodeFailure)?, + ) + }; + let layers = source + .layers + .get(code_block.layers.clone()) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + for layer in layers { + let Some(range) = layer.segments.clone() else { continue; }; - // Row sorting can change precinct traversal order. Assign the offset - // when the payload is appended in its final execution order. - let start = plan.arena.len(); - job.payload = Payload::Gathered(start); - let code_block = &storage.code_blocks[job.code_block]; - // A classic payload is every segment; an HT payload the selected - // set's cleanup and refinement segments. - let cleanup_index = if plan.coding == CodeBlockCoding::Classic { - None - } else { - let selected_set = code_block - .ht_selected_set - .ok_or(DecodingError::CodeBlockDecodeFailure)?; - Some( - selected_set - .checked_mul(2) - .ok_or(DecodingError::CodeBlockDecodeFailure)?, - ) - }; - for layer in &storage.layers[code_block.layers.clone()] { - let Some(range) = layer.segments.clone() else { - continue; - }; - for segment in &storage.segments[range] { - if cleanup_index - .is_none_or(|index| segment.idx == index || segment.idx == index + 1) - { - plan.arena.extend_from_slice(segment.data); + let segments = source + .segments + .get(range) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + for segment in segments { + if cleanup_index.is_none_or(|index| segment.idx == index || segment.idx == index + 1) { + if scratch.len() + segment.data.len() > expected { + return Err(DecodingError::CodeBlockDecodeFailure.into()); } + scratch.extend_from_slice(segment.data); } } - let expected = job.payload_length as usize; - if plan.arena.len() - start != expected { - return Err(DecodingError::CodeBlockDecodeFailure.into()); - } } - Ok(()) + if scratch.len() != expected { + return Err(DecodingError::CodeBlockDecodeFailure.into()); + } + Ok(scratch) } fn charge_actual_workspace( @@ -1180,7 +1222,8 @@ fn decode_strips<'plan, 'coefficients, W: Send>( stale: bool, workspaces: &mut [W], counts: TaskCounts, - decode_block: impl Fn(&BlockJob<'_>, &mut [f32], usize, &mut W) -> Result<()> + Sync, + source: PayloadSource<'_, '_>, + decode_block: impl Fn(&BlockJob<'_>, &[u8], &mut [f32], usize, &mut W) -> Result<()> + Sync, ) -> Result<()> { let mut shared: Vec> = Vec::new(); shared @@ -1241,7 +1284,7 @@ fn decode_strips<'plan, 'coefficients, W: Send>( .par_iter_mut() .with_max_len(1) .try_for_each(|workspace| -> Result<()> { - let mut scratch = Vec::new(); + let mut scratch = BlockScratch::default(); while !failed.load(Ordering::Relaxed) { let next = queue .lock() @@ -1259,7 +1302,14 @@ fn decode_strips<'plan, 'coefficients, W: Send>( if stale { output.fill(0.0); } - decode_strip(plan, strip, output, workspace, &mut scratch, &decode_block) + decode_strip( + plan, + strip, + output, + workspace, + (&mut scratch, source), + &decode_block, + ) } StripTask::Part(output, strip, jobs, _) => decode_part( plan, @@ -1269,7 +1319,7 @@ fn decode_strips<'plan, 'coefficients, W: Send>( stale, counts.scratch_samples, workspace, - &mut scratch, + (&mut scratch, source), &decode_block, ), }; @@ -1297,9 +1347,13 @@ fn decode_part( stale: bool, scratch_samples: usize, workspace: &mut W, - scratch: &mut Vec, - decode_block: &impl Fn(&BlockJob<'_>, &mut [f32], usize, &mut W) -> Result<()>, + (scratch, source): (&mut BlockScratch, PayloadSource<'_, '_>), + decode_block: &impl Fn(&BlockJob<'_>, &[u8], &mut [f32], usize, &mut W) -> Result<()>, ) -> Result<()> { + let BlockScratch { + clipped: scratch, + payload: payload_scratch, + } = scratch; let lock = || { output .lock() @@ -1345,7 +1399,8 @@ fn decode_part( if !placement.is_whole(job.width, job.height) { block.fill(0.0); } - decode_block(job, block, job.width as usize, workspace)?; + let payload = job_payload(plan, job, source, payload_scratch)?; + decode_block(job, payload, block, job.width as usize, workspace)?; placement.copy_from_block(block, job.width, &mut lock()?, strip.stride)?; } Ok(()) @@ -1356,9 +1411,13 @@ fn decode_strip( strip: &Strip, output: &mut [f32], workspace: &mut W, - clipped_scratch: &mut Vec, - decode_block: &impl Fn(&BlockJob<'_>, &mut [f32], usize, &mut W) -> Result<()>, + (scratch, source): (&mut BlockScratch, PayloadSource<'_, '_>), + decode_block: &impl Fn(&BlockJob<'_>, &[u8], &mut [f32], usize, &mut W) -> Result<()>, ) -> Result<()> { + let BlockScratch { + clipped: clipped_scratch, + payload: payload_scratch, + } = scratch; for job in &plan.jobs[strip.jobs.clone()] { // The strip starts at the block row's first stored row. let placement = BlockPlacement { @@ -1382,7 +1441,8 @@ fn decode_strip( block.fill(0.0); (block, job.width as usize) }; - decode_block(job, block_output, output_stride, workspace)?; + let payload = job_payload(plan, job, source, payload_scratch)?; + decode_block(job, payload, block_output, output_stride, workspace)?; if !whole { placement.copy_from_block(clipped_scratch, job.width, output, strip.stride)?; } @@ -1393,13 +1453,14 @@ fn decode_strip( fn decode_strip_block( plan: &TilePlan<'_>, job: &BlockJob<'_>, + payload: &[u8], output: &mut [f32], output_stride: usize, workspace: &mut HtCodeBlockDecodeWorkspace, ) -> Result<()> { let band = plan.bands[job.band]; let decode_job = HtCodeBlockDecodeJob { - data: job_payload(plan, job)?, + data: payload, cleanup_length: job.payload_length - job.refinement_length, refinement_length: job.refinement_length, width: job.width, @@ -1433,6 +1494,7 @@ fn decode_strip_block( fn decode_classic_strip_block( plan: &TilePlan<'_>, job: &BlockJob<'_>, + payload: &[u8], output: &mut [f32], output_stride: usize, workspace: &mut J2kCodeBlockDecodeWorkspace, @@ -1446,7 +1508,7 @@ fn decode_classic_strip_block( use_arithmetic: true, }]; let decode_job = band.job( - job_payload(plan, job)?, + payload, &segments, job.width, job.height, @@ -1457,14 +1519,17 @@ fn decode_classic_strip_block( band.decode(decode_job, output, workspace) } -/// The job's payload, borrowed from the codestream or from the arena. -fn job_payload<'p>(plan: &'p TilePlan<'_>, job: &BlockJob<'p>) -> Result<&'p [u8]> { +/// The job's payload, borrowed from the codestream or gathered into +/// `scratch`. +fn job_payload<'p>( + plan: &TilePlan<'_>, + job: &BlockJob<'p>, + source: PayloadSource<'_, '_>, + scratch: &'p mut Vec, +) -> Result<&'p [u8]> { match job.payload { Payload::Borrowed(data) => Ok(data), - Payload::Gathered(start) => plan - .arena - .get(start..start + job.payload_length as usize) - .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()), + Payload::Gathered => gather_payload(plan.coding, job, source, scratch), } } @@ -1558,7 +1623,7 @@ pub(in crate::j2c::decode) fn plan_band_route_beside<'a>( Some(live) => live, None => DecodeAllocationBudget::for_storage(storage)?, }; - let mut plan = match plan_tile( + let plan = match plan_tile( tile, components, storage, @@ -1600,8 +1665,9 @@ pub(in crate::j2c::decode) fn plan_band_route_beside<'a>( } let workers = threads.min(plan.strips.len()); let prepared = (|| -> Result { - gather_payloads(&mut plan, storage, &mut budget)?; - // Each thread allocates its clipped-block scratch on first use. + // Each thread gathers payloads into scratch of its own as it + // decodes, and allocates its clipped-block scratch on first use. + include_payload_scratch(&mut budget, &plan, workers + 1)?; budget.include_elements::( (workers + 1) .checked_mul(plan.max_clipped_samples) @@ -1677,7 +1743,7 @@ impl BandRoute<'_> { index: usize, output: &mut [f32], workspace: &mut J2kCodeBlockDecodeWorkspace, - clipped_scratch: &mut Vec, + scratch: (&mut BlockScratch, PayloadSource<'_, '_>), ) -> Result<()> { let plan = &self.plan; decode_strip( @@ -1685,9 +1751,9 @@ impl BandRoute<'_> { &plan.strips[index], output, workspace, - clipped_scratch, - &|job, output, stride, workspace| { - decode_classic_strip_block(plan, job, output, stride, workspace) + scratch, + &|job, payload, output, stride, workspace| { + decode_classic_strip_block(plan, job, payload, output, stride, workspace) }, ) } @@ -1699,7 +1765,7 @@ impl BandRoute<'_> { index: usize, output: &mut [f32], workspace: &mut HtCodeBlockDecodeWorkspace, - clipped_scratch: &mut Vec, + scratch: (&mut BlockScratch, PayloadSource<'_, '_>), ) -> Result<()> { let plan = &self.plan; decode_strip( @@ -1707,9 +1773,9 @@ impl BandRoute<'_> { &plan.strips[index], output, workspace, - clipped_scratch, - &|job, output, stride, workspace| { - decode_strip_block(plan, job, output, stride, workspace) + scratch, + &|job, payload, output, stride, workspace| { + decode_strip_block(plan, job, payload, output, stride, workspace) }, ) } From a553abdddf81d08e0e2f4a2029d97c12d0249c6d Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 04:59:29 -0400 Subject: [PATCH 16/94] perf(j2k-native): decode classic pool bands in parts when the rings hold too few bands A streamed decode's pool can only run as many band decodes as the finest level's rings hold bands ahead of the readers. With one component that is two bands in each of three rings, so six of eighteen threads decode Tier-1 at the finest level while the reader decodes the bands it waits for itself: a 5792x5792 gray image streamed in 158 ms against 63 ms for the component planes. When the finest rings hold fewer bands ahead than the executor has threads, each classic band now splits into runs of its blocks of about equal work (`split_runs`, as the tile route splits its strips): enough parts per finest band for two parts per thread (ceil(2 x threads / bands ahead), at most 8), so a thread that finishes a part finds another while a band's last parts finish, and parts of about the same work at the other levels. Threads decode a band's parts at once into its slot, each zeroing its own columns and copying its blocks in under the slot's lock (the tile route's `decode_part`); the last part to finish hands the band over. A ring's slots are cut into whole rows of its sub-band, at least as many as its longest band's, since a part zeroes its columns of the slot row by row. A reader waiting for a running band decodes its unstarted parts. RGB at 18 threads with today's rings has bands enough and plans no parts. Full-ROI streamed decode at the default pool (18 threads): 5792 x 5792 gray 158.0 -> 61.2 ms, 8192 x 8192 gray 322 -> 134 ms; on 6 threads 5792 x 5792 gray 163 -> 135 ms. RGB unchanged (5661 x 8287 112.8 -> 108.5 ms, 4096 x 4096 54.6 -> 52.8 ms, within noise). One part per thread instead of two left the 5792 x 5792 gray image at about 80 ms. Output is identical; new tests force splits of 2, 3, 5 and 7 parts against whole-window Tier-1 and the serial component planes, also for regions that clip the only code-block rows that code anything. --- crates/j2k-native/src/j2c/decode.rs | 2 + .../src/j2c/decode/parallel_bands.rs | 369 ++++++++++++++---- crates/j2k-native/src/j2c/decode/state.rs | 4 + crates/j2k-native/src/j2c/decode/subband.rs | 4 +- .../src/j2c/decode/subband/tile_parallel.rs | 118 +++++- crates/j2k-native/src/tests/parallel_bands.rs | 138 ++++++- crates/j2k-native/src/tests/row_sink.rs | 78 ++++ 7 files changed, 621 insertions(+), 92 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode.rs b/crates/j2k-native/src/j2c/decode.rs index a9d974f4..860259d9 100644 --- a/crates/j2k-native/src/j2c/decode.rs +++ b/crates/j2k-native/src/j2c/decode.rs @@ -698,6 +698,8 @@ pub(crate) struct DecodeDebugCounters { /// Components that gave up the pool for whole windows because a band /// was left running with no worker to finish it. pub(crate) pool_band_fallbacks: usize, + /// Pool bands planned in parts, summed over decoded tiles. + pub(crate) pool_band_parts: usize, /// The bytes the decode's budget starts from after `reset`, retained /// capacity included. pub(crate) decode_baseline_bytes: usize, diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index b4d1bd67..3d9bd39e 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -17,6 +17,10 @@ //! - The IDWT waits only for a band a worker is decoding. A band nobody has //! started it decodes itself, so it never depends on a pool thread being //! free, even when the pool is busy with other work. +//! - When the finest level's slots ahead of the IDWT hold fewer bands than +//! the pool has threads, as with one component, a classic band splits into +//! parts, runs of its blocks that threads decode at once into the band's +//! slot, so every thread has work without holding more bands. //! //! Every band is decoded once, including bands the IDWT never reads, so a //! decode visits, decodes and rejects the code-blocks the whole-window route @@ -43,8 +47,8 @@ use std::sync::{Condvar, Mutex, MutexGuard}; use super::banded_tier1::{self, RowOutput}; use super::subband::{ - plan_band_route, plan_band_route_beside, routes_tile, BandRoute, BandRouting, BlockScratch, - PayloadSource, + plan_band_route, plan_band_route_beside, routes_tile, BandOutput, BandRoute, BandRouting, + BlockScratch, PayloadSource, }; use super::workspace::{TaskScratch, TaskWorkspace}; use super::{ @@ -212,7 +216,16 @@ fn decode_route( profile_enabled: bool, tier1_us: &mut u128, ) -> Result { - let (rings, bands, active_levels) = plan_rings(route, storage, header, component_idx)?; + let (rings, mut bands, active_levels) = plan_rings(route, storage, header, component_idx)?; + if route.coding() == CodeBlockCoding::Classic { + split_bands( + tile_ctx, + &rings, + &mut bands, + route.workers() + 1, + |band, parts| route.strip_parts(band.strip, parts), + )?; + } // The executor's records live beside the route's plan and scratch while // the bands decode. route @@ -340,7 +353,8 @@ pub(super) fn decode_tile_rows<'a>( } } } - let (rings, bands, strip_starts, active_levels) = plan_tile_rings(&routes, storage, header)?; + let (rings, mut bands, strip_starts, active_levels) = + plan_tile_rings(&routes, storage, header)?; let line_plans = plan_tile_lines(tile, header, tile_ctx, storage, &active_levels)?; // The executor's records live beside every route's plan and scratch // while the bands decode. @@ -397,6 +411,18 @@ pub(super) fn decode_tile_rows<'a>( // One reader for every component of a classic tile leaves the pool's // threads to Tier-1, and decodes bands itself while it waits. let helps = readers == 1 && coding == CodeBlockCoding::Classic; + if coding == CodeBlockCoding::Classic { + split_bands( + tile_ctx, + &rings, + &mut bands, + workers + readers, + |band, parts| { + let (route, strip) = route_strip(&routes, &strip_starts, band.strip)?; + route.strip_parts(strip, parts) + }, + )?; + } #[cfg(test)] let strand_once = tile_ctx.strand_pool_band; #[cfg(not(test))] @@ -657,7 +683,9 @@ fn executor_bytes(rings: &[Ring], bands: &[Band]) -> Result { + size_of::() + size_of::() + 2 * size_of::>() - + size_of::, usize)>>(); + + size_of::, usize)>>() + + 2 * size_of::() + + size_of::>(); let per_ring = size_of::() + 3 * size_of::() + size_of::>() @@ -677,6 +705,8 @@ struct Ring { sub_band: usize, stored: IntRect, slots: usize, + /// Whether the sub-band belongs to the finest level the IDWT reads. + finest: bool, /// Whether the ring holds the sub-band's whole window, because its bands /// fit in fewer rows than its slots would: each band then decodes into /// its own rows of the window, and all of them can be in flight at once. @@ -694,6 +724,10 @@ struct Band { /// component, approximately: bands it reads sooner start sooner. order: i64, work: usize, + /// Runs of its blocks the band decodes in, on as many threads at once. + parts: usize, + /// The count its blocks were split for, which gives those `parts` runs. + split: usize, } /// The component's rings, its bands grouped by ring in row order, and the @@ -744,6 +778,7 @@ fn plan_rings( sub_band, stored, slots, + finest: resolution == active_levels, whole_window, bands: 0..0, }); @@ -775,6 +810,8 @@ fn plan_rings( rows: first..last, order: (i64::from(first) - SUPPORT_ROWS) * scales[index], work: strip.work, + parts: 1, + split: 1, }); } bands[start..].sort_unstable_by_key(|band| band.rows.start); @@ -793,6 +830,84 @@ fn plan_rings( Ok((rings, bands, active_levels)) } +/// Most parts a band splits into. +const MAX_BAND_PARTS: usize = 8; + +/// Parts of finest-level bands that a split aims at per thread: with two, a +/// thread that finishes a part finds another while the band's last parts +/// finish. One part per thread left 5792 x 5792 gray 14% slower (80 against +/// 70 ms on 18 threads). +const BAND_PARTS_PER_THREAD: usize = 2; + +/// Split a classic tile's bands into parts for `threads` threads (see +/// [`plan_parts`]), and count the bands that split. +fn split_bands( + tile_ctx: &mut TileDecodeContext, + rings: &[Ring], + bands: &mut [Band], + threads: usize, + split: impl Fn(&Band, usize) -> Result, +) -> Result<()> { + #[cfg(test)] + let forced = tile_ctx.band_parts; + #[cfg(not(test))] + let forced = None; + plan_parts(rings, bands, threads, forced, split)?; + tile_ctx.debug_counters.pool_band_parts += bands.iter().filter(|band| band.parts > 1).count(); + Ok(()) +} + +/// Split `bands` into parts, through `split` (which returns the runs a band +/// splits into for a wanted count), when the finest level's slots ahead of +/// the bands being read hold fewer bands than `threads`: each finest band +/// into as many parts as give [`BAND_PARTS_PER_THREAD`] parts per thread, and +/// every other band into parts of about the same work. A tile whose rings +/// hold bands enough keeps them whole. `forced` splits every band into that +/// many parts instead. +fn plan_parts( + rings: &[Ring], + bands: &mut [Band], + threads: usize, + forced: Option, + split: impl Fn(&Band, usize) -> Result, +) -> Result<()> { + if let Some(parts) = forced { + for band in bands { + band.split = parts.max(1); + band.parts = split(band, band.split)?; + } + return Ok(()); + } + let ahead: usize = rings + .iter() + .filter(|ring| ring.finest && !ring.whole_window) + .map(|ring| ring.slots.saturating_sub(1)) + .sum(); + if ahead == 0 || ahead >= threads { + return Ok(()); + } + let parts = threads + .saturating_mul(BAND_PARTS_PER_THREAD) + .div_ceil(ahead) + .min(MAX_BAND_PARTS); + let (finest_work, finest_bands) = rings + .iter() + .filter(|ring| ring.finest) + .flat_map(|ring| &bands[ring.bands.clone()]) + .fold((0usize, 0usize), |(work, count), band| { + (work.saturating_add(band.work), count + 1) + }); + let part_work = (finest_work / finest_bands.max(1) / parts).max(1); + for band in bands { + let wanted = band.work.div_ceil(part_work).clamp(1, parts); + if wanted > 1 { + band.split = wanted; + band.parts = split(band, wanted)?; + } + } + Ok(()) +} + /// Each ring's free slots and the slot each band of a whole-window ring /// owns. type Slots<'c> = (Vec>, Vec>); @@ -855,9 +970,15 @@ fn split_slots<'c>( offset = start + len; } } else { + // Each slot holds whole rows of the sub-band, as many as the + // ring's share of them: at least its longest band's, which + // can be fewer than the rows the ring was laid out for. + let width = ring.stored.width() as usize; let slot_len = range .len() .checked_div(ring.slots) + .and_then(|len| len.checked_div(width)) + .and_then(|rows| rows.checked_mul(width)) .ok_or(DecodingError::CodeBlockDecodeFailure)?; for _ in 0..ring.slots { let (slot, tail) = core::mem::take(&mut ring_samples).split_at_mut(slot_len); @@ -892,11 +1013,16 @@ struct State<'c, 'w, W> { /// The rows of the window each band of a whole-window ring decodes into, /// until it starts. owned: Vec>, - /// Bands that can start, soonest read first, then largest first. - /// Entries for bands since started are skipped. + /// Bands that can start, or hand out another part, soonest read first, + /// then largest first. Entries for bands with nothing left to hand out + /// are skipped. ready: BinaryHeap, usize)>>, /// Each ring's next band to make ready. next_ready: Vec, + /// Parts of each band handed out, and parts still decoding or to hand + /// out. + next_part: Vec, + parts_left: Vec, /// How many of each ring's bands, from its first, have been freed. freed_prefix: Vec, /// Bands that finished decoding, for the drain to free; bands the IDWT @@ -915,6 +1041,8 @@ struct State<'c, 'w, W> { struct Shared<'p, 'c, 'w, W> { rings: &'p [Ring], bands: &'p [Band], + /// The slot of each band decoding in parts, which its parts share. + outputs: Vec>, max_workers: usize, /// Test-only: the first worker to start a band abandons it. #[cfg(test)] @@ -947,17 +1075,22 @@ impl<'c, 'w, W> Shared<'_, 'c, 'w, W> { } } - /// Take the next ready band, marked running, with a slot to decode into. - fn start_next(&self, state: &mut State<'c, 'w, W>) -> Result> { + /// Take the next ready band, marked running, with a slot to decode + /// into, or the next part of a running band. + fn start_next(&self, state: &mut State<'c, 'w, W>) -> Result>> { while let Some(Reverse((_, _, index))) = state.ready.pop() { - if state.status[index] == Status::Waiting { - return self.start(state, index).map(Some); + match state.status[index] { + Status::Waiting => return self.start(state, index).map(Some), + Status::Running if self.has_part(state, index) => { + return Ok(Some(self.next_part(state, index))); + } + _ => {} } } Ok(None) } - fn start(&self, state: &mut State<'c, 'w, W>, index: usize) -> Result<(usize, &'c mut [f32])> { + fn start(&self, state: &mut State<'c, 'w, W>, index: usize) -> Result> { let slot = match state.owned[index].take() { Some(slot) => slot, None => state.free_slots[self.bands[index].ring] @@ -966,7 +1099,62 @@ impl<'c, 'w, W> Shared<'_, 'c, 'w, W> { }; state.status[index] = Status::Running; state.running += 1; - Ok((index, slot)) + let parts = self.bands[index].parts; + if parts <= 1 { + return Ok(Task::Whole(index, slot)); + } + *self.output(index)? = slot; + state.parts_left[index] = parts; + Ok(self.next_part(state, index)) + } + + /// Whether running band `index` decodes in parts and has parts left to + /// hand out. + fn has_part(&self, state: &State<'c, 'w, W>, index: usize) -> bool { + let parts = self.bands[index].parts; + parts > 1 && state.next_part[index] < parts + } + + /// Hand out the next part of running band `index`, leaving the band + /// ready while it has more. + fn next_part(&self, state: &mut State<'c, 'w, W>, index: usize) -> Task<'c> { + let part = state.next_part[index]; + state.next_part[index] += 1; + if self.has_part(state, index) { + let band = &self.bands[index]; + state + .ready + .push(Reverse((band.order, Reverse(band.work), index))); + } + Task::Part(index, part) + } + + fn output(&self, index: usize) -> Result> { + self.outputs[index] + .lock() + .map_err(|_| DecodingError::CodeBlockDecodeFailure.into()) + } + + /// Record a part of band `index`, decoded outside the lock; the last + /// part to finish leaves the band decoded. + fn finish_part(&self, index: usize, result: Result<()>) -> Result<()> { + let mut state = self.lock()?; + match result { + Ok(()) => { + state.parts_left[index] -= 1; + if state.parts_left[index] == 0 { + let slot = core::mem::take(&mut *self.output(index)?); + state.held[index] = Some(slot); + Self::decoded(&mut state, index); + } + } + Err(error) => { + state.error.get_or_insert(error); + } + } + drop(state); + self.decoded.notify_all(); + Ok(()) } /// Band `index`, running, has decoded. @@ -1013,6 +1201,13 @@ impl<'c, 'w, W> Shared<'_, 'c, 'w, W> { } } +/// A thread's next decode: a band of one part, straight into its slot, or +/// part `.1` of band `.0`, into the slot its parts share. +enum Task<'c> { + Whole(usize, &'c mut [f32]), + Part(usize, usize), +} + /// Decode `bands` into `slots` on up to `workers` pool threads while `run` /// drives the IDWT through `readers` views of them, then decode the bands it /// didn't read. Each view reads its own sub-bands and decodes a band nobody @@ -1030,7 +1225,7 @@ fn run_bands( bands: &[Band], slots: Slots<'_>, bank: &mut [TaskWorkspace], - decode: impl Fn(usize, &mut [f32], &mut W, &mut BlockScratch) -> Result<()> + Sync, + decode: impl Fn(usize, BandOutput<'_, '_>, &mut W, &mut BlockScratch) -> Result<()> + Sync, run: impl FnOnce(&mut [&mut (dyn SubBandRows + Send)]) -> Result<()>, profile_enabled: bool, tier1_us: &mut u128, @@ -1059,6 +1254,8 @@ fn run_bands( owned: slots.1, ready: BinaryHeap::new(), next_ready: try_collect(rings.iter().map(|ring| ring.bands.start), rings.len())?, + next_part: try_filled(bands.len(), 0)?, + parts_left: try_filled(bands.len(), 0)?, freed_prefix: try_filled(rings.len(), 0)?, decoded: Vec::new(), running: 0, @@ -1079,6 +1276,7 @@ fn run_bands( let shared = Shared { rings, bands, + outputs: try_filled_with(bands.len(), || Mutex::new(<&mut [f32]>::default()))?, max_workers: workers, #[cfg(test)] strand_once: core::sync::atomic::AtomicBool::new(strand_once), @@ -1178,10 +1376,38 @@ fn try_filled_with(len: usize, value: impl FnMut() -> T) -> Result> { Ok(values) } +/// Run `task` with `workspace` and `scratch`, and record it. +fn run_task<'c, W>( + shared: &Shared<'_, 'c, '_, W>, + decode: &(impl Fn(usize, BandOutput<'_, '_>, &mut W, &mut BlockScratch) -> Result<()> + Sync), + task: Task<'c>, + workspace: &mut W, + scratch: &mut BlockScratch, +) -> Result<()> { + match task { + Task::Whole(index, slot) => { + slot.fill(0.0); + let strip = shared.bands[index].strip; + let result = decode(strip, BandOutput::Whole(&mut *slot), workspace, scratch); + shared.finish(index, slot, result) + } + Task::Part(index, part) => { + let band = &shared.bands[index]; + let output = BandOutput::Part { + part, + parts: band.split, + slot: &shared.outputs[index], + }; + let result = decode(band.strip, output, workspace, scratch); + shared.finish_part(index, result) + } + } +} + /// A pool worker: decode ready bands until none is left to start. fn work( shared: &Shared<'_, '_, '_, W>, - decode: &(impl Fn(usize, &mut [f32], &mut W, &mut BlockScratch) -> Result<()> + Sync), + decode: &(impl Fn(usize, BandOutput<'_, '_>, &mut W, &mut BlockScratch) -> Result<()> + Sync), ) { let Ok(mut state) = shared.lock() else { return; @@ -1197,8 +1423,8 @@ fn work( } else { shared.start_next(&mut state) }; - let (index, slot) = match next { - Ok(Some(started)) => started, + let task = match next { + Ok(Some(task)) => task, Ok(None) => break, Err(error) => { state.error.get_or_insert(error); @@ -1217,9 +1443,7 @@ fn work( } return; } - slot.fill(0.0); - let result = decode(shared.bands[index].strip, slot, workspace, &mut scratch); - if shared.finish(index, slot, result).is_err() { + if run_task(shared, decode, task, workspace, &mut scratch).is_err() { return; } state = match shared.lock() { @@ -1255,7 +1479,7 @@ struct PoolRows<'s, 'p, 'c, 'w, W, D> { impl<'p, 'c, W, D> PoolRows<'_, 'p, 'c, '_, W, D> where W: TaskScratch + 'p, - D: Fn(usize, &mut [f32], &mut W, &mut BlockScratch) -> Result<()> + Sync, + D: Fn(usize, BandOutput<'_, '_>, &mut W, &mut BlockScratch) -> Result<()> + Sync, 'c: 'p, { /// Send pool workers, up to the plan's count, while bands are ready. @@ -1303,68 +1527,66 @@ where if let Some(error) = state.error { return Err(error); } - match state.status[index] { + let task = match state.status[index] { Status::Done => { return state.held[index] .take() .ok_or(DecodingError::CodeBlockDecodeFailure.into()); } - Status::Running => { - // A reader that helps decodes the next band nobody has - // started while this one runs. - if self.helps { - if let Some((other, slot)) = self.shared.start_next(&mut state)? { - // A reader decoding a band counts as an active - // worker for anyone waiting on it. - state.active += 1; - drop(state); - self.send_workers()?; - let decoded = self.decode_for_pool(other, slot); - state = self.shared.lock()?; - state.active -= 1; - decoded?; - continue; - } - } - state = self - .shared - .decoded - .wait_timeout(state, STRANDED_CHECK) - .map_err(|_| DecodingError::CodeBlockDecodeFailure)? - .0; - // A failed band stays running; its error ends the wait. - if state.error.is_none() - && state.status[index] == Status::Running - && state.active == 0 - { - state.stranded = true; - return Err(DecodingError::CodeBlockDecodeFailure.into()); - } - } - Status::Waiting => { - let (index, slot) = self.shared.start(&mut state, index)?; - drop(state); - self.decode_here(index, slot)?; - return Ok(slot); - } Status::Freed => return Err(DecodingError::CodeBlockDecodeFailure.into()), + // A band nobody has started, or its first part, decodes + // here; its other parts go to the pool. + Status::Waiting => Some(self.shared.start(&mut state, index)?), + // A reader decodes a part of this band nobody has started, + // and one that helps the next band or part nobody has + // started, while this one runs. + Status::Running if self.shared.has_part(&state, index) => { + Some(self.shared.next_part(&mut state, index)) + } + Status::Running if self.helps => self.shared.start_next(&mut state)?, + Status::Running => None, + }; + if let Some(task) = task { + // A reader decoding a band counts as an active worker for + // anyone waiting on it. + state.active += 1; + drop(state); + self.send_workers()?; + let decoded = self.decode_for_pool(task); + state = self.shared.lock()?; + state.active -= 1; + decoded?; + continue; + } + state = self + .shared + .decoded + .wait_timeout(state, STRANDED_CHECK) + .map_err(|_| DecodingError::CodeBlockDecodeFailure)? + .0; + // A failed band stays running; its error ends the wait. + if state.error.is_none() && state.status[index] == Status::Running && state.active == 0 + { + state.stranded = true; + drop(state); + return Err(DecodingError::CodeBlockDecodeFailure.into()); } } } - /// Decode band `index`, which another reader or the drain will take, - /// and hand it over as a worker does. - fn decode_for_pool(&mut self, index: usize, slot: &'c mut [f32]) -> Result<()> { + /// Decode `task`, a band or part this reader, another one or the drain + /// will take, and hand it over as a worker does. + fn decode_for_pool(&mut self, task: Task<'c>) -> Result<()> { let start = profile::profile_now(self.profile_enabled); - slot.fill(0.0); - let result = (self.decode)( - self.shared.bands[index].strip, - slot, + let result = run_task( + self.shared, + self.decode, + task, self.workspace, &mut self.scratch, ); self.tier1_us += profile::elapsed_us(start); - self.shared.finish(index, slot, result) + result } fn decode_here(&mut self, index: usize, slot: &mut [f32]) -> Result<()> { @@ -1372,7 +1594,7 @@ where slot.fill(0.0); let result = (self.decode)( self.shared.bands[index].strip, - slot, + BandOutput::Whole(slot), self.workspace, &mut self.scratch, ); @@ -1418,12 +1640,17 @@ where let next = self.shared.start_next(&mut state)?; let running = state.running > 0; match next { - Some((index, slot)) => { + Some(Task::Whole(index, slot)) => { drop(state); self.send_workers()?; self.decode_here(index, slot)?; self.shared.lock()?.held[index] = Some(slot); } + Some(part @ Task::Part(..)) => { + drop(state); + self.send_workers()?; + self.decode_for_pool(part)?; + } None if running => { let mut state = self .shared @@ -1451,7 +1678,7 @@ where impl<'p, 'c, W, D> SubBandRows for PoolRows<'_, 'p, 'c, '_, W, D> where W: TaskScratch + 'p, - D: Fn(usize, &mut [f32], &mut W, &mut BlockScratch) -> Result<()> + Sync, + D: Fn(usize, BandOutput<'_, '_>, &mut W, &mut BlockScratch) -> Result<()> + Sync, 'c: 'p, { fn prepare_row(&mut self, sub_band: usize, y: u32) -> Result<()> { diff --git a/crates/j2k-native/src/j2c/decode/state.rs b/crates/j2k-native/src/j2c/decode/state.rs index b311d78a..84822eaf 100644 --- a/crates/j2k-native/src/j2c/decode/state.rs +++ b/crates/j2k-native/src/j2c/decode/state.rs @@ -263,6 +263,10 @@ pub(crate) struct TileDecodeContext { /// band is left running with no worker to finish it. #[cfg(all(test, feature = "parallel"))] pub(crate) strand_pool_band: bool, + /// Test-only: each pool band splits into this many parts, as far as its + /// blocks allow, whatever the pool's size. + #[cfg(all(test, feature = "parallel"))] + pub(crate) band_parts: Option, /// Test-only: the average work per band below which a component's /// bands decode over whole windows instead of on the pool; zero, so /// unit tests' tiny fixtures take the pool, unless a test sets it. diff --git a/crates/j2k-native/src/j2c/decode/subband.rs b/crates/j2k-native/src/j2c/decode/subband.rs index 5e80a706..3014cee3 100644 --- a/crates/j2k-native/src/j2c/decode/subband.rs +++ b/crates/j2k-native/src/j2c/decode/subband.rs @@ -44,8 +44,8 @@ pub(crate) use self::placement::BlockPlacement; pub(crate) use self::tile_parallel::MIN_BAND_WORK; #[cfg(feature = "parallel")] pub(in crate::j2c::decode) use self::tile_parallel::{ - plan_band_route, plan_band_route_beside, routes_tile, BandRoute, BandRouting, BlockScratch, - PayloadSource, + plan_band_route, plan_band_route_beside, routes_tile, BandOutput, BandRoute, BandRouting, + BlockScratch, PayloadSource, }; /// Decode the code-blocks of the tile's `components` into their sub-bands. diff --git a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs index ce24006d..6566d370 100644 --- a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs +++ b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs @@ -1533,6 +1533,19 @@ fn job_payload<'p>( } } +/// Where a band decodes: into its slot, or as run `part` of its blocks split +/// for `parts` parts (see [`BandRoute::strip_parts`]), into the slot its +/// parts share. A part zeroes its own columns of the slot, from its first +/// block to the next part's, and copies each block in under the lock. +pub(in crate::j2c::decode) enum BandOutput<'o, 'c> { + Whole(&'o mut [f32]), + Part { + part: usize, + parts: usize, + slot: &'o Mutex<&'c mut [f32]>, + }, +} + /// One code-block row of a sub-band, as [`plan_band_route`] plans it: the /// band's rows, counted from the top of the sub-band's stored rectangle. #[derive(Clone, Copy)] @@ -1551,9 +1564,10 @@ pub(in crate::j2c::decode) struct BandRoute<'a> { plan: TilePlan<'a>, workers: usize, growths: usize, - /// The decode's budget with the plan, the payload arena and every - /// thread's clipped-block scratch charged, which stay live while the - /// bands decode. + /// The decode's budget with the plan, every thread's payload and block + /// scratch and the bank's growth charged, which stay live while the + /// bands decode, beside whatever the budget the route was planned in + /// held already. budget: DecodeAllocationBudget, } @@ -1666,11 +1680,12 @@ pub(in crate::j2c::decode) fn plan_band_route_beside<'a>( let workers = threads.min(plan.strips.len()); let prepared = (|| -> Result { // Each thread gathers payloads into scratch of its own as it - // decodes, and allocates its clipped-block scratch on first use. + // decodes, and allocates its block scratch on first use: a clipped + // block, or any block of a band it decodes a part of. include_payload_scratch(&mut budget, &plan, workers + 1)?; budget.include_elements::( (workers + 1) - .checked_mul(plan.max_clipped_samples) + .checked_mul(band_scratch_samples(&plan)?) .ok_or(ValidationError::ImageTooLarge)?, )?; let (width, height) = (plan.max_width, plan.max_height); @@ -1708,6 +1723,15 @@ pub(in crate::j2c::decode) fn plan_band_route_beside<'a>( } } +/// The block scratch each thread of a band route holds: as large as any +/// block, since a part decodes every block of its run through it. +fn band_scratch_samples(plan: &TilePlan<'_>) -> Result { + let block = (plan.max_width as usize) + .checked_mul(plan.max_height as usize) + .ok_or(ValidationError::ImageTooLarge)?; + Ok(block.max(plan.max_clipped_samples)) +} + impl BandRoute<'_> { pub(in crate::j2c::decode) fn coding(&self) -> CodeBlockCoding { self.plan.coding @@ -1736,19 +1760,35 @@ impl BandRoute<'_> { }) } - /// Decode strip `index` of a classic plan into `output`, which holds the - /// band's rows from its first. + /// The runs of blocks strip `index` splits into when it aims at + /// `parts` parts of about equal work: `parts` at most, fewer when a few + /// blocks hold most of its work. + pub(in crate::j2c::decode) fn strip_parts(&self, index: usize, parts: usize) -> Result { + let strip = self + .plan + .strips + .get(index) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let mut runs = 0usize; + split_strip(&self.plan, strip, parts, |_, _| { + runs += 1; + Ok(()) + })?; + Ok(runs.max(1)) + } + + /// Decode strip `index` of a classic plan into `output`, whose slot + /// holds the band's rows from its first. pub(in crate::j2c::decode) fn decode_classic( &self, index: usize, - output: &mut [f32], + output: BandOutput<'_, '_>, workspace: &mut J2kCodeBlockDecodeWorkspace, scratch: (&mut BlockScratch, PayloadSource<'_, '_>), ) -> Result<()> { let plan = &self.plan; - decode_strip( - plan, - &plan.strips[index], + self.decode_band( + index, output, workspace, scratch, @@ -1758,19 +1798,18 @@ impl BandRoute<'_> { ) } - /// Decode strip `index` of an HT plan into `output`, which holds the - /// band's rows from its first. + /// Decode strip `index` of an HT plan into `output`, whose slot holds + /// the band's rows from its first. pub(in crate::j2c::decode) fn decode_ht( &self, index: usize, - output: &mut [f32], + output: BandOutput<'_, '_>, workspace: &mut HtCodeBlockDecodeWorkspace, scratch: (&mut BlockScratch, PayloadSource<'_, '_>), ) -> Result<()> { let plan = &self.plan; - decode_strip( - plan, - &plan.strips[index], + self.decode_band( + index, output, workspace, scratch, @@ -1780,6 +1819,49 @@ impl BandRoute<'_> { ) } + fn decode_band( + &self, + index: usize, + output: BandOutput<'_, '_>, + workspace: &mut W, + scratch: (&mut BlockScratch, PayloadSource<'_, '_>), + decode_block: &impl Fn(&BlockJob<'_>, &[u8], &mut [f32], usize, &mut W) -> Result<()>, + ) -> Result<()> { + let plan = &self.plan; + let strip = plan + .strips + .get(index) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + match output { + BandOutput::Whole(slot) => { + decode_strip(plan, strip, slot, workspace, scratch, decode_block) + } + BandOutput::Part { part, parts, slot } => { + let mut jobs = None; + let mut run = 0usize; + split_strip(plan, strip, parts, |range, _| { + if run == part { + jobs = Some(range); + } + run += 1; + Ok(()) + })?; + let jobs = jobs.ok_or(DecodingError::CodeBlockDecodeFailure)?; + decode_part( + plan, + strip, + jobs, + slot, + true, + band_scratch_samples(plan)?, + workspace, + scratch, + decode_block, + ) + } + } + } + /// Count the plan's blocks and record the decode's peak with the plan /// and every thread's clipped-block scratch live. pub(in crate::j2c::decode) fn record( @@ -1791,7 +1873,7 @@ impl BandRoute<'_> { let counts = TaskCounts { tasks: 0, shared: 0, - scratch_samples: self.plan.max_clipped_samples, + scratch_samples: band_scratch_samples(&self.plan)?, part_work: None, }; record_route_peak(tile_ctx, storage, &self.plan, self.workers + 1, counts)?; diff --git a/crates/j2k-native/src/tests/parallel_bands.rs b/crates/j2k-native/src/tests/parallel_bands.rs index b74ce4e5..5f32e32d 100644 --- a/crates/j2k-native/src/tests/parallel_bands.rs +++ b/crates/j2k-native/src/tests/parallel_bands.rs @@ -17,13 +17,25 @@ struct RegionDecode { decoded_blocks: usize, skipped_blocks: usize, pool_components: usize, + parted_bands: usize, peak_coefficients: usize, } /// Decode `roi` with Auto parallelism on the current pool, in bands on the /// pool unless `whole_windows`. fn decode(image: &Image<'_>, roi: (u32, u32, u32, u32), whole_windows: bool) -> RegionDecode { - decode_with_cutoff(image, roi, whole_windows, 0) + decode_with(image, roi, whole_windows, None, 0) +} + +/// [`decode`], with every pool band split into `parts` parts as far as its +/// blocks allow. +fn decode_in_parts( + image: &Image<'_>, + roi: (u32, u32, u32, u32), + whole_windows: bool, + parts: Option, +) -> RegionDecode { + decode_with(image, roi, whole_windows, parts, 0) } /// [`decode`] with bands averaging less than `band_work_cutoff` decoding over @@ -33,11 +45,23 @@ fn decode_with_cutoff( roi: (u32, u32, u32, u32), whole_windows: bool, band_work_cutoff: usize, +) -> RegionDecode { + decode_with(image, roi, whole_windows, None, band_work_cutoff) +} + +/// [`decode_in_parts`] and [`decode_with_cutoff`] at once. +fn decode_with( + image: &Image<'_>, + roi: (u32, u32, u32, u32), + whole_windows: bool, + parts: Option, + band_work_cutoff: usize, ) -> RegionDecode { let mut context = DecoderContext::default(); context.tile_decode_context.band_work_cutoff = band_work_cutoff; context.set_cpu_decode_parallelism(CpuDecodeParallelism::Auto); context.tile_decode_context.whole_window_tier1 = whole_windows; + context.tile_decode_context.band_parts = parts; let native = image .decode_native_region_with_context(roi, &mut context) .expect("decode region") @@ -48,6 +72,7 @@ fn decode_with_cutoff( decoded_blocks: counters.decoded_code_blocks, skipped_blocks: counters.skipped_code_blocks, pool_components: counters.pool_band_components, + parted_bands: counters.pool_band_parts, peak_coefficients: counters.workspace_peak.bytes().coefficients, } } @@ -142,6 +167,62 @@ fn pool_bands_match_whole_window_tier1_bit_for_bit() { } } +/// Threads may decode a classic band in parts, runs of its blocks, at once: +/// each part zeroes its own columns of the band's slot and copies its blocks +/// in under the slot's lock. Every split must match whole-window Tier-1 bit +/// for bit and decode and skip the same code-blocks. +#[test] +fn pool_band_parts_match_whole_window_tier1_bit_for_bit() { + let mut parted = 0; + for threads in [2, 5] { + let pool = pool(threads); + for case in cases() + .into_iter() + .filter(|case| !case.options.use_ht_block_coding && case.options.num_layers <= 1) + { + let pixels = noisy_pixels(case.components); + let bytes = encode( + &pixels, + WIDTH, + HEIGHT, + u16::from(case.components), + 8, + false, + &case.options, + ) + .expect("encode band part fixture"); + for reduction in [0_u8, 1] { + let image = + Image::new_with_reduction(&bytes, &DecodeSettings::default(), reduction) + .expect("parse band part fixture"); + let (width, height) = (image.width(), image.height()); + let tile = TILE >> reduction; + for roi in [ + (0, 0, width, height), + (width / 3, height / 3, width / 7, height / 9), + (tile - 5, tile - 3, 11, 7), + (width - 1, height - 1, 1, 1), + ] { + let whole = pool.install(|| decode(&image, roi, true)); + for parts in [2, 3, 7] { + let label = format!( + "{} threads={threads} reduction={reduction} roi={roi:?} parts={parts}", + case.label + ); + let banded = + pool.install(|| decode_in_parts(&image, roi, false, Some(parts))); + assert_eq!(banded.native, whole.native, "{label}"); + assert_eq!(banded.decoded_blocks, whole.decoded_blocks, "{label}"); + assert_eq!(banded.skipped_blocks, whole.skipped_blocks, "{label}"); + parted += banded.parted_bands; + } + } + } + } + } + assert!(parted > 0, "no band decoded in parts"); +} + fn rgb_fixture() -> Vec { let pixels = noisy_pixels(3); encode( @@ -452,3 +533,58 @@ fn cheap_bands_decode_over_whole_windows() { assert_eq!(cheap.native, pixels); assert_eq!(cheap.native, unlimited.native); } + +/// A region can need a sub-band's rows only from a code-block row it clips, +/// when the rows of code-blocks below it code nothing: the bands then hold +/// fewer rows than the ring was laid out for, and the ring cuts more slots +/// from its rows than it was laid out with. Each slot must still hold whole +/// rows, so that the bands, split into parts, decode as on one thread. +#[test] +fn pool_band_parts_decode_where_only_clipped_code_block_rows_code() { + let (width, height) = (256, 256); + // Noise in the top rows, flat below: of the finest sub-bands' rows of + // 32 x 32 code-blocks, only the first codes anything. + let mut state = 0x2545_f491_u32; + let pixels: Vec = (0..width * height) + .map(|at| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + if at / width < 48 { + state.to_le_bytes()[3] + } else { + 128 + } + }) + .collect(); + let bytes = encode( + &pixels, + width, + height, + 1, + 8, + false, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + code_block_width_exp: 3, + code_block_height_exp: 3, + ..EncodeOptions::default() + }, + ) + .expect("encode clipped-rows fixture"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("parse clipped-rows fixture"); + // The regions clip the first row of code-blocks of every sub-band, to + // 20 down to 12 of the finest sub-bands' 32 rows: 96 rows laid out for + // three slots then hold four to eight bands of the clipped rows, five + // and seven of which leave rows over. + for top in (40..=56).step_by(2) { + let roi = (0, top, width, height - top); + let serial = pool(1).install(|| decode(&image, roi, false)); + for parts in [2, 4] { + let pooled = pool(4).install(|| decode_in_parts(&image, roi, false, Some(parts))); + assert!(pooled.parted_bands > 0, "{roi:?} in {parts} parts"); + assert_eq!(pooled.native, serial.native, "{roi:?} in {parts} parts"); + } + } +} diff --git a/crates/j2k-native/src/tests/row_sink.rs b/crates/j2k-native/src/tests/row_sink.rs index 6ecba420..f9d03f64 100644 --- a/crates/j2k-native/src/tests/row_sink.rs +++ b/crates/j2k-native/src/tests/row_sink.rs @@ -89,6 +89,11 @@ struct Decoded { expect(dead_code, reason = "only the pool band tests compare the counters") )] skipped_blocks: usize, + #[cfg_attr( + not(feature = "parallel"), + expect(dead_code, reason = "only the pool band tests compare the counters") + )] + parted_bands: usize, } fn decode_planes(image: &Image<'_>, roi: (u32, u32, u32, u32)) -> Decoded { @@ -131,6 +136,7 @@ fn decode_planes(image: &Image<'_>, roi: (u32, u32, u32, u32)) -> Decoded { pool_components: counters.pool_band_components, decoded_blocks: counters.decoded_code_blocks, skipped_blocks: counters.skipped_code_blocks, + parted_bands: counters.pool_band_parts, } } @@ -138,9 +144,19 @@ fn decode_rows( image: &Image<'_>, roi: (u32, u32, u32, u32), parallelism: CpuDecodeParallelism, +) -> Decoded { + decode_rows_with(image, roi, parallelism, |_| {}) +} + +fn decode_rows_with( + image: &Image<'_>, + roi: (u32, u32, u32, u32), + parallelism: CpuDecodeParallelism, + configure: impl FnOnce(&mut DecoderContext), ) -> Decoded { let mut context = DecoderContext::default(); context.set_cpu_decode_parallelism(parallelism); + configure(&mut context); let mut sink = CollectingSink::new(roi.2, roi.3); image .decode_region_rows_for_integer_output_with_context(roi, &mut context, &mut sink) @@ -167,6 +183,7 @@ fn decode_rows( pool_components: counters.pool_band_components, decoded_blocks: counters.decoded_code_blocks, skipped_blocks: counters.skipped_code_blocks, + parted_bands: counters.pool_band_parts, } } @@ -348,6 +365,67 @@ fn pool_band_rows_match_component_planes_bit_for_bit() { } } +/// A streamed decode whose classic bands split into parts, which threads +/// decode at once into each band's slot, must give the serial planes' rows +/// bit for bit, decode and skip the same code-blocks, and hold no planes. +#[cfg(feature = "parallel")] +#[test] +fn pool_band_part_rows_match_component_planes_bit_for_bit() { + let mut parted = 0; + for threads in [2, 13] { + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("build pool"); + for case in row_sink_cases() + .into_iter() + .filter(|case| !case.options.use_ht_block_coding && case.options.num_layers <= 1) + { + let pixels = noisy_pixels(case.components); + let bytes = encode( + &pixels, + WIDTH, + HEIGHT, + u16::from(case.components), + 8, + false, + &case.options, + ) + .expect("encode band part row fixture"); + let image = Image::new_with_reduction(&bytes, &DecodeSettings::default(), 0) + .expect("parse band part row fixture"); + let (width, height) = (image.width(), image.height()); + for roi in [ + (0, 0, width, height), + (width / 3, height / 3, width / 7, height / 9), + (0, height - 9, width, 9), + ] { + let planes = decode_planes(&image, roi); + for parts in [None, Some(2), Some(5)] { + let label = format!( + "{} threads={threads} roi={roi:?} parts={parts:?}", + case.label + ); + let rows = pool.install(|| { + decode_rows_with(&image, roi, CpuDecodeParallelism::Auto, |context| { + context.tile_decode_context.band_parts = parts; + }) + }); + assert_eq!(rows.planes, planes.planes, "{label}"); + if rows.pool_components > 0 { + assert!(rows.streamed, "{label}: streamed"); + assert!(rows.channel_bytes <= 256, "{label}: no planes"); + assert_eq!(rows.decoded_blocks, planes.decoded_blocks, "{label}"); + assert_eq!(rows.skipped_blocks, planes.skipped_blocks, "{label}"); + } + parted += rows.parted_bands; + } + } + } + } + assert!(parted > 0, "no band decoded in parts"); +} + /// Each component's route plans its own clipped code-blocks, while a pool /// worker or reader decodes the strips of every component with one scratch. /// Components with detail only along the right edge code only the narrow From 4802de72cdb9e4093953008e0071e659cf2a9f19 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 04:59:30 -0400 Subject: [PATCH 17/94] perf(j2k-native): keep layer records in 8 bytes and route strips to the rows planned A tile keeps a layer record for every layer of every code-block it reads, 24 bytes each (`Option>`): 6.5 MiB for a 5661x8287 RGB image with 8 layers, 25.6 MiB for an 18000x12000 one. A record is now a u32 start and length into the tile's segment records, with an unread layer marked by a length of u32::MAX; a range that does not fit is a packet parse error. The tile and band routes also reserved one strip per block and kept it; the strips now shrink to the code-block rows planned (1.5 and 5.3 MiB). Full-ROI streamed decode heap at the default pool: 5661x8287 85.6 -> 79.8 MiB, 18000x12000 229.2 -> 207.1 MiB; one thread 25.8 -> 21.4 MiB. Time unchanged. The layer-metadata amplification test now rewrites its header to 8192^2: at 8 bytes per record the 4096^2 header fits the decode budget and fails in packet parsing instead of being refused up front. --- .../j2k-native/src/j2c/bitplane/schedule.rs | 2 +- crates/j2k-native/src/j2c/build.rs | 51 +++++++++++++++---- .../src/j2c/build/allocation/tests.rs | 24 +++++++++ crates/j2k-native/src/j2c/decode.rs | 12 ++--- .../src/j2c/decode/direct_plan/classic.rs | 6 +-- .../src/j2c/decode/subband/tile_parallel.rs | 16 ++++-- crates/j2k-native/src/j2c/decode/tier1.rs | 2 +- .../src/j2c/encode/single_tile/tests.rs | 2 +- .../src/j2c/ht_block_decode/segments.rs | 2 +- .../src/j2c/ht_block_decode/segments/owned.rs | 4 +- .../src/j2c/ht_block_decode/segments/tests.rs | 22 +++----- crates/j2k-native/src/j2c/segment.rs | 2 +- crates/j2k-native/src/j2c/segment/classic.rs | 5 +- crates/j2k-native/src/j2c/segment/ht.rs | 7 +-- crates/j2k-native/src/tests.rs | 4 +- 15 files changed, 109 insertions(+), 52 deletions(-) diff --git a/crates/j2k-native/src/j2c/bitplane/schedule.rs b/crates/j2k-native/src/j2c/bitplane/schedule.rs index c9799b8f..fdcd184b 100644 --- a/crates/j2k-native/src/j2c/bitplane/schedule.rs +++ b/crates/j2k-native/src/j2c/bitplane/schedule.rs @@ -46,7 +46,7 @@ fn decode_inner_with_observer( // Build a list so that we can associate coding passes with their segments // and data more easily. for layer in storage.layers.get(code_block.layers.clone())? { - if let Some(range) = layer.segments.clone() { + if let Some(range) = layer.segments() { let layer_segments = storage.segments.get(range)?; for segment in layer_segments { if segment.idx != last_segment_idx { diff --git a/crates/j2k-native/src/j2c/build.rs b/crates/j2k-native/src/j2c/build.rs index 700ad4d2..733a9c9e 100644 --- a/crates/j2k-native/src/j2c/build.rs +++ b/crates/j2k-native/src/j2c/build.rs @@ -444,14 +444,10 @@ fn build_code_blocks( if end > storage.layers.capacity() { return Err(DecodingError::HostAllocationFailed.into()); } - storage.layers.extend(iter::repeat_n( - Layer { - // This will be updated once we actually read the - // layer segments. - segments: None, - }, - layer_count, - )); + // Each layer is updated once its packet's segments are read. + storage + .layers + .extend(iter::repeat_n(Layer::UNREAD, layer_count)); if storage.layers.len() != end { return Err(DecodingError::InvalidPrecinct.into()); } @@ -744,7 +740,42 @@ pub(crate) struct Segment<'a> { pub(crate) data: &'a [u8], } -#[derive(Clone)] +/// The segments one layer of a code-block contributed: a range of the +/// tile's segment records, once the layer's packet has been read. A tile +/// holds one per layer of every code-block, so it is kept to 8 bytes. +#[derive(Clone, Copy)] pub(crate) struct Layer { - pub(crate) segments: Option>, + start: u32, + /// The range's length, or [`Layer::UNREAD_LEN`] before the packet is + /// read. + len: u32, +} + +impl Layer { + const UNREAD_LEN: u32 = u32::MAX; + + /// A layer whose packet has not been read. + pub(crate) const UNREAD: Self = Self { + start: 0, + len: Self::UNREAD_LEN, + }; + + /// A layer that contributed the segments `range`, or `None` when the + /// range does not fit the record. + pub(crate) fn with_segments(range: Range) -> Option { + let start = u32::try_from(range.start).ok()?; + let len = u32::try_from(range.end.checked_sub(range.start)?) + .ok() + .filter(|len| *len != Self::UNREAD_LEN)?; + start.checked_add(len)?; + Some(Self { start, len }) + } + + /// The segments the layer contributed, once its packet has been read. + pub(crate) fn segments(&self) -> Option> { + (self.len != Self::UNREAD_LEN).then(|| { + let start = self.start as usize; + start..start + self.len as usize + }) + } } diff --git a/crates/j2k-native/src/j2c/build/allocation/tests.rs b/crates/j2k-native/src/j2c/build/allocation/tests.rs index b5285a0a..2b19100c 100644 --- a/crates/j2k-native/src/j2c/build/allocation/tests.rs +++ b/crates/j2k-native/src/j2c/build/allocation/tests.rs @@ -520,3 +520,27 @@ fn region_decode_scratch_preserves_the_allocation_boundary() { } } } + +/// A layer record keeps its segment range in 8 bytes, tells a layer whose +/// packet is unread from one that contributed no segments, and refuses a +/// range it can't hold rather than truncating it. +#[test] +fn layer_records_keep_their_segment_ranges() { + use crate::j2c::build::Layer; + + assert_eq!(size_of::(), 8); + assert_eq!(Layer::UNREAD.segments(), None); + for range in [0..0, 0..3, 7..7, 41..48] { + let layer = Layer::with_segments(range.clone()).expect("range fits"); + assert_eq!(layer.segments(), Some(range)); + } + let last = u32::MAX as usize; + assert_eq!( + Layer::with_segments(last - 2..last - 1).and_then(|layer| layer.segments()), + Some(last - 2..last - 1) + ); + assert!(Layer::with_segments(last..last + 1).is_none()); + assert!(Layer::with_segments(0..last).is_none()); + let start = core::hint::black_box(5); + assert!(Layer::with_segments(start..start - 2).is_none()); +} diff --git a/crates/j2k-native/src/j2c/decode.rs b/crates/j2k-native/src/j2c/decode.rs index 860259d9..8581d9b3 100644 --- a/crates/j2k-native/src/j2c/decode.rs +++ b/crates/j2k-native/src/j2c/decode.rs @@ -1491,9 +1491,9 @@ mod tests { #[test] fn collect_classic_code_block_data_preserves_zero_length_segments() { let mut storage = DecompositionStorage::default(); - storage.layers.push(Layer { - segments: Some(0..3), - }); + storage + .layers + .push(Layer::with_segments(0..3).expect("test segment range")); storage.segments.push(Segment { idx: 0, coding_pases: 1, @@ -1541,9 +1541,9 @@ mod tests { #[test] fn collect_classic_code_block_data_rejects_non_contiguous_segment_indices() { let mut storage = DecompositionStorage::default(); - storage.layers.push(Layer { - segments: Some(0..2), - }); + storage + .layers + .push(Layer::with_segments(0..2).expect("test segment range")); storage.segments.push(Segment { idx: 0, coding_pases: 1, diff --git a/crates/j2k-native/src/j2c/decode/direct_plan/classic.rs b/crates/j2k-native/src/j2c/decode/direct_plan/classic.rs index e555c438..e297ab9b 100644 --- a/crates/j2k-native/src/j2c/decode/direct_plan/classic.rs +++ b/crates/j2k-native/src/j2c/decode/direct_plan/classic.rs @@ -54,7 +54,7 @@ pub(crate) fn collect_classic_code_block_data( let is_normal_mode = style.single_codeword_segment(); for layer in &storage.layers[code_block.layers.start..code_block.layers.end] { - let Some(range) = layer.segments.clone() else { + let Some(range) = layer.segments() else { continue; }; @@ -144,7 +144,7 @@ pub(crate) fn collect_referenced_classic_code_block_data( let is_normal_mode = style.single_codeword_segment(); for layer in &storage.layers[code_block.layers.start..code_block.layers.end] { - let Some(range) = layer.segments.clone() else { + let Some(range) = layer.segments() else { continue; }; @@ -227,7 +227,7 @@ pub(crate) fn classic_allocation_counts( let mut fragment_count = 0_usize; for layer in &storage.layers[code_block.layers.start..code_block.layers.end] { - let Some(range) = layer.segments.clone() else { + let Some(range) = layer.segments() else { continue; }; for segment in &storage.segments[range] { diff --git a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs index 6566d370..40374a35 100644 --- a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs +++ b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs @@ -667,6 +667,14 @@ fn plan_tile<'a>( if !banded && !strips_are_disjoint(&mut plan.strips, storage.coefficients.len()) { return Ok(None); } + // Strips were reserved one per block; keep room for the rows planned. + let reserved = plan.strips.capacity(); + plan.strips.shrink_to_fit(); + budget.release_bytes( + reserved + .saturating_sub(plan.strips.capacity()) + .saturating_mul(size_of::()), + )?; budget.include_elements::>(plan.strips.len())?; Ok(Some(plan)) } @@ -727,7 +735,7 @@ fn required_code_blocks( count += 1; segment_bytes = storage.layers[code_block.layers.clone()] .iter() - .filter_map(|layer| layer.segments.clone()) + .filter_map(|layer| layer.segments()) .flat_map(|range| &storage.segments[range]) .fold(segment_bytes, |bytes, segment| { bytes.saturating_add(segment.data.len()) @@ -801,7 +809,7 @@ fn classic_payload<'a>( let single = (counts.fragments <= 1).then(|| { storage.layers[code_block.layers.clone()] .iter() - .filter_map(|layer| layer.segments.clone()) + .filter_map(|layer| layer.segments()) .flat_map(|range| &storage.segments[range]) .map(|segment| segment.data) .find(|data| !data.is_empty()) @@ -823,7 +831,7 @@ fn single_cleanup_segment<'a>( .ok_or(DecodingError::CodeBlockDecodeFailure)?; let mut cleanup = None; for layer in &storage.layers[code_block.layers.clone()] { - let Some(range) = layer.segments.clone() else { + let Some(range) = layer.segments() else { continue; }; for segment in &storage.segments[range] { @@ -975,7 +983,7 @@ fn gather_payload<'s>( .get(code_block.layers.clone()) .ok_or(DecodingError::CodeBlockDecodeFailure)?; for layer in layers { - let Some(range) = layer.segments.clone() else { + let Some(range) = layer.segments() else { continue; }; let segments = source diff --git a/crates/j2k-native/src/j2c/decode/tier1.rs b/crates/j2k-native/src/j2c/decode/tier1.rs index f5db2b15..77a94322 100644 --- a/crates/j2k-native/src/j2c/decode/tier1.rs +++ b/crates/j2k-native/src/j2c/decode/tier1.rs @@ -48,7 +48,7 @@ impl Tier1Requirements { .get(code_block.layers.clone()) .ok_or(DecodingError::CodeBlockDecodeFailure)?; for layer in layers { - let Some(segment_range) = layer.segments.clone() else { + let Some(segment_range) = layer.segments() else { continue; }; let segments = storage diff --git a/crates/j2k-native/src/j2c/encode/single_tile/tests.rs b/crates/j2k-native/src/j2c/encode/single_tile/tests.rs index fe6ec852..2bf0e747 100644 --- a/crates/j2k-native/src/j2c/encode/single_tile/tests.rs +++ b/crates/j2k-native/src/j2c/encode/single_tile/tests.rs @@ -337,7 +337,7 @@ fn contribution_final_bytes(codestream: &[u8]) -> Vec { segment::parse(tile, &parsed.header, &mut storage).expect("parse encoded packets"); for code_block in &storage.code_blocks { for layer in &storage.layers[code_block.layers.clone()] { - let Some(segments) = layer.segments.clone() else { + let Some(segments) = layer.segments() else { continue; }; if let Some(&last) = storage.segments[segments] diff --git a/crates/j2k-native/src/j2c/ht_block_decode/segments.rs b/crates/j2k-native/src/j2c/ht_block_decode/segments.rs index 8b7c4098..8a64b1a2 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/segments.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/segments.rs @@ -71,7 +71,7 @@ pub(crate) fn visit_code_block_segments<'a>( let mut cleanup_seen = false; for layer in &storage.layers[code_block.layers.start..code_block.layers.end] { - let Some(range) = layer.segments.clone() else { + let Some(range) = layer.segments() else { continue; }; diff --git a/crates/j2k-native/src/j2c/ht_block_decode/segments/owned.rs b/crates/j2k-native/src/j2c/ht_block_decode/segments/owned.rs index 00aae36f..5f93250c 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/segments/owned.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/segments/owned.rs @@ -17,7 +17,7 @@ pub(crate) fn selected_code_block_segment_lengths( let mut refinement_length = 0usize; for layer in &storage.layers[code_block.layers.clone()] { - let Some(range) = layer.segments.clone() else { + let Some(range) = layer.segments() else { continue; }; @@ -64,7 +64,7 @@ pub(crate) fn collect_code_block_data_into( data.clear(); for layer in &storage.layers[code_block.layers.clone()] { - let Some(range) = layer.segments.clone() else { + let Some(range) = layer.segments() else { continue; }; for segment in &storage.segments[range] { diff --git a/crates/j2k-native/src/j2c/ht_block_decode/segments/tests.rs b/crates/j2k-native/src/j2c/ht_block_decode/segments/tests.rs index 94411535..8d145952 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/segments/tests.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/segments/tests.rs @@ -23,9 +23,9 @@ fn collector_uses_the_selected_non_empty_ht_set() { non_empty_layer_count: 1, }; let mut storage = DecompositionStorage::default(); - storage.layers.push(Layer { - segments: Some(0..4), - }); + storage + .layers + .push(Layer::with_segments(0..4).expect("test segment range")); storage.segments.extend([ Segment { idx: 0, @@ -88,12 +88,8 @@ fn referenced_collector_visits_refinement_fragments_in_packet_order() { }; let mut storage = DecompositionStorage::default(); storage.layers.extend([ - Layer { - segments: Some(0..2), - }, - Layer { - segments: Some(2..3), - }, + Layer::with_segments(0..2).expect("test segment range"), + Layer::with_segments(2..3).expect("test segment range"), ]); storage.segments.extend([ Segment { @@ -152,12 +148,8 @@ fn owned_collector_concatenates_refinement_fragments_in_packet_order() { }; let mut storage = DecompositionStorage::default(); storage.layers.extend([ - Layer { - segments: Some(0..2), - }, - Layer { - segments: Some(2..3), - }, + Layer::with_segments(0..2).expect("test segment range"), + Layer::with_segments(2..3).expect("test segment range"), ]); storage.segments.extend([ Segment { diff --git a/crates/j2k-native/src/j2c/segment.rs b/crates/j2k-native/src/j2c/segment.rs index e24fc302..e0435b58 100644 --- a/crates/j2k-native/src/j2c/segment.rs +++ b/crates/j2k-native/src/j2c/segment.rs @@ -612,7 +612,7 @@ fn parse_inner<'a>( let layer = &mut storage.layers[code_block.layers.clone()] [progression_data.layer_num as usize]; - if let Some(segments) = layer.segments.clone() { + if let Some(segments) = layer.segments() { let segments = &mut storage.segments[segments.clone()]; for segment in segments { diff --git a/crates/j2k-native/src/j2c/segment/classic.rs b/crates/j2k-native/src/j2c/segment/classic.rs index 6fb191c8..177b68f6 100644 --- a/crates/j2k-native/src/j2c/segment/classic.rs +++ b/crates/j2k-native/src/j2c/segment/classic.rs @@ -6,7 +6,7 @@ use super::{ code_block_is_read, decode_num_coding_passes, read_lblock_increment, read_segment_length, resolve_code_block_inclusion, PacketResult, PacketSkips, SegmentSink, MAX_BITPLANE_COUNT, }; -use crate::j2c::build::{CodeBlock, CodeBlockCoding, Segment}; +use crate::j2c::build::{CodeBlock, CodeBlockCoding, Layer, Segment}; use crate::j2c::codestream::{CodeBlockStyle, ComponentInfo}; use crate::j2c::decode::DecompositionStorage; use crate::j2c::progression::ProgressionData; @@ -92,7 +92,8 @@ pub(super) fn resolve_segments( packet_workspace_error: &mut storage.packet_workspace_error, }, )?; - layer.segments = Some(start..storage.segments.len()); + *layer = Layer::with_segments(start..storage.segments.len()) + .ok_or("classic segment records exceed the layer record range")?; } else { let mut bytes = 0; parse_contribution_lengths( diff --git a/crates/j2k-native/src/j2c/segment/ht.rs b/crates/j2k-native/src/j2c/segment/ht.rs index 03ec2f07..16c194ac 100644 --- a/crates/j2k-native/src/j2c/segment/ht.rs +++ b/crates/j2k-native/src/j2c/segment/ht.rs @@ -7,7 +7,7 @@ use super::{ read_segment_length, resolve_code_block_inclusion, PacketResult, PacketSkips, SegmentSink, MAX_BITPLANE_COUNT, }; -use crate::j2c::build::{CodeBlock, CodeBlockCoding, Segment}; +use crate::j2c::build::{CodeBlock, CodeBlockCoding, Layer, Segment}; use crate::j2c::codestream::{CodeBlockStyle, ComponentInfo}; use crate::j2c::decode::DecompositionStorage; use crate::j2c::progression::ProgressionData; @@ -69,7 +69,7 @@ pub(super) fn resolve_segments( let layer = storage.layers[code_block.layers.clone()] .get_mut(layer_num) .ok_or("HT packet references a missing quality layer")?; - if layer.segments.is_some() { + if layer.segments().is_some() { return Err("HT quality layer already owns segment metadata"); } Some(layer) @@ -112,7 +112,8 @@ pub(super) fn resolve_segments( packet_workspace_error: &mut storage.packet_workspace_error, }, )?; - layer.segments = Some(start..storage.segments.len()); + *layer = Layer::with_segments(start..storage.segments.len()) + .ok_or("HT segment records exceed the layer record range")?; } else { let mut bytes = 0; parse_segment_lengths( diff --git a/crates/j2k-native/src/tests.rs b/crates/j2k-native/src/tests.rs index 3c5e10a3..cf26bc60 100644 --- a/crates/j2k-native/src/tests.rs +++ b/crates/j2k-native/src/tests.rs @@ -1407,9 +1407,9 @@ fn decode_region_rejects_code_block_layer_metadata_amplification() { }; let mut bytes = encode(&pixels, 8, 8, 1, 8, false, &options).expect("encode many-layer classic fixture"); - // About a million 4x4 code-blocks with 32 layers each: their layer + // About four million 4x4 code-blocks with 32 layers each: their layer // records alone would exceed the decode budget. - rewrite_siz_to_single_large_tile(&mut bytes, 4_096); + rewrite_siz_to_single_large_tile(&mut bytes, 8_192); let image = Image::new(&bytes, &DecodeSettings::default()).expect("large SIZ parses"); // A tiny region builds layer records only for the code-blocks it reads, From a4d369370d676b89b5bf8c74e67f539315ec5d8e Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 05:19:50 -0400 Subject: [PATCH 18/94] perf(j2k-native): halve the streamed row batch of tiles with deep band rings A classic tile whose components reconstruct on their own readers, at 12 pool threads or more, holds deeper band rings and a single batch of reconstructed rows, which it transforms before the next batch reconstructs. Nothing overlaps that batch, so a shorter one costs no speed: such a tile now reconstructs half as many rows per batch, as long as that leaves at least 128 rows. Tiles with two pipelined batches (HTJ2K, and classic tiles at fewer threads) keep their batches, as do rows wider than 8192 samples: halving an 18000-wide tile's 116-row batches cost 2 to 5%, and halving every tile's batch cost 5% on a 9000-wide HTJ2K overview. Measured with full-image region decodes to RGB8 (interleaved rounds, median of each run; output byte-identical) on an 18-core ARM64 machine, against the parent: - 5661 x 8287 RGB, one 9/7 tile: 109.6-112.5 ms before and after at 18 and default threads; peak heap 102.8 -> 90.8 MiB (-12.0 MiB, -11.7%). At 1/2 scale 55.9 -> 51.3 MiB at equal speed. At 6 and 1 threads the rings are not deep and nothing changes. - 4096-wide classic RGB mosaics (2bpp and lossless) and a 4096-wide classic RGB image at default threads: equal speed; peak heap 56.7 -> 44.7 MiB and 59.2 -> 47.2 MiB (-21%, -20%). - HTJ2K mosaics, a 9000-wide HTJ2K overview and an 18000 x 12000 classic image keep their batches. - One decode per fresh process against the code before the streamed full routes, at 1, 6, 18 and default threads: the 5661 x 8287 image at full, 1/2, 1/4 and 1/8 scale and the four mosaics run 0.98-1.04x (the noise of paths this leaves alone); the 5661 x 8287 image's footprint (max RSS less input and output) at 18 threads drops from 104.7 to 92.8 MiB. --- crates/j2k-native/src/j2c/build.rs | 2 +- .../src/j2c/build/allocation/tests.rs | 2 +- .../src/j2c/decode/parallel_bands.rs | 8 ++++-- crates/j2k-native/src/j2c/idwt/roi.rs | 27 ++++++++++++++---- crates/j2k-native/src/j2c/idwt/tests.rs | 28 +++++++++++++++++++ 5 files changed, 57 insertions(+), 10 deletions(-) diff --git a/crates/j2k-native/src/j2c/build.rs b/crates/j2k-native/src/j2c/build.rs index 733a9c9e..57500721 100644 --- a/crates/j2k-native/src/j2c/build.rs +++ b/crates/j2k-native/src/j2c/build.rs @@ -772,7 +772,7 @@ impl Layer { } /// The segments the layer contributed, once its packet has been read. - pub(crate) fn segments(&self) -> Option> { + pub(crate) fn segments(self) -> Option> { (self.len != Self::UNREAD_LEN).then(|| { let start = self.start as usize; start..start + self.len as usize diff --git a/crates/j2k-native/src/j2c/build/allocation/tests.rs b/crates/j2k-native/src/j2c/build/allocation/tests.rs index 2b19100c..1d294c86 100644 --- a/crates/j2k-native/src/j2c/build/allocation/tests.rs +++ b/crates/j2k-native/src/j2c/build/allocation/tests.rs @@ -536,7 +536,7 @@ fn layer_records_keep_their_segment_ranges() { } let last = u32::MAX as usize; assert_eq!( - Layer::with_segments(last - 2..last - 1).and_then(|layer| layer.segments()), + Layer::with_segments(last - 2..last - 1).and_then(Layer::segments), Some(last - 2..last - 1) ); assert!(Layer::with_segments(last..last + 1).is_none()); diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index 3d9bd39e..0103f91b 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -391,8 +391,12 @@ pub(super) fn decode_tile_rows<'a>( // Rows reconstruct in batches, charged by the region's width (see // `roi_idwt_peak_elements`): two of them with a reader per component, so // one finishes while the next reconstructs, unless the band rings are - // deep instead. - let batch = streamed_batch_rows(region.width() as usize, copy.height() as usize); + // deep instead: then one batch of half the rows. + let batch = streamed_batch_rows( + region.width() as usize, + copy.height() as usize, + storage.deep_band_rings, + ); let scratch_len = (copy.width() as usize) .checked_mul(components) .and_then(|samples| samples.checked_mul(batch)) diff --git a/crates/j2k-native/src/j2c/idwt/roi.rs b/crates/j2k-native/src/j2c/idwt/roi.rs index ba9b9991..75752e29 100644 --- a/crates/j2k-native/src/j2c/idwt/roi.rs +++ b/crates/j2k-native/src/j2c/idwt/roi.rs @@ -26,12 +26,23 @@ const STREAMED_BATCH_SAMPLES: usize = 2 * 1024 * 1024; /// samples. const STREAMED_MIN_BATCHES: usize = 4; +/// Rows a batch of a tile with deep band rings keeps at least when it is +/// halved: halving the 116-row batches of an 18000-wide tile cost 2 to 5%. +const STREAMED_DEEP_MIN_ROWS: usize = 128; + /// Rows of each component a tile streaming from pool bands reconstructs at -/// once, for `height` rows of `width` samples. -pub(crate) fn streamed_batch_rows(width: usize, height: usize) -> usize { - (STREAMED_BATCH_SAMPLES / width.max(1)) - .min(height / STREAMED_MIN_BATCHES) - .clamp(1, 512) +/// once, for `height` rows of `width` samples. A tile whose band rings are +/// deep holds one batch, which is transformed before the next one +/// reconstructs, so its batches take half the rows while that leaves at +/// least `STREAMED_DEEP_MIN_ROWS`: as fast, with half the rows held. +pub(crate) fn streamed_batch_rows(width: usize, height: usize, deep_band_rings: bool) -> usize { + let rows = STREAMED_BATCH_SAMPLES / width.max(1); + let rows = if deep_band_rings && rows / 2 >= STREAMED_DEEP_MIN_ROWS { + rows / 2 + } else { + rows + }; + rows.min(height / STREAMED_MIN_BATCHES).clamp(1, 512) } /// The most f32 samples [`apply_roi`] holds at once for any one component. @@ -95,7 +106,11 @@ pub(crate) fn roi_idwt_peak_elements( // more bands instead. let batches = if storage.deep_band_rings { 1 } else { 2 }; width - .saturating_mul(streamed_batch_rows(width, height)) + .saturating_mul(streamed_batch_rows( + width, + height, + storage.deep_band_rings, + )) .saturating_mul(batches) } else { width diff --git a/crates/j2k-native/src/j2c/idwt/tests.rs b/crates/j2k-native/src/j2c/idwt/tests.rs index 58ef9102..934433d1 100644 --- a/crates/j2k-native/src/j2c/idwt/tests.rs +++ b/crates/j2k-native/src/j2c/idwt/tests.rs @@ -841,3 +841,31 @@ fn streamed_rows_match_the_whole_window_levels_bit_for_bit() { } } } + +/// A tile streaming from pool bands with deep band rings holds one batch of +/// rows instead of two, so its batch takes half the rows unless that leaves +/// fewer than 128; tiles too short for full batches still split their rows +/// into at least four batches, and a row wider than a batch is a batch of +/// its own. +#[test] +fn deep_band_rings_halve_the_streamed_row_batch() { + use super::roi::streamed_batch_rows; + + // 5661 x 8287 (one tile): 370 rows of each component in each of two + // pipelined batches, 185 rows in the one batch of deep rings. + assert_eq!(streamed_batch_rows(5661, 8287, false), 370); + assert_eq!(streamed_batch_rows(5661, 8287, true), 185); + // 4096 wide: 512 rows, or 256 with deep rings. + assert_eq!(streamed_batch_rows(4096, 4096, false), 512); + assert_eq!(streamed_batch_rows(4096, 4096, true), 256); + for deep_band_rings in [false, true] { + // 18000 wide: 116 rows, too few to halve. + assert_eq!(streamed_batch_rows(18000, 12000, deep_band_rings), 116); + // 8200 wide: 255 rows, too few to halve. + assert_eq!(streamed_batch_rows(8200, 12000, deep_band_rings), 255); + assert_eq!(streamed_batch_rows(256, 200, deep_band_rings), 50); + assert_eq!(streamed_batch_rows(256, 3, deep_band_rings), 1); + assert_eq!(streamed_batch_rows(3 << 20, 4096, deep_band_rings), 1); + assert_eq!(streamed_batch_rows(1024, 1 << 16, deep_band_rings), 512); + } +} From c60942d545489b69c7c61ff890a6b40dc07c5133 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 05:19:50 -0400 Subject: [PATCH 19/94] docs(changelog): note the streamed-core memory and empty-coarse-level fixes --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5818dac3..5136c6d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,12 @@ Staged workspace version: `0.13.2`. Staged windows are admitted by planned execution memory, and an admission claim that cannot fit beside the shared plan fails instead of waiting indefinitely. Successful output is unchanged. +- Region decodes of a tile whose coarse resolutions are empty while finer + ones are not (a one-sample tile row or column at an odd position with two + or more levels) reconstruct it from the first level that has samples, as + full-image decodes do. Before, 8-bit region decodes of such tiles failed + with `IncompletePlaneStore` or `CodeBlockDecodeFailure`, and 16-bit and + component-plane region decodes returned mid-gray samples for them. - x86-64 RGB8 decodes of 9/7 images round and pack the inverse ICT output with the runtime SIMD level (AVX2 where present) instead of a scalar library call per sample: the centered rounding of the inverse MCT runs eight @@ -133,6 +139,10 @@ Staged workspace version: `0.13.2`. batch of rows instead of two. On an 18000 x 12000 RGB image, 1024 x 1024 regions and overviews at 1/8 and 1/16 scale decode 4 to 6% faster at 18 threads, with less peak heap. +- Those tiles reconstruct half as many rows at a time, while that leaves + at least 128 rows, since they hold only one batch. A full-image region + decode of a 5661 x 8287 RGB image to RGB8 peaks 12 MiB (12%) lower at 18 + threads, at the same speed. ## [0.13.1] - 2026-10-29 From 189891d07006eb080500228e7e14473e9587d32c Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 00:54:13 -0400 Subject: [PATCH 20/94] fix(j2k): decode 8-bit full images past the decode budget as a streamed region A full-image decode keeps every component in an image-sized f32 plane, and the native decoder refuses one whose planes and largest tile's workspace pass its budget, so `decode_into` and `decode_scaled_into` of large images failed with a resource-limit error while a region decode of the same pixels streams its rows and fits. An 8-bit full-image decode now asks the native decoder, before decoding, what the planes route would charge, and runs as a region decode of the whole image when that passes the budget. The new hidden `Image::full_decode_planes_floor` plans the charge from the parsed tiles without allocating planes or coefficients: the tiles' structural bytes, every plane as `reset` sizes it, and the largest tile's records, coefficients and IDWT workspace as `build` plans them. Each term is at most what the planes decode charges, so an image the planes decode finishes is never rerouted. It returns None (keep the planes) where the whole-image region would not stream its rows, and for tiles whose coarse resolutions are empty while finer ones are not, which the region IDWTs do not reconstruct yet. A header-only estimate skips the tile parse for images far below the budget. `J2kDecoder` takes the budget from one helper (the native default today); a hidden setter lowers it for tests. Against the planes route's choice (the route disabled) over 26 test images of growing size at 1/1 to 1/8, 41 cases route and 63 keep the planes, none disagreeing, and the routed outputs equal the planes path's with its budget raised. Over 87 boundary-scan sizes no image that decodes today is rerouted; 11 within 1.5% of the budget still fail, as before (the plan leaves out Tier-1 workspaces and, with tiles, earlier tiles' retained scratch). The 5661 x 8287 RGB workload image decodes at full resolution with 102.8 MiB peak heap on the default pool (25.8 MiB on one thread) against 983.7 MiB (952.3) for the planes path with its budget raised, in 132 ms against 123 ms (1367 against 1345 ms on one thread). Untiled RGB images from 5792^2 to 16384^2 take 2-8% longer than that path with 6-17x less heap; 1024^2-tiled RGB 35-45% longer and grayscale 2.4-2.8x as long, the streamed route's own speed. --- crates/j2k-native/src/image/output_api.rs | 38 +++ crates/j2k-native/src/j2c/decode.rs | 2 + .../j2k-native/src/j2c/decode/planes_floor.rs | 315 ++++++++++++++++++ crates/j2k-native/src/j2c/mod.rs | 2 +- crates/j2k/src/decode.rs | 27 ++ crates/j2k/src/decode/route.rs | 53 +++ crates/j2k/src/decode/route/tests.rs | 115 +++++++ crates/j2k/src/view.rs | 74 +++- crates/j2k/src/view/tests.rs | 1 + crates/j2k/tests/full_image_streaming.rs | 215 ++++++++++++ crates/j2k/tests/resource_limits.rs | 10 +- ...able-api-1.0.implementation-public-api.txt | 2 + 12 files changed, 840 insertions(+), 14 deletions(-) create mode 100644 crates/j2k-native/src/j2c/decode/planes_floor.rs create mode 100644 crates/j2k/src/decode/route.rs create mode 100644 crates/j2k/src/decode/route/tests.rs create mode 100644 crates/j2k/tests/full_image_streaming.rs diff --git a/crates/j2k-native/src/image/output_api.rs b/crates/j2k-native/src/image/output_api.rs index 5adfd2d4..f21cb324 100644 --- a/crates/j2k-native/src/image/output_api.rs +++ b/crates/j2k-native/src/image/output_api.rs @@ -523,6 +523,44 @@ impl<'a> Image<'a> { Ok(()) } + /// The bytes a full-image decode into component planes charges the + /// decode budget at least, planned from the codestream's tiles without + /// decoding, when [`Self::decode_region_rows_for_integer_output_with_context`] + /// over the whole image could stream its rows instead and so need no + /// planes. A caller holding the planes to `budget` bytes runs that + /// region decode when the floor passes it: the planes decode would be + /// refused, and an image it can finish never has a floor past the + /// budget. + /// + /// `None` when the region decode could not stream (a palette, channel + /// definition, sYCC or CIELab image, components of more than 24 bits, + /// subsampling, offsets, or a tile whose coarse resolutions are empty + /// while finer ones are not), or when the header alone puts the planes + /// and one tile's coefficients and IDWT buffers within half of `budget`. + /// + /// # Errors + /// + /// Returns the error parsing the tiles meets, which the planes decode + /// meets as well. + #[doc(hidden)] + pub fn full_decode_planes_floor(&self, budget: usize) -> Result> { + if !self.permits_line_region_idwt() + || self + .header + .component_infos + .iter() + .any(|component| component.size_info.precision > 24) + { + return Ok(None); + } + j2c::full_decode_planes_floor( + self.codestream, + &self.header, + self.retained_header_metadata_bytes()?, + budget, + ) + } + /// Rounded-for-integer-output counterpart of /// [`Self::decode_components_with_ht_decoder`]. /// diff --git a/crates/j2k-native/src/j2c/decode.rs b/crates/j2k-native/src/j2c/decode.rs index 8581d9b3..ab9ba605 100644 --- a/crates/j2k-native/src/j2c/decode.rs +++ b/crates/j2k-native/src/j2c/decode.rs @@ -39,6 +39,7 @@ mod direct_plan; #[cfg(feature = "parallel")] mod parallel_bands; mod peak; +mod planes_floor; mod reuse; mod state; mod store; @@ -59,6 +60,7 @@ pub(crate) use self::direct_plan::{ build_referenced_htj2k_grayscale_plan, build_referenced_htj2k_rgba_plan, }; pub(crate) use self::peak::{record_workspace, WorkspaceBytes, WorkspacePeak}; +pub(crate) use self::planes_floor::full_decode_planes_floor; #[cfg(feature = "std")] pub(crate) use self::reuse::is_capacity_error; pub(crate) use self::state::{DecompositionStorage, TileDecodeContext, TileDecompositions}; diff --git a/crates/j2k-native/src/j2c/decode/planes_floor.rs b/crates/j2k-native/src/j2c/decode/planes_floor.rs new file mode 100644 index 00000000..e05bfeb4 --- /dev/null +++ b/crates/j2k-native/src/j2c/decode/planes_floor.rs @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! What a full-image decode into component planes charges the decode budget +//! at least, planned from the codestream before anything is decoded. +//! +//! The planes decode charges, in order: the parsed tiles' structural bytes +//! and every component's image-sized plane (`TileDecodeContext::reset`), then +//! for each tile its code-block records, the coefficients of its largest +//! component and that component's IDWT workspace (`build`'s allocation plan, +//! with the component coefficient layout). Each charge here is the same or +//! smaller, so a decode whose floor passes the budget is refused on the +//! planes, serially or on tile workers, and a decode the planes can finish +//! never has a floor past the budget. The floor leaves out what `build` +//! counts by walking precincts (most tag-tree nodes) and every later stage. + +use core::mem::size_of; + +use super::TileDecompositions; +use crate::checked_decode_sample_count; +use crate::error::{Result, ValidationError}; +use crate::j2c::build::{CodeBlock, Decomposition, Layer, Precinct, SubBand, SubBandType}; +use crate::j2c::codestream::Header; +use crate::j2c::codestream::WaveletTransform; +use crate::j2c::idwt::planned_workspace_elements; +use crate::j2c::rect::IntRect; +use crate::j2c::tag_tree::TagNode; +use crate::j2c::tile::{self, ComponentTile, ResolutionTile, Tile}; +use crate::j2c::ComponentData; +use crate::math::{SimdBuffer, SIMD_WIDTH}; +use crate::reader::BitReader; + +/// The bytes a full-image decode of `data` into component planes charges at +/// least, when its full-image region decode could stream its rows instead: +/// the image and its tiles have no offsets, one to four components at full +/// sampling, an inverse MCT only over three or more components whose first +/// three share a wavelet (as `streams_rows` requires), and no tile-component +/// whose coarse resolutions are empty while finer ones are not, which the +/// region IDWTs do not reconstruct. `None` when it could not, or when the +/// header alone puts the planes, one tile's coefficients and its IDWT +/// buffers within half of `budget`, which leaves the planes decode far from +/// it unless its metadata is unusually large. +/// +/// `retained_image_bytes` is what the image holds while it decodes, the +/// tile parse's baseline. Errors are the tile parse's, which the planes +/// decode reports too. +pub(crate) fn full_decode_planes_floor( + data: &[u8], + header: &Header<'_>, + retained_image_bytes: usize, + budget: usize, +) -> Result> { + let planes = planes_bytes(header)?; + if header_estimate(header, planes).saturating_mul(2) <= budget || !image_streams_rows(header) { + return Ok(None); + } + let mut reader = BitReader::new(data); + let tiles = tile::parse(&mut reader, header, retained_image_bytes)?; + if tiles.is_empty() { + return Ok(None); + } + let skipped = header.skipped_resolution_levels; + let mut tile_floor = 0_usize; + for tile in tiles.iter() { + if !tile_streams_rows(tile, header.component_infos.len(), skipped) { + return Ok(None); + } + tile_floor = tile_floor.max(tile_workspace_floor(tile, skipped)); + } + Ok(Some( + tiles + .structural_workspace_bytes() + .saturating_add(planes) + .saturating_add(tile_floor), + )) +} + +/// What `reset` charges for every component's plane of the output image. +fn planes_bytes(header: &Header<'_>) -> Result { + let samples = checked_decode_sample_count( + header.size_data.image_width(), + header.size_data.image_height(), + )?; + let padded = + SimdBuffer::::padded_len(samples).ok_or(ValidationError::ImageTooLarge)?; + let components = header.component_infos.len(); + Ok(padded + .saturating_mul(size_of::()) + .saturating_mul(components) + .saturating_add(components.saturating_mul(size_of::()))) +} + +/// The planes, plus the coefficients and twice the output-resolution +/// samples of one SIZ tile in `f32`: about what the planes decode charges +/// for its largest tile, the IDWT's output, next-level scratch and band +/// scratch included, without records or metadata. +fn header_estimate(header: &Header<'_>, planes: usize) -> usize { + let size = &header.size_data; + let full = |tile: u32, extent: u32, offset: u32| tile.min(extent.saturating_sub(offset)); + let width = full( + size.tile_width, + size.reference_grid_width, + size.tile_x_offset, + ); + let height = full( + size.tile_height, + size.reference_grid_height, + size.tile_y_offset, + ); + let reduced = |length: u32| { + u32::try_from(u64::from(length) >> header.skipped_resolution_levels) + .unwrap_or(u32::MAX) + .saturating_add(2) + }; + let full_samples = (width as usize).saturating_mul(height as usize); + let output_samples = (reduced(width) as usize).saturating_mul(reduced(height) as usize); + planes.saturating_add( + full_samples + .saturating_add(output_samples.saturating_mul(2)) + .saturating_mul(size_of::()), + ) +} + +/// The image-level conditions of `streams_rows`: no offsets and one to four +/// components. +fn image_streams_rows(header: &Header<'_>) -> bool { + let size = &header.size_data; + size.image_area_x_offset == 0 + && size.image_area_y_offset == 0 + && size.tile_x_offset == 0 + && size.tile_y_offset == 0 + && (1..=4).contains(&header.component_infos.len()) +} + +/// The tile-level conditions of `streams_rows` and the region IDWTs that +/// `tile`'s header settles. +fn tile_streams_rows(tile: &Tile<'_>, components: usize, skipped: u8) -> bool { + let infos = &tile.component_infos; + infos.len() == components + && infos.iter().all(|component| { + component.size_info.horizontal_resolution == 1 + && component.size_info.vertical_resolution == 1 + && !component.requires_exact_integer_decode() + }) + && (!tile.mct + || (components >= 3 + && infos[1..3].iter().all(|component| { + component.wavelet_transform() == infos[0].wavelet_transform() + }))) + && !tile + .component_tiles() + .any(|component_tile| has_empty_coarse_resolution(&component_tile, skipped)) +} + +/// Whether a resolution the decode reconstructs, up to its output, is empty +/// while the component tile is not: a one-sample tile row or column at an +/// odd position, say, has no samples at its coarsest levels. +fn has_empty_coarse_resolution(component_tile: &ComponentTile<'_>, skipped: u8) -> bool { + if component_tile.rect.is_empty() { + return false; + } + let output_resolution = component_tile + .component_info + .num_decomposition_levels() + .saturating_sub(skipped); + component_tile + .resolution_tiles() + .take(usize::from(output_resolution) + 1) + .any(|resolution_tile| resolution_tile.rect.is_empty()) +} + +/// What `build` charges for `tile` at least beside the planes: its records, +/// its largest component's `f32` coefficients at full resolution, and the +/// largest component IDWT workspace at the output resolution. +fn tile_workspace_floor(tile: &Tile<'_>, skipped: u8) -> usize { + let mut records = Records::default(); + let mut coefficients = 0_usize; + let mut idwt = 0_usize; + for component_tile in tile.component_tiles() { + coefficients = coefficients.max(rect_samples(component_tile.rect)); + records.tile_decompositions = records.tile_decompositions.saturating_add(1); + let active = component_tile + .component_info + .num_decomposition_levels() + .saturating_sub(skipped); + let mut highest = None; + let mut second_highest = None; + let mut third_highest = None; + let mut ll_rect = IntRect::from_xywh(0, 0, 0, 0); + for resolution_tile in component_tile.resolution_tiles() { + if resolution_tile.resolution == 0 { + ll_rect = resolution_tile.sub_band_rect(SubBandType::LowLow); + records.add_sub_band(&resolution_tile, SubBandType::LowLow, tile.num_layers); + continue; + } + records.decompositions = records.decompositions.saturating_add(1); + if resolution_tile.resolution <= active { + third_highest = second_highest; + second_highest = highest; + highest = Some(resolution_tile.rect); + } + for sub_band_type in [ + SubBandType::HighLow, + SubBandType::LowHigh, + SubBandType::HighHigh, + ] { + records.add_sub_band(&resolution_tile, sub_band_type, tile.num_layers); + } + } + let transform = component_tile.component_info.wavelet_transform(); + idwt = idwt.max(idwt_workspace( + highest, + [second_highest, third_highest], + ll_rect, + transform, + )); + } + records.bytes().saturating_add( + coefficients + .saturating_add(idwt) + .saturating_mul(size_of::()), + ) +} + +/// `IdwtWorkspaceTracker::finish`: the IDWT workspace of the highest active +/// level and the two below it (see `idwt::planned_workspace_elements`), or +/// the LL band when no level is active. A size `build` cannot represent is +/// refused there, so it counts as more than any budget. +fn idwt_workspace( + highest: Option, + below: [Option; 2], + ll_rect: IntRect, + transform: WaveletTransform, +) -> usize { + let Some(highest) = highest else { + return rect_samples(ll_rect); + }; + planned_workspace_elements(highest, below, transform).unwrap_or(usize::MAX) +} + +fn rect_samples(rect: IntRect) -> usize { + (rect.width() as usize).saturating_mul(rect.height() as usize) +} + +/// Record counts `build` plans for a tile, tag-tree nodes only as the two +/// trees of each precinct need at least: a root, and a leaf per code-block. +#[derive(Default)] +struct Records { + tile_decompositions: usize, + decompositions: usize, + sub_bands: usize, + precincts: usize, + code_blocks: usize, + layers: usize, + tag_tree_nodes: usize, +} + +impl Records { + fn add_sub_band( + &mut self, + resolution_tile: &ResolutionTile<'_>, + sub_band_type: SubBandType, + layers: u8, + ) { + let precincts = usize::try_from(resolution_tile.num_precincts()).unwrap_or(usize::MAX); + let code_blocks = code_block_count( + resolution_tile, + resolution_tile.sub_band_rect(sub_band_type), + ); + self.sub_bands = self.sub_bands.saturating_add(1); + self.precincts = self.precincts.saturating_add(precincts); + self.code_blocks = self.code_blocks.saturating_add(code_blocks); + self.layers = self + .layers + .saturating_add(code_blocks.saturating_mul(usize::from(layers))); + self.tag_tree_nodes = self + .tag_tree_nodes + .saturating_add(precincts.max(code_blocks).saturating_mul(2)); + } + + fn bytes(&self) -> usize { + [ + self.tile_decompositions + .saturating_mul(size_of::()), + self.decompositions + .saturating_mul(size_of::()), + self.sub_bands.saturating_mul(size_of::()), + self.precincts.saturating_mul(size_of::()), + self.code_blocks.saturating_mul(size_of::()), + self.layers.saturating_mul(size_of::()), + self.tag_tree_nodes.saturating_mul(size_of::()), + ] + .into_iter() + .fold(0, usize::saturating_add) + } +} + +/// The code-blocks of a sub-band, as `build` counts them before walking +/// precincts. +fn code_block_count(resolution_tile: &ResolutionTile<'_>, sub_band_rect: IntRect) -> usize { + if sub_band_rect.is_empty() { + return 0; + } + let cells = |start: u32, end: u32, size: u32| end.div_ceil(size).saturating_sub(start / size); + let columns = cells( + sub_band_rect.x0, + sub_band_rect.x1, + resolution_tile.code_block_width(), + ); + let rows = cells( + sub_band_rect.y0, + sub_band_rect.y1, + resolution_tile.code_block_height(), + ); + (columns as usize).saturating_mul(rows as usize) +} diff --git a/crates/j2k-native/src/j2c/mod.rs b/crates/j2k-native/src/j2c/mod.rs index 5f8c71d8..555d684d 100644 --- a/crates/j2k-native/src/j2c/mod.rs +++ b/crates/j2k-native/src/j2c/mod.rs @@ -59,7 +59,7 @@ pub(crate) use decode::{ build_referenced_classic_rgba_plan, build_referenced_htj2k_color_plan, build_referenced_htj2k_grayscale_plan, build_referenced_htj2k_rgba_plan, decode_preparsed_with_capacity_retry as decode_preparsed, - decode_rows_with_capacity_retry as decode_rows, prepare_region_tiles, + decode_rows_with_capacity_retry as decode_rows, full_decode_planes_floor, prepare_region_tiles, }; pub use decode::{CpuDecodeParallelism, DecoderContext, DecoderWorkspace, DecoderWorkspaceStats}; pub use recode::Reversible53CoefficientImage; diff --git a/crates/j2k/src/decode.rs b/crates/j2k/src/decode.rs index bb7aeeb4..3196cd52 100644 --- a/crates/j2k/src/decode.rs +++ b/crates/j2k/src/decode.rs @@ -12,12 +12,14 @@ pub use component_handoff::{ J2kNativeComponentPlane, }; mod output; +mod route; mod settings; mod srgb8; use output::{ can_decode_u8_directly, expand_rgb8_to_rgba8, write_components_u16_output, write_components_u8_output, write_u16_output, write_u8_output, U8RowSink, }; +pub(crate) use route::{full_image_route, FullImageRoute}; pub use settings::DecodeSettings; pub(crate) use srgb8::decode_image_srgb8; pub use srgb8::{J2kSrgb8Image, J2kSrgb8Layout}; @@ -62,6 +64,31 @@ pub(crate) fn decode_warnings_for_image(image: &Image<'_>) -> Vec( + image: &Image<'a>, + native_context: &mut j2k_native::DecoderContext<'a>, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + route: FullImageRoute, +) -> Result<(), J2kError> { + match route { + FullImageRoute::Planes => { + decode_image_into_with_native_context(image, native_context, out, stride, fmt) + } + FullImageRoute::StreamedRegion => decode_image_region_into_with_native_context( + image, + native_context, + out, + stride, + fmt, + Rect::full((image.width(), image.height())), + ), + } +} + pub(crate) fn decode_image_into_with_native_context<'a>( image: &Image<'a>, native_context: &mut j2k_native::DecoderContext<'a>, diff --git a/crates/j2k/src/decode/route.rs b/crates/j2k/src/decode/route.rs new file mode 100644 index 00000000..6bbb8c6e --- /dev/null +++ b/crates/j2k/src/decode/route.rs @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Plan-time choice of how a full-image 8-bit decode runs. +//! +//! The planes route decodes every component into an image-sized plane of +//! `f32` samples and packs the output from them. Its working memory grows +//! with the image, and the native decoder refuses a decode whose planes and +//! largest tile's workspace pass its budget. Such an image instead runs as a +//! region decode of the whole image, which hands finished rows to the output +//! and keeps no planes (see +//! `Image::decode_region_rows_for_integer_output_with_context`). The native +//! decoder plans the planes route's charge from the codestream before +//! anything is decoded (see `Image::full_decode_planes_floor`), so a decode +//! never reruns after a failure, and an image the planes route can decode +//! keeps it, with its output and speed. + +use crate::backend::Image; +use j2k_core::PixelFormat; + +/// How a full-image decode runs. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum FullImageRoute { + /// Component planes for the whole image. + Planes, + /// A region decode of the whole image that streams its rows. + StreamedRegion, +} + +/// How a full-image decode of `image` to `fmt` runs when its planes may +/// charge `planes_budget` bytes: as a streamed region when the output has +/// 8 bits per sample and the planes would pass the budget. +/// +/// A planning error keeps the planes route, which meets the same error and +/// reports it. +pub(crate) fn full_image_route( + image: &Image<'_>, + fmt: PixelFormat, + planes_budget: usize, +) -> FullImageRoute { + if !matches!( + fmt, + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Gray8 + ) { + return FullImageRoute::Planes; + } + match image.full_decode_planes_floor(planes_budget) { + Ok(Some(floor)) if floor > planes_budget => FullImageRoute::StreamedRegion, + _ => FullImageRoute::Planes, + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/j2k/src/decode/route/tests.rs b/crates/j2k/src/decode/route/tests.rs new file mode 100644 index 00000000..e8f16c03 --- /dev/null +++ b/crates/j2k/src/decode/route/tests.rs @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::{full_image_route, FullImageRoute}; +use crate::backend::image_at_scale; +use crate::DecodeSettings; +use j2k_core::{Downscale, PixelFormat}; +use j2k_native::{encode, EncodeOptions}; + +fn codestream(components: u16, options: &EncodeOptions) -> Vec { + let (width, height) = (96_u32, 80_u32); + let pixels: Vec = (0..width * height * u32::from(components)) + .map(|index| (index.wrapping_mul(2_654_435_761) >> 24) as u8) + .collect(); + encode(&pixels, width, height, components, 8, false, options).expect("encode fixture") +} + +fn route(bytes: &[u8], scale: Downscale, fmt: PixelFormat, budget: usize) -> FullImageRoute { + let image = image_at_scale(bytes, DecodeSettings::default(), scale).expect("image"); + full_image_route(&image, fmt, budget) +} + +/// The floor the native decoder plans for the planes of `bytes` at `scale`. +fn floor(bytes: &[u8], scale: Downscale) -> usize { + let image = image_at_scale(bytes, DecodeSettings::default(), scale).expect("image"); + image + .full_decode_planes_floor(0) + .expect("plan") + .expect("a streamable image has a floor") +} + +#[test] +fn an_image_streams_only_when_its_planes_pass_the_budget() { + let fixtures = [ + codestream( + 3, + &EncodeOptions { + num_decomposition_levels: 3, + reversible: false, + ..EncodeOptions::default() + }, + ), + codestream( + 1, + &EncodeOptions { + num_decomposition_levels: 3, + tile_size: Some((32, 32)), + use_ht_block_coding: true, + ..EncodeOptions::default() + }, + ), + ]; + for bytes in &fixtures { + for scale in [Downscale::None, Downscale::Half, Downscale::Quarter] { + let floor = floor(bytes, scale); + // The planes alone of the smallest output are more than this. + assert!(floor > 20 * 24 * 4, "{scale:?}: {floor}"); + for fmt in [PixelFormat::Rgb8, PixelFormat::Rgba8, PixelFormat::Gray8] { + assert_eq!( + route(bytes, scale, fmt, floor - 1), + FullImageRoute::StreamedRegion + ); + assert_eq!(route(bytes, scale, fmt, floor), FullImageRoute::Planes); + assert_eq!( + route(bytes, scale, fmt, j2k_native::DEFAULT_MAX_DECODE_BYTES), + FullImageRoute::Planes + ); + } + } + } +} + +#[test] +fn reduced_decodes_plan_full_resolution_coefficients() { + let bytes = codestream( + 1, + &EncodeOptions { + num_decomposition_levels: 3, + ..EncodeOptions::default() + }, + ); + // A quarter of the planes and IDWT buffers, but every coefficient. + let full = floor(&bytes, Downscale::None); + let half = floor(&bytes, Downscale::Half); + assert!(half < full, "{half} < {full}"); + assert!(half > 96 * 80 * 4, "{half}"); +} + +#[test] +fn sixteen_bit_output_keeps_the_planes() { + let bytes = codestream(3, &EncodeOptions::default()); + for fmt in [PixelFormat::Rgb16, PixelFormat::Rgba16, PixelFormat::Gray16] { + assert_eq!( + route(&bytes, Downscale::None, fmt, 0), + FullImageRoute::Planes + ); + } +} + +/// A tile whose coarse resolutions are empty while finer ones are not keeps +/// the planes, which reconstruct it, whatever the budget. +#[test] +fn tiles_with_empty_coarse_levels_keep_the_planes() { + let gray: &[u8] = include_bytes!( + "../../../../j2k-test-support/fixtures/empty_coarse_levels/openjpeg_gray_8x6_tile8x5_53.j2k" + ); + let rgb: &[u8] = include_bytes!( + "../../../../j2k-test-support/fixtures/empty_coarse_levels/openjpeg_rgb_6x6_tile5x5_97.j2k" + ); + for (bytes, fmt) in [(gray, PixelFormat::Gray8), (rgb, PixelFormat::Rgb8)] { + let image = + image_at_scale(bytes, DecodeSettings::default(), Downscale::None).expect("image"); + assert_eq!(image.full_decode_planes_floor(0).expect("plan"), None); + assert_eq!(full_image_route(&image, fmt, 0), FullImageRoute::Planes); + } +} diff --git a/crates/j2k/src/view.rs b/crates/j2k/src/view.rs index 99777626..ea3f6fb8 100644 --- a/crates/j2k/src/view.rs +++ b/crates/j2k/src/view.rs @@ -7,9 +7,9 @@ use crate::{ inspect_info_from_image, Image, }, decode::{ - decode_image_into_with_native_context, decode_image_region_into_with_native_context, - decode_warnings_for_image, validate_buffer, validate_region, J2kDecodeOutcome, - J2kDecodedComponents, J2kDecodedNativeComponents, + decode_full_image_into_with_native_context, decode_image_region_into_with_native_context, + decode_warnings_for_image, full_image_route, validate_buffer, validate_region, + FullImageRoute, J2kDecodeOutcome, J2kDecodedComponents, J2kDecodedNativeComponents, }, parse::{parse_image_info, parse_info}, scratch::J2kScratchPool, @@ -123,6 +123,9 @@ pub struct J2kDecoder<'a> { image: Option>, settings: DecodeSettings, native_context: j2k_native::DecoderContext<'a>, + /// Set by [`Self::set_full_decode_planes_budget`]; see + /// [`Self::full_decode_planes_budget`]. + full_decode_planes_budget: Option, } /// Options for bounded J2K row decoding. @@ -190,6 +193,7 @@ impl<'a> J2kDecoder<'a> { image: view.image, settings: view.settings, native_context: j2k_native::DecoderContext::default(), + full_decode_planes_budget: None, }) } @@ -316,9 +320,15 @@ impl<'a> J2kDecoder<'a> { /// Decode the full image into `out` using `stride` bytes per output row. /// + /// An 8-bit decode whose component planes would pass the native decode + /// budget runs as a region decode of the whole image, which writes rows + /// as they are finished instead of holding the planes. + /// /// # Errors /// Returns [`J2kError`] when the format is unsupported, the output buffer - /// is too small, or the codestream fails during decode. + /// is too small, or the codestream fails during decode. A decode that + /// runs as a region decode and fails partway may already have written + /// some rows of `out`. pub fn decode_into( &mut self, out: &mut [u8], @@ -374,13 +384,29 @@ impl<'a> J2kDecoder<'a> { ) -> Result { validate_buffer(self.info.dimensions, out.len(), stride, fmt)?; self.ensure_image()?; + let planes_budget = self.full_decode_planes_budget(); let (Some(image), native_context) = (self.image.as_ref(), &mut self.native_context) else { return Err(J2kError::internal_backend("internal image cache missing")); }; - decode_image_into_with_native_context(image, native_context, out, stride, fmt)?; + let route = full_image_route(image, fmt, planes_budget); + let decoded = decode_full_image_into_with_native_context( + image, + native_context, + out, + stride, + fmt, + route, + ); + let warnings = decode_warnings_for_image(image); + if route == FullImageRoute::StreamedRegion { + // As after a region decode, release the decode buffers rather + // than hold them while the decoder is idle. + self.native_context = self.fresh_native_context(); + } + decoded?; Ok(j2k_core::DecodeOutcome::new( Rect::full(self.info.dimensions), - decode_warnings_for_image(image), + warnings, )) } @@ -440,12 +466,16 @@ impl<'a> J2kDecoder<'a> { /// Decode the full image at a reduced resolution. /// /// `scale` uses the shared [`Downscale`] contract; `Downscale::None` - /// delegates to full-resolution decode. + /// delegates to full-resolution decode. As with [`Self::decode_into`], + /// an 8-bit decode whose component planes would pass the native decode + /// budget runs as a region decode of the whole reduced image. /// /// # Errors /// Returns [`J2kError`] when the format or scale request is unsupported, /// including a scale deeper than the codestream's resolution ladder, the - /// output buffer is too small, or decode fails. + /// output buffer is too small, or decode fails. A decode that runs as a + /// region decode and fails partway may already have written some rows of + /// `out`. pub fn decode_scaled_into( &mut self, pool: &mut J2kScratchPool, @@ -462,7 +492,15 @@ impl<'a> J2kDecoder<'a> { let image_dims = (image.width(), image.height()); validate_buffer(image_dims, out.len(), stride, fmt)?; let mut native_context = self.fresh_native_context(); - decode_image_into_with_native_context(&image, &mut native_context, out, stride, fmt)?; + let route = full_image_route(&image, fmt, self.full_decode_planes_budget()); + decode_full_image_into_with_native_context( + &image, + &mut native_context, + out, + stride, + fmt, + route, + )?; Ok(j2k_core::DecodeOutcome::new( Rect::full(image_dims), warnings, @@ -529,6 +567,24 @@ impl<'a> J2kDecoder<'a> { .ok_or_else(|| J2kError::internal_backend("internal image cache missing")) } + /// The bytes a full-image 8-bit decode's component planes may charge + /// before it runs as a streamed region decode of the whole image + /// instead: the native decode budget, past which the planes decode is + /// refused. + fn full_decode_planes_budget(&self) -> usize { + self.full_decode_planes_budget + .unwrap_or(j2k_native::DEFAULT_MAX_DECODE_BYTES) + } + + /// Test hook: hold full-image 8-bit decodes' component planes to + /// `bytes`, so that a small image takes the streamed region route that + /// only images past the native decode budget take otherwise. The + /// native decode budget itself is unchanged. + #[doc(hidden)] + pub fn set_full_decode_planes_budget(&mut self, bytes: usize) { + self.full_decode_planes_budget = Some(bytes); + } + fn fresh_native_context(&self) -> j2k_native::DecoderContext<'a> { let mut native_context = j2k_native::DecoderContext::default(); native_context.set_cpu_decode_parallelism(self.native_context.cpu_decode_parallelism()); diff --git a/crates/j2k/src/view/tests.rs b/crates/j2k/src/view/tests.rs index 2e136f93..78461475 100644 --- a/crates/j2k/src/view/tests.rs +++ b/crates/j2k/src/view/tests.rs @@ -29,6 +29,7 @@ fn fresh_native_context_preserves_configured_parallelism() { image: None, settings: DecodeSettings::strict(), native_context: j2k_native::DecoderContext::default(), + full_decode_planes_budget: None, }; decoder.set_cpu_decode_parallelism(CpuDecodeParallelism::Serial); diff --git a/crates/j2k/tests/full_image_streaming.rs b/crates/j2k/tests/full_image_streaming.rs new file mode 100644 index 00000000..4f9e2513 --- /dev/null +++ b/crates/j2k/tests/full_image_streaming.rs @@ -0,0 +1,215 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! An 8-bit full-image decode whose component planes would pass the decode +//! budget runs as a region decode of the whole image that streams its rows. +//! A decoder whose planes budget is lowered takes that route for small +//! images, and must write the bytes the planes route writes, at every scale, +//! stride and parallelism, and fail where it fails. + +use j2k::{CpuDecodeParallelism, J2kDecoder, J2kScratchPool}; +use j2k_core::{Downscale, PixelFormat}; +use j2k_native::{encode, EncodeOptions}; +use j2k_test_support::wrap_jp2_rgba_codestream; + +const WIDTH: u32 = 203; +const HEIGHT: u32 = 157; +const PADDING: u8 = 0xab; + +fn noisy_pixels(components: u32) -> Vec { + let mut state = 0x2545_f491_u32; + (0..WIDTH * HEIGHT * components) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + (state >> 24) as u8 + }) + .collect() +} + +fn codestream(components: u16, options: &EncodeOptions) -> Vec { + let pixels = noisy_pixels(u32::from(components)); + encode(&pixels, WIDTH, HEIGHT, components, 8, false, options).expect("encode fixture") +} + +/// The image at `scale` in a buffer padded past each row, or `None` when the +/// decode fails, from a decoder whose planes budget is `planes_budget` when +/// one is given. +fn decode( + bytes: &[u8], + planes_budget: Option, + parallelism: CpuDecodeParallelism, + fmt: PixelFormat, + scale: Downscale, + padding: usize, +) -> Option> { + let mut decoder = J2kDecoder::new(bytes).expect("decoder"); + decoder.set_cpu_decode_parallelism(parallelism); + if let Some(bytes) = planes_budget { + decoder.set_full_decode_planes_budget(bytes); + } + let (width, height) = decoder.info().dimensions; + let divisor = scale.denominator(); + let (width, height) = (width.div_ceil(divisor), height.div_ceil(divisor)); + let row = width as usize * fmt.bytes_per_pixel(); + let stride = row + padding; + let mut out = vec![PADDING; stride * height as usize]; + let pool = &mut J2kScratchPool::new(); + let result = match scale { + Downscale::None => decoder.decode_into(&mut out, stride, fmt), + _ => decoder.decode_scaled_into(pool, &mut out, stride, fmt, scale), + }; + result.ok()?; + for line in out.chunks(stride) { + assert!( + line[row..].iter().all(|&byte| byte == PADDING), + "stride padding untouched" + ); + } + Some(out) +} + +#[test] +fn streamed_full_image_matches_planes() { + let base = EncodeOptions { + num_decomposition_levels: 4, + ..EncodeOptions::default() + }; + let rgb = codestream( + 3, + &EncodeOptions { + reversible: false, + ..base.clone() + }, + ); + let rgb_tiled_ht = codestream( + 3, + &EncodeOptions { + reversible: true, + tile_size: Some((64, 64)), + use_ht_block_coding: true, + ..base.clone() + }, + ); + let gray = codestream( + 1, + &EncodeOptions { + reversible: true, + ..base.clone() + }, + ); + let gray_tiled = codestream( + 1, + &EncodeOptions { + reversible: false, + tile_size: Some((96, 64)), + ..base.clone() + }, + ); + let rgba = wrap_jp2_rgba_codestream( + &codestream( + 4, + &EncodeOptions { + reversible: false, + ..base + }, + ), + WIDTH, + HEIGHT, + 8, + ); + let (rgb_formats, gray_formats) = ( + &[PixelFormat::Rgb8, PixelFormat::Rgba8, PixelFormat::Gray8][..], + &[PixelFormat::Gray8, PixelFormat::Rgb8][..], + ); + let fixtures = [ + ("rgb", rgb, rgb_formats), + ("rgb tiled HT", rgb_tiled_ht, rgb_formats), + ("gray", gray, gray_formats), + ("gray tiled", gray_tiled, gray_formats), + ("rgba", rgba, rgb_formats), + ]; + let mut decoded = 0; + for (label, bytes, formats) in &fixtures { + for &fmt in *formats { + for scale in [Downscale::None, Downscale::Half, Downscale::Eighth] { + for (parallelism, padding) in [ + (CpuDecodeParallelism::Serial, 5), + (CpuDecodeParallelism::Auto, 0), + ] { + let planes = decode(bytes, None, parallelism, fmt, scale, padding); + let streamed = decode(bytes, Some(0), parallelism, fmt, scale, padding); + assert!( + planes == streamed, + "{label} {fmt:?} {scale:?} {parallelism:?}: planes {} streamed {}", + planes.is_some(), + streamed.is_some() + ); + decoded += usize::from(planes.is_some()); + } + } + } + } + // Gray to RGB and RGB to gray are rejected; the rest must decode. + assert!(decoded >= 7 * 3 * 2, "{decoded} decodes succeeded"); +} + +/// Tiles whose coarse resolutions are empty while finer ones are not (a +/// one-sample tile row or column at an odd position, with two levels) keep +/// the planes route, which reconstructs them, however low the budget. +#[test] +fn tiles_with_empty_coarse_levels_keep_the_planes() { + let gray: &[u8] = include_bytes!( + "../../j2k-test-support/fixtures/empty_coarse_levels/openjpeg_gray_8x6_tile8x5_53.j2k" + ); + let rgb: &[u8] = include_bytes!( + "../../j2k-test-support/fixtures/empty_coarse_levels/openjpeg_rgb_6x6_tile5x5_97.j2k" + ); + for (label, bytes, fmt) in [ + ("gray 5/3", gray, PixelFormat::Gray8), + ("rgb 9/7", rgb, PixelFormat::Rgb8), + ] { + for parallelism in [CpuDecodeParallelism::Serial, CpuDecodeParallelism::Auto] { + for scale in [Downscale::None, Downscale::Half] { + let planes = decode(bytes, None, parallelism, fmt, scale, 0); + let lowered = decode(bytes, Some(0), parallelism, fmt, scale, 0); + assert!(planes.is_some(), "{label} {parallelism:?} {scale:?}"); + assert_eq!(planes, lowered, "{label} {parallelism:?} {scale:?}"); + } + } + } +} + +/// A decoder that streamed one full-image decode decodes the next on the +/// planes, and the other way round, from the context it kept. +#[test] +fn routes_alternate_on_one_decoder() { + let bytes = codestream( + 3, + &EncodeOptions { + num_decomposition_levels: 3, + reversible: false, + ..EncodeOptions::default() + }, + ); + let fmt = PixelFormat::Rgb8; + let stride = WIDTH as usize * 3; + let expected = decode( + &bytes, + None, + CpuDecodeParallelism::Auto, + fmt, + Downscale::None, + 0, + ) + .expect("planes decode"); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + for budget in [usize::MAX, 0, usize::MAX, 0] { + decoder.set_full_decode_planes_budget(budget); + let mut out = vec![0; stride * HEIGHT as usize]; + decoder + .decode_into(&mut out, stride, fmt) + .unwrap_or_else(|error| panic!("budget {budget}: {error:?}")); + assert!(out == expected, "budget {budget}"); + } +} diff --git a/crates/j2k/tests/resource_limits.rs b/crates/j2k/tests/resource_limits.rs index 3b82a1eb..d7af1d1c 100644 --- a/crates/j2k/tests/resource_limits.rs +++ b/crates/j2k/tests/resource_limits.rs @@ -71,15 +71,17 @@ fn decoder_component_planes_over_the_cap_report_requested_and_cap_bytes() { #[test] fn decode_into_over_the_cap_reports_requested_and_cap_bytes() { // 11600x11600 gray needs 538 MB of f32 planes, over the 512 MiB decode - // cap. The decode is rejected before it writes the caller's buffer. + // cap. The decode is rejected before it writes the caller's buffer. An + // 8-bit decode past the cap streams as a region of the whole image + // instead; 16-bit output has no streamed route and keeps the planes. let dimension = 11_600_u32; let codestream = large_siz_codestream(dimension); let mut decoder = J2kDecoder::new(&codestream).expect("large SIZ parses"); - let stride = dimension as usize; - let mut out = vec![0_u8; stride * stride]; + let stride = dimension as usize * 2; + let mut out = vec![0_u8; stride * dimension as usize]; let error = decoder - .decode_into(&mut out, stride, PixelFormat::Gray8) + .decode_into(&mut out, stride, PixelFormat::Gray16) .expect_err("the planes exceed the decode cap"); let (requested, cap) = resource_limit(&error); diff --git a/xtask/api/stable-api-1.0.implementation-public-api.txt b/xtask/api/stable-api-1.0.implementation-public-api.txt index cf5d0d7e..3e2e1341 100644 --- a/xtask/api/stable-api-1.0.implementation-public-api.txt +++ b/xtask/api/stable-api-1.0.implementation-public-api.txt @@ -42,6 +42,7 @@ pub fn j2k::J2kDecoder<'a>::decode_scaled_into(&mut self, &mut Self::Pool, &mut pub fn j2k::J2kDecoder<'a>::from_view(Self::View) -> core::result::Result pub fn j2k::J2kDecoder<'a>::inspect(&'a [u8]) -> core::result::Result pub fn j2k::J2kDecoder<'a>::parse(&'a [u8]) -> core::result::Result +pub fn j2k::J2kDecoder<'a>::set_full_decode_planes_budget(&mut self, usize) pub fn j2k::J2kError::is_buffer_error(&self) -> bool pub fn j2k::J2kError::is_not_implemented(&self) -> bool pub fn j2k::J2kError::is_truncated(&self) -> bool @@ -2507,6 +2508,7 @@ pub fn j2k_native::Image<'a>::decode_region_components_with_ht_decoder<'ctx>(&se pub fn j2k_native::Image<'a>::decode_region_rows_for_integer_output_with_context(&self, (u32, u32, u32, u32), &mut j2k_native::DecoderContext<'a>, &mut dyn j2k_native::ComponentRowSink) -> j2k_native::Result<()> pub fn j2k_native::Image<'a>::decode_reversible_53_coefficients(&self) -> j2k_native::Result pub fn j2k_native::Image<'a>::decode_reversible_53_coefficients_with_context(&self, &mut j2k_native::DecoderContext<'a>) -> j2k_native::Result +pub fn j2k_native::Image<'a>::full_decode_planes_floor(&self, usize) -> j2k_native::Result> pub fn j2k_native::Image<'a>::new_with_reduction(&'a [u8], &j2k_native::DecodeSettings, u8) -> j2k_native::Result pub fn j2k_native::Image<'a>::new_with_retained_baseline(&'a [u8], &j2k_native::DecodeSettings, usize) -> j2k_native::Result pub fn j2k_native::Image<'a>::prepare_region_decoder_with_context<'image, 'context>(&'image self, &'context mut j2k_native::DecoderContext<'a>) -> j2k_native::Result> From 109d640ea84c56d80e8d2ef72c170a6144f532d7 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 04:03:06 -0400 Subject: [PATCH 21/94] perf(j2k): stream full-image decodes by default where streaming is faster A full-image 8-bit decode now runs on the streamed route not only past the decode budget but wherever streaming was measured at least as fast, chosen before decoding from the header and tiles (no rerun after a failure): - reduced-resolution decodes (decode_scaled_into) of untiled three-component classic images at any thread count: on the 5661 x 8287 workload at 1/2, 1/4, 1/8, 5-47% faster warm and 22-64% faster cold on 6 threads or more, equal on one, with 4-13x less heap; - full-resolution untiled ones of a megapixel or more on 2 to 11 pool threads (about 10% faster warm on 6 threads, 6-12% faster cold); from 12 threads on, where classic tiles reconstruct on pool readers, the planes stay 3-6% faster; - images with a busy tile for every worker, whose tiles now decode at once (new StreamedTiles route): the calling thread and one scoped thread per pool thread each region-decode one tile on their own thread into a tile buffer and copy it out. 4096 x 8192 RGB in 1024 x 1024 tiles: 99.6 vs 135.7 ms on the planes at 18 threads; over-budget 8192^2 RGB / 16384^2 gray in 1024^2 tiles: 242 -> 189 ms, 1350 -> 470 ms. Gray (single-component band lookahead), HTJ2K (5-50% slower streamed), small images and few-tile images keep the planes. The test hook set_full_decode_planes_budget now fixes the budget and drops the default preference, so tests can force either route. New doc-hidden native API: Image::output_tile_rects, cpu_decode_pool_threads. --- crates/j2k-native/src/image/output_api.rs | 64 +++++ crates/j2k-native/src/lib.rs | 11 + crates/j2k/src/decode.rs | 7 +- crates/j2k/src/decode/route.rs | 212 ++++++++++++-- crates/j2k/src/decode/route/tests.rs | 261 +++++++++++++++++- crates/j2k/src/decode/route/tiles.rs | 131 +++++++++ crates/j2k/src/view.rs | 53 ++-- crates/j2k/tests/full_image_streaming.rs | 36 ++- ...able-api-1.0.implementation-public-api.txt | 2 + 9 files changed, 722 insertions(+), 55 deletions(-) create mode 100644 crates/j2k/src/decode/route/tiles.rs diff --git a/crates/j2k-native/src/image/output_api.rs b/crates/j2k-native/src/image/output_api.rs index f21cb324..f883c3cb 100644 --- a/crates/j2k-native/src/image/output_api.rs +++ b/crates/j2k-native/src/image/output_api.rs @@ -561,6 +561,70 @@ impl<'a> Image<'a> { ) } + /// The output rectangles `(x, y, width, height)` of the image's tiles at + /// this image's resolution, in raster order, leaving out tiles with no + /// output samples. A region decode of each rectangle decodes that tile + /// alone, and together they cover the image once. + /// + /// # Errors + /// + /// Returns an error when the header's sampling factors are invalid. + #[doc(hidden)] + pub fn output_tile_rects(&self) -> Result> { + let size = &self.header.size_data; + let (x_shrink, y_shrink) = ( + size.checked_x_shrink_factor()?, + size.checked_y_shrink_factor()?, + ); + let (x_origin, y_origin) = ( + size.image_area_x_offset.div_ceil(x_shrink), + size.image_area_y_offset.div_ceil(y_shrink), + ); + // One axis of tile `index` on the shrunk grid, relative to the image. + let span = |index: u32, + offset: u32, + tile: u32, + area: u32, + extent: u32, + shrink: u32, + origin: u32| { + let start = offset.saturating_add(index.saturating_mul(tile)).max(area); + let end = offset + .saturating_add(index.saturating_add(1).saturating_mul(tile)) + .min(extent); + let start = start.div_ceil(shrink).saturating_sub(origin); + let end = end.div_ceil(shrink).saturating_sub(origin); + (start, end.saturating_sub(start)) + }; + let mut rects = Vec::new(); + for ty in 0..size.num_y_tiles() { + let (y, height) = span( + ty, + size.tile_y_offset, + size.tile_height, + size.image_area_y_offset, + size.reference_grid_height, + y_shrink, + y_origin, + ); + for tx in 0..size.num_x_tiles() { + let (x, width) = span( + tx, + size.tile_x_offset, + size.tile_width, + size.image_area_x_offset, + size.reference_grid_width, + x_shrink, + x_origin, + ); + if width > 0 && height > 0 { + rects.push((x, y, width, height)); + } + } + } + Ok(rects) + } + /// Rounded-for-integer-output counterpart of /// [`Self::decode_components_with_ht_decoder`]. /// diff --git a/crates/j2k-native/src/lib.rs b/crates/j2k-native/src/lib.rs index 07ea9315..aca22e07 100644 --- a/crates/j2k-native/src/lib.rs +++ b/crates/j2k-native/src/lib.rs @@ -309,6 +309,17 @@ pub(crate) const MAX_J2K_TILE_COUNT: u64 = u16::MAX as u64 + 1; #[doc(hidden)] pub use j2k_types::{DEFAULT_MAX_CODEC_BYTES, DEFAULT_MAX_DECODE_BYTES}; +/// The threads a [`CpuDecodeParallelism::Auto`] decode runs on: the global +/// worker pool's size with the `parallel` feature, else one. +#[doc(hidden)] +#[must_use] +pub fn cpu_decode_pool_threads() -> usize { + #[cfg(feature = "parallel")] + return rayon::current_num_threads(); + #[cfg(not(feature = "parallel"))] + 1 +} + #[inline] pub(crate) fn checked_decode_usize_product2(left: usize, right: usize) -> Result { left.checked_mul(right) diff --git a/crates/j2k/src/decode.rs b/crates/j2k/src/decode.rs index 3196cd52..d9d35e00 100644 --- a/crates/j2k/src/decode.rs +++ b/crates/j2k/src/decode.rs @@ -19,7 +19,7 @@ use output::{ can_decode_u8_directly, expand_rgb8_to_rgba8, write_components_u16_output, write_components_u8_output, write_u16_output, write_u8_output, U8RowSink, }; -pub(crate) use route::{full_image_route, FullImageRoute}; +pub(crate) use route::{full_image_route, FullImageRoute, PlanesBudget, RouteRequest}; pub use settings::DecodeSettings; pub(crate) use srgb8::decode_image_srgb8; pub use srgb8::{J2kSrgb8Image, J2kSrgb8Layout}; @@ -72,7 +72,7 @@ pub(crate) fn decode_full_image_into_with_native_context<'a>( out: &mut [u8], stride: usize, fmt: PixelFormat, - route: FullImageRoute, + route: &FullImageRoute, ) -> Result<(), J2kError> { match route { FullImageRoute::Planes => { @@ -86,6 +86,9 @@ pub(crate) fn decode_full_image_into_with_native_context<'a>( fmt, Rect::full((image.width(), image.height())), ), + FullImageRoute::StreamedTiles { tiles, workers } => { + route::tiles::decode_tiles_into(image, tiles, *workers, (out, stride, fmt)) + } } } diff --git a/crates/j2k/src/decode/route.rs b/crates/j2k/src/decode/route.rs index 6bbb8c6e..692ac004 100644 --- a/crates/j2k/src/decode/route.rs +++ b/crates/j2k/src/decode/route.rs @@ -5,47 +5,215 @@ //! The planes route decodes every component into an image-sized plane of //! `f32` samples and packs the output from them. Its working memory grows //! with the image, and the native decoder refuses a decode whose planes and -//! largest tile's workspace pass its budget. Such an image instead runs as a -//! region decode of the whole image, which hands finished rows to the output -//! and keeps no planes (see -//! `Image::decode_region_rows_for_integer_output_with_context`). The native -//! decoder plans the planes route's charge from the codestream before -//! anything is decoded (see `Image::full_decode_planes_floor`), so a decode -//! never reruns after a failure, and an image the planes route can decode -//! keeps it, with its output and speed. +//! largest tile's workspace pass its budget. The streamed route runs the +//! decode as a region decode of the whole image, which hands finished rows +//! to the output and keeps no planes (see +//! `Image::decode_region_rows_for_integer_output_with_context`); an image of +//! several tiles runs it on several tiles at once (see [`tiles`]). The +//! native decoder plans the planes route's charge and whether the region +//! decode streams from the codestream before anything is decoded (see +//! `Image::full_decode_planes_floor`), so the choice is made once, before +//! the decode, and a decode never reruns after a failure. + +use j2k_core::{PixelFormat, Rect}; +use j2k_native::CpuDecodeParallelism; use crate::backend::Image; -use j2k_core::PixelFormat; + +pub(crate) mod tiles; /// How a full-image decode runs. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum FullImageRoute { /// Component planes for the whole image. Planes, /// A region decode of the whole image that streams its rows. StreamedRegion, + /// Streamed region decodes of one tile each, `workers` at once, each on + /// its worker's thread alone. + StreamedTiles { tiles: Vec, workers: usize }, +} + +impl FullImageRoute { + /// Whether the decode streams, holding no component planes. + pub(crate) fn streams(&self) -> bool { + !matches!(self, Self::Planes) + } } -/// How a full-image decode of `image` to `fmt` runs when its planes may -/// charge `planes_budget` bytes: as a streamed region when the output has -/// 8 bits per sample and the planes would pass the budget. +/// What a full-image decode's component planes may charge. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum PlanesBudget { + /// The native decode budget; an image within it still streams where + /// streaming is the faster route (see [`streams_by_default`]). + Default, + /// These bytes and no preference: the planes route unless the planes + /// would pass them. A test hook that forces either route. + Fixed(usize), +} + +/// The facts of a decode request the route depends on besides the image. +#[derive(Debug, Clone, Copy)] +pub(crate) struct RouteRequest { + pub(crate) fmt: PixelFormat, + pub(crate) budget: PlanesBudget, + pub(crate) parallelism: CpuDecodeParallelism, + /// The image's components. + pub(crate) components: u16, + /// Whether the image is decoded below full resolution. + pub(crate) reduced: bool, +} + +/// How a full-image decode of `image` runs for `request`: streamed when the +/// output has 8 bits per sample, the region decode streams, and either the +/// planes would pass the budget or the streamed route is the faster one +/// for this image. /// /// A planning error keeps the planes route, which meets the same error and /// reports it. -pub(crate) fn full_image_route( - image: &Image<'_>, - fmt: PixelFormat, - planes_budget: usize, -) -> FullImageRoute { +pub(crate) fn full_image_route(image: &Image<'_>, request: RouteRequest) -> FullImageRoute { if !matches!( - fmt, + request.fmt, PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Gray8 ) { return FullImageRoute::Planes; } - match image.full_decode_planes_floor(planes_budget) { - Ok(Some(floor)) if floor > planes_budget => FullImageRoute::StreamedRegion, - _ => FullImageRoute::Planes, + let planes_budget = match request.budget { + PlanesBudget::Default => j2k_native::DEFAULT_MAX_DECODE_BYTES, + PlanesBudget::Fixed(bytes) => bytes, + }; + // A budget of zero skips the planner's header-only shortcut, so the + // floor is known whenever the region decode would stream; the shortcut + // saves the tile parse when only the budget can choose streaming. + let streams = if request.budget == PlanesBudget::Default && streams_by_default(image, request) { + matches!(image.full_decode_planes_floor(0), Ok(Some(_))) + } else { + matches!( + image.full_decode_planes_floor(planes_budget), + Ok(Some(floor)) if floor > planes_budget + ) + }; + if streams { + streamed_route(image, request.fmt, request.parallelism) + } else { + FullImageRoute::Planes + } +} + +/// Pool threads from which a classic tile of several components +/// reconstructs each component on its own pool reader (`parallel_synthesis` +/// in the native decoder's pool bands). From there, a full-resolution +/// untiled image decodes 3-6% faster on the planes. +const CLASSIC_SYNTHESIS_THREADS: usize = 12; + +/// Whole-image samples from which a full-resolution untiled image streams: +/// smaller images spend more of their time setting up the pool bands. +const MIN_STREAMED_SAMPLES: u64 = 1 << 20; + +/// Whether an image the planes route can decode streams anyway, because +/// streaming was measured at least as fast for its kind: +/// - an RGB image of several tiles, when it has a tile for every thread +/// that decodes tiles at once (see [`tile_workers`]); +/// - an untiled three-component classic image when it is reduced, at any +/// thread count, or of a megapixel or more on 2 to 11 pool threads. +/// +/// Untiled gray images (whose streamed bands wait on a single component), +/// HTJ2K images (whose streamed IDWT and pack cost more per sample than the +/// planes' fused last level) and small images keep the planes. +fn streams_by_default(image: &Image<'_>, request: RouteRequest) -> bool { + let threads = pool_threads(request.parallelism); + if image.tile_count() > 1 { + return request.components == 3 + && image + .output_tile_rects() + .is_ok_and(|rects| tile_workers(&rects, threads).is_some()); + } + if request.components != 3 || image.uses_high_throughput_block_coding() { + return false; + } + if request.reduced { + return true; + } + let samples = u64::from(image.width()) * u64::from(image.height()); + samples >= MIN_STREAMED_SAMPLES && (2..CLASSIC_SYNTHESIS_THREADS).contains(&threads) +} + +/// The threads a decode with `parallelism` runs on. +fn pool_threads(parallelism: CpuDecodeParallelism) -> usize { + match parallelism { + CpuDecodeParallelism::Auto => j2k_native::cpu_decode_pool_threads(), + CpuDecodeParallelism::Serial => 1, + } +} + +/// Tiles from which an image decodes one tile after another rather than at +/// once: each tile's region decode parses the tile-part headers of the whole +/// image again, a cost that grows with the square of the tiles. +const MAX_TILES_AT_ONCE: usize = 1024; + +/// Output samples a tile needs on average for its own region decode to pay +/// for setting it up. +const MIN_SAMPLES_PER_TILE: usize = 256 * 256; + +/// Workers that decode the tiles at `rects` at once on `threads` pool +/// threads: the calling thread and one per pool thread, the threads a +/// streamed region decode runs on. Each worker decodes its tile alone, so +/// the image needs a tile for every worker, and its tiles must keep at least +/// 80% of the workers busy over the rounds they take: 16 tiles on 7 workers +/// take three rounds with a third of them idle, and decode no faster than +/// the planes. Other images decode one tile after another with the pool +/// behind each. +fn tile_workers(rects: &[(u32, u32, u32, u32)], threads: usize) -> Option { + let tiles = rects.len(); + let samples: usize = rects + .iter() + .map(|&(_, _, w, h)| w as usize * h as usize) + .sum(); + let workers = threads.checked_add(1)?; + let rounds = tiles.div_ceil(workers); + (threads >= 2 + && (workers..=MAX_TILES_AT_ONCE).contains(&tiles) + && samples >= tiles.saturating_mul(MIN_SAMPLES_PER_TILE) + && tiles.saturating_mul(5) >= rounds.saturating_mul(workers).saturating_mul(4)) + .then_some(workers) +} + +/// Bytes a tile worker needs per sample of its tile beside the tile's +/// output rows: a streamed decode of one tile on one thread. +const EST_STREAMED_BYTES_PER_TILE_SAMPLE: usize = 16; + +/// The streamed route for `image`: its tiles at once when it has a tile for +/// every worker and the workers' tiles fit in the native decode budget, else +/// one region decode of the whole image. +fn streamed_route( + image: &Image<'_>, + fmt: PixelFormat, + parallelism: CpuDecodeParallelism, +) -> FullImageRoute { + let Ok(rects) = image.output_tile_rects() else { + return FullImageRoute::StreamedRegion; + }; + let Some(workers) = tile_workers(&rects, pool_threads(parallelism)) else { + return FullImageRoute::StreamedRegion; + }; + let largest = rects + .iter() + .map(|&(_, _, w, h)| w as usize * h as usize) + .max() + .unwrap_or(0); + let per_worker = largest + .saturating_mul(EST_STREAMED_BYTES_PER_TILE_SAMPLE + fmt.bytes_per_pixel()) + .max(1); + let workers = workers.min(j2k_native::DEFAULT_MAX_DECODE_BYTES / per_worker); + if workers < 2 { + return FullImageRoute::StreamedRegion; + } + FullImageRoute::StreamedTiles { + tiles: rects + .into_iter() + .map(|(x, y, w, h)| Rect { x, y, w, h }) + .collect(), + workers, } } diff --git a/crates/j2k/src/decode/route/tests.rs b/crates/j2k/src/decode/route/tests.rs index e8f16c03..3106c93c 100644 --- a/crates/j2k/src/decode/route/tests.rs +++ b/crates/j2k/src/decode/route/tests.rs @@ -1,10 +1,11 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use super::{full_image_route, FullImageRoute}; +use super::{full_image_route, FullImageRoute, PlanesBudget, RouteRequest}; use crate::backend::image_at_scale; use crate::DecodeSettings; -use j2k_core::{Downscale, PixelFormat}; -use j2k_native::{encode, EncodeOptions}; +use j2k_core::{Downscale, PixelFormat, Rect}; +use j2k_native::DecoderContext; +use j2k_native::{encode, CpuDecodeParallelism, EncodeOptions}; fn codestream(components: u16, options: &EncodeOptions) -> Vec { let (width, height) = (96_u32, 80_u32); @@ -16,7 +17,17 @@ fn codestream(components: u16, options: &EncodeOptions) -> Vec { fn route(bytes: &[u8], scale: Downscale, fmt: PixelFormat, budget: usize) -> FullImageRoute { let image = image_at_scale(bytes, DecodeSettings::default(), scale).expect("image"); - full_image_route(&image, fmt, budget) + full_image_route(&image, request(fmt, budget, CpuDecodeParallelism::Serial)) +} + +fn request(fmt: PixelFormat, budget: usize, parallelism: CpuDecodeParallelism) -> RouteRequest { + RouteRequest { + fmt, + budget: PlanesBudget::Fixed(budget), + parallelism, + components: 3, + reduced: false, + } } /// The floor the native decoder plans for the planes of `bytes` at `scale`. @@ -110,6 +121,246 @@ fn tiles_with_empty_coarse_levels_keep_the_planes() { let image = image_at_scale(bytes, DecodeSettings::default(), Downscale::None).expect("image"); assert_eq!(image.full_decode_planes_floor(0).expect("plan"), None); - assert_eq!(full_image_route(&image, fmt, 0), FullImageRoute::Planes); + for parallelism in [CpuDecodeParallelism::Serial, CpuDecodeParallelism::Auto] { + assert_eq!( + full_image_route(&image, request(fmt, 0, parallelism)), + FullImageRoute::Planes + ); + } + } +} + +/// The output rectangles of an image's tiles cover it once, at every scale. +#[test] +fn tile_rects_cover_the_output_once() { + let bytes = codestream( + 3, + &EncodeOptions { + num_decomposition_levels: 3, + tile_size: Some((40, 24)), + ..EncodeOptions::default() + }, + ); + for scale in [ + Downscale::None, + Downscale::Half, + Downscale::Quarter, + Downscale::Eighth, + ] { + let image = image_at_scale(&bytes, DecodeSettings::default(), scale).expect("image"); + let (width, height) = (image.width(), image.height()); + let mut covered = vec![0_u8; width as usize * height as usize]; + let rects = image.output_tile_rects().expect("tile rects"); + assert!(rects.len() > 1, "{scale:?}"); + for (x, y, w, h) in rects { + for row in y..y + h { + for column in x..x + w { + covered[(row * width + column) as usize] += 1; + } + } + } + assert!(covered.iter().all(|&count| count == 1), "{scale:?}"); + } +} + +/// A tiled image that streams decodes its tiles on several workers when the +/// decode may use the pool and has a busy tile for each worker, and as one +/// region otherwise. +#[test] +fn tiled_images_stream_tiles_at_once_on_the_pool() { + let bytes = codestream( + 3, + &EncodeOptions { + num_decomposition_levels: 2, + tile_size: Some((8, 8)), + ..EncodeOptions::default() + }, + ); + let image = image_at_scale(&bytes, DecodeSettings::default(), Downscale::None).expect("image"); + let rects = image.output_tile_rects().expect("tile rects"); + assert_eq!(rects.len(), 120); + assert_eq!(rects[0], (0, 0, 8, 8)); + assert_eq!(rects[119], (88, 72, 8, 8)); + let serial = full_image_route( + &image, + request(PixelFormat::Rgb8, 0, CpuDecodeParallelism::Serial), + ); + assert_eq!(serial, FullImageRoute::StreamedRegion); + // 8 x 8 tiles are too small to decode on their own. + let auto = full_image_route( + &image, + request(PixelFormat::Rgb8, 0, CpuDecodeParallelism::Auto), + ); + assert_eq!(auto, FullImageRoute::StreamedRegion); +} + +/// Tiles at once take the calling thread and every pool thread, and need a +/// tile for each that keeps them busy, of a size worth its own decode. +#[test] +fn tiles_at_once_need_a_busy_tile_for_every_worker() { + let grid = |tiles: usize, side: u32| vec![(0, 0, side, side); tiles]; + let workers = |tiles, threads| super::tile_workers(&grid(tiles, 1024), threads); + assert_eq!(workers(32, 18), Some(19)); + assert_eq!(workers(256, 18), Some(19)); + assert_eq!(workers(32, 6), Some(7)); + // Fewer tiles than workers, or rounds that leave a fifth of them idle. + assert_eq!(workers(16, 18), None); + assert_eq!(workers(16, 6), None); + assert_eq!(workers(20, 18), None); + // One thread decodes tiles one after another. + assert_eq!(workers(100, 1), None); + // Small tiles, or so many that reparsing the tile headers dominates. + assert_eq!(super::tile_workers(&grid(32, 128), 6), None); + assert_eq!(super::tile_workers(&grid(32, 256), 6), Some(7)); + assert_eq!(workers(2048, 6), None); +} + +/// Without a fixed budget, reduced three-component classic images stream, +/// while HTJ2K, gray and small full-resolution images keep the planes. +#[test] +fn the_default_streams_where_streaming_measured_faster() { + let options = |use_ht_block_coding| EncodeOptions { + num_decomposition_levels: 3, + reversible: false, + use_ht_block_coding, + ..EncodeOptions::default() + }; + let classic = codestream(3, &options(false)); + let ht = codestream(3, &options(true)); + let gray = codestream(1, &options(false)); + let default = |bytes: &[u8], scale, fmt, components, parallelism| { + let image = image_at_scale(bytes, DecodeSettings::default(), scale).expect("image"); + full_image_route( + &image, + RouteRequest { + fmt, + budget: PlanesBudget::Default, + parallelism, + components, + reduced: scale != Downscale::None, + }, + ) + }; + for parallelism in [CpuDecodeParallelism::Serial, CpuDecodeParallelism::Auto] { + for scale in [Downscale::Half, Downscale::Quarter] { + for fmt in [PixelFormat::Rgb8, PixelFormat::Rgba8, PixelFormat::Gray8] { + assert!( + default(&classic, scale, fmt, 3, parallelism).streams(), + "{scale:?} {fmt:?} {parallelism:?}" + ); + assert_eq!( + default(&ht, scale, fmt, 3, parallelism), + FullImageRoute::Planes + ); + } + assert_eq!( + default(&gray, scale, PixelFormat::Gray8, 1, parallelism), + FullImageRoute::Planes + ); + } + // 96 x 80 samples are far below a megapixel. + assert_eq!( + default(&classic, Downscale::None, PixelFormat::Rgb8, 3, parallelism), + FullImageRoute::Planes + ); + assert_eq!( + default( + &classic, + Downscale::Half, + PixelFormat::Rgb16, + 3, + parallelism + ), + FullImageRoute::Planes + ); + } +} + +/// Decoding the tiles at once on several workers writes the bytes the planes +/// write, at every scale, into padded rows, and only the image's bytes. +#[test] +fn tiles_at_once_write_the_planes_bytes() { + let bytes = codestream( + 3, + &EncodeOptions { + num_decomposition_levels: 3, + reversible: false, + tile_size: Some((24, 16)), + ..EncodeOptions::default() + }, + ); + for scale in [Downscale::None, Downscale::Half, Downscale::Quarter] { + let image = image_at_scale(&bytes, DecodeSettings::default(), scale).expect("image"); + let tiles: Vec = image + .output_tile_rects() + .expect("tile rects") + .into_iter() + .map(|(x, y, w, h)| Rect { x, y, w, h }) + .collect(); + for fmt in [PixelFormat::Rgb8, PixelFormat::Rgba8] { + let row = image.width() as usize * fmt.bytes_per_pixel(); + let stride = row + 3; + let len = stride * image.height() as usize; + let mut planes = vec![0xab; len]; + crate::decode::decode_full_image_into_with_native_context( + &image, + &mut DecoderContext::default(), + &mut planes, + stride, + fmt, + &FullImageRoute::Planes, + ) + .expect("planes"); + for workers in [2, 3, 7] { + let mut streamed = vec![0xab; len]; + super::tiles::decode_tiles_into( + &image, + &tiles, + workers, + (&mut streamed, stride, fmt), + ) + .expect("tiles at once"); + assert!(streamed == planes, "{scale:?} {fmt:?} {workers}"); + } + } + } +} + +/// A codestream cut short fails the same way however many workers decode +/// its tiles: with the error of the first tile that fails. +#[test] +fn tiles_at_once_report_the_first_failing_tile() { + let bytes = codestream( + 3, + &EncodeOptions { + num_decomposition_levels: 2, + tile_size: Some((32, 32)), + ..EncodeOptions::default() + }, + ); + let truncated = &bytes[..bytes.len() * 3 / 4]; + let Ok(image) = image_at_scale(truncated, DecodeSettings::default(), Downscale::None) else { + return; + }; + let tiles: Vec = image + .output_tile_rects() + .expect("tile rects") + .into_iter() + .map(|(x, y, w, h)| Rect { x, y, w, h }) + .collect(); + let stride = image.width() as usize * 3; + let mut out = vec![0; stride * image.height() as usize]; + let one = + super::tiles::decode_tiles_into(&image, &tiles, 1, (&mut out, stride, PixelFormat::Rgb8)) + .map_err(|error| error.to_string()); + for workers in [2, 5] { + let many = super::tiles::decode_tiles_into( + &image, + &tiles, + workers, + (&mut out, stride, PixelFormat::Rgb8), + ) + .map_err(|error| error.to_string()); + assert_eq!(one, many, "{workers}"); } } diff --git a/crates/j2k/src/decode/route/tiles.rs b/crates/j2k/src/decode/route/tiles.rs new file mode 100644 index 00000000..6a8c3691 --- /dev/null +++ b/crates/j2k/src/decode/route/tiles.rs @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! The streamed full-image route for images of several tiles: workers +//! decode one tile each at a time, as a region decode of that tile into a +//! tile-sized buffer, and copy it into the output. +//! +//! A region decode of one tile decodes that tile alone, and tiles do not +//! share samples, so each output sample comes from exactly one tile decode +//! whichever worker runs it and whenever it finishes. A worker holds one +//! tile's decode buffers and its output rows, so the decode's memory grows +//! with the workers and the tile size, never with the image. + +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Mutex; + +use j2k_core::{PixelFormat, Rect}; +use j2k_native::CpuDecodeParallelism; + +use crate::backend::Image; +use crate::decode::decode_image_region_into_with_native_context; +use crate::J2kError; + +/// Decode the tiles at `tiles`, output rectangles that cover `image` once, +/// on `workers` threads into `out`: the calling thread and `workers - 1` +/// scoped ones. Each tile decodes serially on its worker's thread. +/// +/// The error is the one of the first tile, in raster order, that failed: +/// tiles are claimed in that order and every claimed tile finishes, so the +/// first failing tile is always among them. +pub(crate) fn decode_tiles_into( + image: &Image<'_>, + tiles: &[Rect], + workers: usize, + (out, stride, fmt): (&mut [u8], usize, PixelFormat), +) -> Result<(), J2kError> { + let next = AtomicUsize::new(0); + let failed = AtomicBool::new(false); + let first_error: Mutex> = Mutex::new(None); + let out = Mutex::new(out); + let work = || { + let mut native_context = j2k_native::DecoderContext::default(); + native_context.set_cpu_decode_parallelism(CpuDecodeParallelism::Serial); + let mut buffer = Vec::new(); + while !failed.load(Ordering::Relaxed) { + let index = next.fetch_add(1, Ordering::Relaxed); + let Some(&tile) = tiles.get(index) else { + break; + }; + let decoded = decode_tile(image, &mut native_context, &mut buffer, tile, fmt).and_then( + |tile_stride| { + let mut out = out + .lock() + .map_err(|_| J2kError::internal_backend("tile output lock poisoned"))?; + copy_tile( + &buffer[..tile_stride * tile.h as usize], + tile_stride, + &mut out, + stride, + tile, + fmt, + ); + Ok(()) + }, + ); + if let Err(error) = decoded { + failed.store(true, Ordering::Relaxed); + if let Ok(mut first) = first_error.lock() { + if first.as_ref().is_none_or(|(at, _)| index < *at) { + *first = Some((index, error)); + } + } + } + } + }; + std::thread::scope(|scope| { + for _ in 1..workers { + scope.spawn(work); + } + work(); + }); + match first_error.into_inner() { + Ok(None) => Ok(()), + Ok(Some((_, error))) => Err(error), + Err(_) => Err(J2kError::internal_backend("tile error lock poisoned")), + } +} + +/// Region-decode `tile` into the start of `buffer`, grown to fit it, and +/// return its stride. The decode writes every byte it covers, so what an +/// earlier tile left there is overwritten. +fn decode_tile<'a>( + image: &Image<'a>, + native_context: &mut j2k_native::DecoderContext<'a>, + buffer: &mut Vec, + tile: Rect, + fmt: PixelFormat, +) -> Result { + let tile_stride = tile.w as usize * fmt.bytes_per_pixel(); + let len = tile_stride * tile.h as usize; + if buffer.len() < len { + buffer + .try_reserve_exact(len - buffer.len()) + .map_err(|_| J2kError::internal_backend("tile output buffer allocation failed"))?; + buffer.resize(len, 0); + } + decode_image_region_into_with_native_context( + image, + native_context, + &mut buffer[..len], + tile_stride, + fmt, + tile, + )?; + Ok(tile_stride) +} + +/// Copy a decoded tile's rows from `buffer` into `out` at the tile. +fn copy_tile( + buffer: &[u8], + tile_stride: usize, + out: &mut [u8], + stride: usize, + tile: Rect, + fmt: PixelFormat, +) { + let x = tile.x as usize * fmt.bytes_per_pixel(); + for (row, source) in buffer.chunks_exact(tile_stride).enumerate() { + let start = (tile.y as usize + row) * stride + x; + out[start..start + tile_stride].copy_from_slice(source); + } +} diff --git a/crates/j2k/src/view.rs b/crates/j2k/src/view.rs index ea3f6fb8..bcbb0ccf 100644 --- a/crates/j2k/src/view.rs +++ b/crates/j2k/src/view.rs @@ -9,7 +9,8 @@ use crate::{ decode::{ decode_full_image_into_with_native_context, decode_image_region_into_with_native_context, decode_warnings_for_image, full_image_route, validate_buffer, validate_region, - FullImageRoute, J2kDecodeOutcome, J2kDecodedComponents, J2kDecodedNativeComponents, + J2kDecodeOutcome, J2kDecodedComponents, J2kDecodedNativeComponents, PlanesBudget, + RouteRequest, }, parse::{parse_image_info, parse_info}, scratch::J2kScratchPool, @@ -384,21 +385,30 @@ impl<'a> J2kDecoder<'a> { ) -> Result { validate_buffer(self.info.dimensions, out.len(), stride, fmt)?; self.ensure_image()?; - let planes_budget = self.full_decode_planes_budget(); + let budget = self.full_decode_planes_budget(); let (Some(image), native_context) = (self.image.as_ref(), &mut self.native_context) else { return Err(J2kError::internal_backend("internal image cache missing")); }; - let route = full_image_route(image, fmt, planes_budget); + let route = full_image_route( + image, + RouteRequest { + fmt, + budget, + parallelism: native_context.cpu_decode_parallelism(), + components: self.info.components, + reduced: false, + }, + ); let decoded = decode_full_image_into_with_native_context( image, native_context, out, stride, fmt, - route, + &route, ); let warnings = decode_warnings_for_image(image); - if route == FullImageRoute::StreamedRegion { + if route.streams() { // As after a region decode, release the decode buffers rather // than hold them while the decoder is idle. self.native_context = self.fresh_native_context(); @@ -492,14 +502,23 @@ impl<'a> J2kDecoder<'a> { let image_dims = (image.width(), image.height()); validate_buffer(image_dims, out.len(), stride, fmt)?; let mut native_context = self.fresh_native_context(); - let route = full_image_route(&image, fmt, self.full_decode_planes_budget()); + let route = full_image_route( + &image, + RouteRequest { + fmt, + budget: self.full_decode_planes_budget(), + parallelism: native_context.cpu_decode_parallelism(), + components: self.info.components, + reduced: true, + }, + ); decode_full_image_into_with_native_context( &image, &mut native_context, out, stride, fmt, - route, + &route, )?; Ok(j2k_core::DecodeOutcome::new( Rect::full(image_dims), @@ -567,19 +586,21 @@ impl<'a> J2kDecoder<'a> { .ok_or_else(|| J2kError::internal_backend("internal image cache missing")) } - /// The bytes a full-image 8-bit decode's component planes may charge - /// before it runs as a streamed region decode of the whole image - /// instead: the native decode budget, past which the planes decode is - /// refused. - fn full_decode_planes_budget(&self) -> usize { + /// What a full-image 8-bit decode's component planes may charge: the + /// native decode budget, past which the planes decode is refused, with + /// streaming wherever it is the faster route, unless a test set a fixed + /// budget. + fn full_decode_planes_budget(&self) -> PlanesBudget { self.full_decode_planes_budget - .unwrap_or(j2k_native::DEFAULT_MAX_DECODE_BYTES) + .map_or(PlanesBudget::Default, PlanesBudget::Fixed) } /// Test hook: hold full-image 8-bit decodes' component planes to - /// `bytes`, so that a small image takes the streamed region route that - /// only images past the native decode budget take otherwise. The - /// native decode budget itself is unchanged. + /// `bytes` and drop the preference for streaming, so that a decode + /// takes the streamed region route exactly when its planes would pass + /// `bytes`: 0 streams every decode that can stream, `usize::MAX` keeps + /// every decode the planes can finish on the planes. The native decode + /// budget itself is unchanged. #[doc(hidden)] pub fn set_full_decode_planes_budget(&mut self, bytes: usize) { self.full_decode_planes_budget = Some(bytes); diff --git a/crates/j2k/tests/full_image_streaming.rs b/crates/j2k/tests/full_image_streaming.rs index 4f9e2513..d026d9e2 100644 --- a/crates/j2k/tests/full_image_streaming.rs +++ b/crates/j2k/tests/full_image_streaming.rs @@ -1,10 +1,12 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 //! An 8-bit full-image decode whose component planes would pass the decode -//! budget runs as a region decode of the whole image that streams its rows. -//! A decoder whose planes budget is lowered takes that route for small -//! images, and must write the bytes the planes route writes, at every scale, -//! stride and parallelism, and fail where it fails. +//! budget, or for which streaming is the faster route, runs as a region +//! decode of the whole image that streams its rows (several tiles at once +//! when it has them and the pool to run them). A decoder whose planes budget +//! is fixed takes either route on demand, and the streamed route must write +//! the bytes the planes route writes, at every scale, stride and +//! parallelism, and fail where it fails. use j2k::{CpuDecodeParallelism, J2kDecoder, J2kScratchPool}; use j2k_core::{Downscale, PixelFormat}; @@ -91,6 +93,15 @@ fn streamed_full_image_matches_planes() { ..base.clone() }, ); + // Enough tiles for every worker of a large pool: decoded tiles at once. + let rgb_many_tiles = codestream( + 3, + &EncodeOptions { + reversible: false, + tile_size: Some((16, 16)), + ..base.clone() + }, + ); let gray = codestream( 1, &EncodeOptions { @@ -125,6 +136,7 @@ fn streamed_full_image_matches_planes() { let fixtures = [ ("rgb", rgb, rgb_formats), ("rgb tiled HT", rgb_tiled_ht, rgb_formats), + ("rgb many tiles", rgb_many_tiles, rgb_formats), ("gray", gray, gray_formats), ("gray tiled", gray_tiled, gray_formats), ("rgba", rgba, rgb_formats), @@ -137,13 +149,15 @@ fn streamed_full_image_matches_planes() { (CpuDecodeParallelism::Serial, 5), (CpuDecodeParallelism::Auto, 0), ] { - let planes = decode(bytes, None, parallelism, fmt, scale, padding); + let planes = decode(bytes, Some(usize::MAX), parallelism, fmt, scale, padding); let streamed = decode(bytes, Some(0), parallelism, fmt, scale, padding); + let default = decode(bytes, None, parallelism, fmt, scale, padding); assert!( - planes == streamed, - "{label} {fmt:?} {scale:?} {parallelism:?}: planes {} streamed {}", + planes == streamed && planes == default, + "{label} {fmt:?} {scale:?} {parallelism:?}: planes {} streamed {} default {}", planes.is_some(), - streamed.is_some() + streamed.is_some(), + default.is_some() ); decoded += usize::from(planes.is_some()); } @@ -151,7 +165,7 @@ fn streamed_full_image_matches_planes() { } } // Gray to RGB and RGB to gray are rejected; the rest must decode. - assert!(decoded >= 7 * 3 * 2, "{decoded} decodes succeeded"); + assert!(decoded >= 9 * 3 * 2, "{decoded} decodes succeeded"); } /// Tiles whose coarse resolutions are empty while finer ones are not (a @@ -171,10 +185,12 @@ fn tiles_with_empty_coarse_levels_keep_the_planes() { ] { for parallelism in [CpuDecodeParallelism::Serial, CpuDecodeParallelism::Auto] { for scale in [Downscale::None, Downscale::Half] { - let planes = decode(bytes, None, parallelism, fmt, scale, 0); + let planes = decode(bytes, Some(usize::MAX), parallelism, fmt, scale, 0); let lowered = decode(bytes, Some(0), parallelism, fmt, scale, 0); + let default = decode(bytes, None, parallelism, fmt, scale, 0); assert!(planes.is_some(), "{label} {parallelism:?} {scale:?}"); assert_eq!(planes, lowered, "{label} {parallelism:?} {scale:?}"); + assert_eq!(planes, default, "{label} {parallelism:?} {scale:?}"); } } } diff --git a/xtask/api/stable-api-1.0.implementation-public-api.txt b/xtask/api/stable-api-1.0.implementation-public-api.txt index 3e2e1341..e51766e8 100644 --- a/xtask/api/stable-api-1.0.implementation-public-api.txt +++ b/xtask/api/stable-api-1.0.implementation-public-api.txt @@ -2511,6 +2511,7 @@ pub fn j2k_native::Image<'a>::decode_reversible_53_coefficients_with_context(&se pub fn j2k_native::Image<'a>::full_decode_planes_floor(&self, usize) -> j2k_native::Result> pub fn j2k_native::Image<'a>::new_with_reduction(&'a [u8], &j2k_native::DecodeSettings, u8) -> j2k_native::Result pub fn j2k_native::Image<'a>::new_with_retained_baseline(&'a [u8], &j2k_native::DecodeSettings, usize) -> j2k_native::Result +pub fn j2k_native::Image<'a>::output_tile_rects(&self) -> j2k_native::Result> pub fn j2k_native::Image<'a>::prepare_region_decoder_with_context<'image, 'context>(&'image self, &'context mut j2k_native::DecoderContext<'a>) -> j2k_native::Result> pub fn j2k_native::Image<'a>::primary_icc_profile(&self) -> core::option::Option<&[u8]> pub fn j2k_native::Image<'a>::retained_allocation_bytes(&self) -> j2k_native::Result @@ -2546,6 +2547,7 @@ pub fn j2k_native::ResidentHtj2kEncodeError::fmt(&self, &mut core::fmt::Formatte pub fn j2k_native::ResidentHtj2kEncodeError::source(&self) -> core::option::Option<&(dyn core::error::Error + 'static)> pub fn j2k_native::Reversible53CoefficientImage::encode_htj2k(&self, &j2k_native::EncodeOptions) -> j2k_native::EncodeResult> pub fn j2k_native::collect_ht_cleanup_encode_distribution(&[i32], u32, u32, u8) -> j2k_native::EncodeResult +pub fn j2k_native::cpu_decode_pool_threads() -> usize pub fn j2k_native::decode_ht_code_block_scalar(j2k_native::HtCodeBlockDecodeJob<'_>, &mut [f32]) -> j2k_native::Result<()> pub fn j2k_native::decode_ht_code_block_scalar_until_phase(j2k_native::HtCodeBlockDecodeJob<'_>, &mut [f32], j2k_native::HtCodeBlockDecodePhaseLimit) -> j2k_native::Result<()> pub fn j2k_native::decode_ht_code_block_scalar_with_workspace(j2k_native::HtCodeBlockDecodeJob<'_>, &mut [f32], &mut j2k_native::HtCodeBlockDecodeWorkspace) -> j2k_native::Result<()> From 9a0a83986abb1e3196dd16bb46a1da89b9d7eeb8 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 04:55:05 -0400 Subject: [PATCH 22/94] perf(j2k): stream reduced HTJ2K decodes and keep low-rate classic images on the planes Refines the default full-image route from measurements on an 18-core ARM64 machine (interleaved, warm / cold medians): - reduced-resolution decodes of untiled three-component HTJ2K images now stream on two pool threads or more: a 4096 x 4096 HTJ2K RGB image at 1/2 and 1/4 takes 21% and 47% less time warm on 6 threads (cold 48% and 70% less), 26% and 53% less on the default pool; the 4096 x 2048 HTJ2K mosaic at 2 bpp at 1/2 and 1/4 9% and 35% less on 6 threads. One thread keeps the planes (1/2 is 2-9% faster there). - full-resolution untiled classic images stream on 2 to 11 pool threads only from 4 coded bits per pixel (coded bytes = the decoder's input): the 2.1 bits-per-pixel mosaic was 1-3% slower streamed on 6 threads (32.6 / 32.7 ms on two baseline builds, 33.7 ms streamed), while the 5.3-10.6 bits-per-pixel images stay 10-17% faster. - a tile worker thread the system refuses to spawn leaves fewer workers instead of panicking. Byte checks over 292 benchmark rows and 62 workload decodes (the 5661 x 8287 full decode included) pass: output identical to that of the planes route and to the reference hashes. --- crates/j2k/src/decode/route.rs | 47 +++++++++++++++++++++++----- crates/j2k/src/decode/route/tests.rs | 45 +++++++++++++++++++++----- crates/j2k/src/decode/route/tiles.rs | 9 +++++- crates/j2k/src/view.rs | 2 ++ 4 files changed, 86 insertions(+), 17 deletions(-) diff --git a/crates/j2k/src/decode/route.rs b/crates/j2k/src/decode/route.rs index 692ac004..350c25ca 100644 --- a/crates/j2k/src/decode/route.rs +++ b/crates/j2k/src/decode/route.rs @@ -60,6 +60,8 @@ pub(crate) struct RouteRequest { pub(crate) parallelism: CpuDecodeParallelism, /// The image's components. pub(crate) components: u16, + /// The bytes of the coded image, its codestream and container. + pub(crate) coded_bytes: usize, /// Whether the image is decoded below full resolution. pub(crate) reduced: bool, } @@ -110,16 +112,27 @@ const CLASSIC_SYNTHESIS_THREADS: usize = 12; /// smaller images spend more of their time setting up the pool bands. const MIN_STREAMED_SAMPLES: u64 = 1 << 20; +/// Coded bits per sample from which a full-resolution untiled classic image +/// streams: below it Tier-1 is light, the decode is bound by the inverse +/// transforms and the pack, and the planes' fused last level does those +/// faster (a 2 bits-per-pixel image is 0-3% slower streamed, 5-11 bits per +/// pixel images are 10% faster). +const MIN_STREAMED_CODED_BITS_PER_SAMPLE: u64 = 4; + /// Whether an image the planes route can decode streams anyway, because /// streaming was measured at least as fast for its kind: /// - an RGB image of several tiles, when it has a tile for every thread /// that decodes tiles at once (see [`tile_workers`]); -/// - an untiled three-component classic image when it is reduced, at any -/// thread count, or of a megapixel or more on 2 to 11 pool threads. +/// - an untiled three-component image when it is reduced: classic at any +/// thread count, HTJ2K on two threads or more (one thread decodes a half +/// resolution HTJ2K image 2-9% faster on the planes); +/// - an untiled three-component classic image of a megapixel or more and at +/// least 4 coded bits per pixel on 2 to 11 pool threads. /// /// Untiled gray images (whose streamed bands wait on a single component), -/// HTJ2K images (whose streamed IDWT and pack cost more per sample than the -/// planes' fused last level) and small images keep the planes. +/// full-resolution HTJ2K images (whose streamed IDWT and pack cost more per +/// sample than the planes' fused last level) and small images keep the +/// planes. fn streams_by_default(image: &Image<'_>, request: RouteRequest) -> bool { let threads = pool_threads(request.parallelism); if image.tile_count() > 1 { @@ -128,14 +141,32 @@ fn streams_by_default(image: &Image<'_>, request: RouteRequest) -> bool { .output_tile_rects() .is_ok_and(|rects| tile_workers(&rects, threads).is_some()); } - if request.components != 3 || image.uses_high_throughput_block_coding() { + if request.components != 3 { return false; } + let htj2k = image.uses_high_throughput_block_coding(); if request.reduced { - return true; + return !htj2k || threads >= 2; + } + if htj2k { + return false; } - let samples = u64::from(image.width()) * u64::from(image.height()); - samples >= MIN_STREAMED_SAMPLES && (2..CLASSIC_SYNTHESIS_THREADS).contains(&threads) + full_resolution_streams( + u64::from(image.width()) * u64::from(image.height()), + request.coded_bytes, + threads, + ) +} + +/// Whether a full-resolution untiled three-component classic image of +/// `samples` pixels coded in `coded_bytes` streams on `threads` pool threads. +fn full_resolution_streams(samples: u64, coded_bytes: usize, threads: usize) -> bool { + let coded_bits = u64::try_from(coded_bytes) + .unwrap_or(u64::MAX) + .saturating_mul(8); + samples >= MIN_STREAMED_SAMPLES + && coded_bits >= samples.saturating_mul(MIN_STREAMED_CODED_BITS_PER_SAMPLE) + && (2..CLASSIC_SYNTHESIS_THREADS).contains(&threads) } /// The threads a decode with `parallelism` runs on. diff --git a/crates/j2k/src/decode/route/tests.rs b/crates/j2k/src/decode/route/tests.rs index 3106c93c..239ebc0c 100644 --- a/crates/j2k/src/decode/route/tests.rs +++ b/crates/j2k/src/decode/route/tests.rs @@ -26,6 +26,7 @@ fn request(fmt: PixelFormat, budget: usize, parallelism: CpuDecodeParallelism) - budget: PlanesBudget::Fixed(budget), parallelism, components: 3, + coded_bytes: 0, reduced: false, } } @@ -215,8 +216,9 @@ fn tiles_at_once_need_a_busy_tile_for_every_worker() { assert_eq!(workers(2048, 6), None); } -/// Without a fixed budget, reduced three-component classic images stream, -/// while HTJ2K, gray and small full-resolution images keep the planes. +/// Without a fixed budget, reduced three-component images stream (HTJ2K +/// ones on a pool), while gray and small full-resolution images keep the +/// planes. #[test] fn the_default_streams_where_streaming_measured_faster() { let options = |use_ht_block_coding| EncodeOptions { @@ -237,6 +239,7 @@ fn the_default_streams_where_streaming_measured_faster() { budget: PlanesBudget::Default, parallelism, components, + coded_bytes: bytes.len(), reduced: scale != Downscale::None, }, ) @@ -248,9 +251,12 @@ fn the_default_streams_where_streaming_measured_faster() { default(&classic, scale, fmt, 3, parallelism).streams(), "{scale:?} {fmt:?} {parallelism:?}" ); + let pool = parallelism == CpuDecodeParallelism::Auto + && j2k_native::cpu_decode_pool_threads() >= 2; assert_eq!( - default(&ht, scale, fmt, 3, parallelism), - FullImageRoute::Planes + default(&ht, scale, fmt, 3, parallelism).streams(), + pool, + "{scale:?} {fmt:?} {parallelism:?}" ); } assert_eq!( @@ -259,10 +265,12 @@ fn the_default_streams_where_streaming_measured_faster() { ); } // 96 x 80 samples are far below a megapixel. - assert_eq!( - default(&classic, Downscale::None, PixelFormat::Rgb8, 3, parallelism), - FullImageRoute::Planes - ); + for bytes in [classic.as_slice(), ht.as_slice()] { + assert_eq!( + default(bytes, Downscale::None, PixelFormat::Rgb8, 3, parallelism), + FullImageRoute::Planes + ); + } assert_eq!( default( &classic, @@ -364,3 +372,24 @@ fn tiles_at_once_report_the_first_failing_tile() { assert_eq!(one, many, "{workers}"); } } + +/// Full-resolution untiled classic images stream from a megapixel and 4 coded +/// bits per pixel, on 2 to 11 pool threads. +#[test] +fn full_resolution_images_stream_when_tier1_bound_on_a_mid_sized_pool() { + use super::full_resolution_streams; + let mosaic = 4096 * 2048; + // 11 MB lossless and 2.2 MB at 2 bits per pixel. + assert!(full_resolution_streams(mosaic, 11_104_128, 6)); + assert!(!full_resolution_streams(mosaic, 2_246_803, 6)); + for threads in [1, 12, 18] { + assert!( + !full_resolution_streams(mosaic, 11_104_128, threads), + "{threads}" + ); + } + assert!(full_resolution_streams(mosaic, 11_104_128, 2)); + assert!(full_resolution_streams(mosaic, 11_104_128, 11)); + // Below a megapixel. + assert!(!full_resolution_streams(1000 * 1000, 2_000_000, 6)); +} diff --git a/crates/j2k/src/decode/route/tiles.rs b/crates/j2k/src/decode/route/tiles.rs index 6a8c3691..427ccbc8 100644 --- a/crates/j2k/src/decode/route/tiles.rs +++ b/crates/j2k/src/decode/route/tiles.rs @@ -74,7 +74,14 @@ pub(crate) fn decode_tiles_into( }; std::thread::scope(|scope| { for _ in 1..workers { - scope.spawn(work); + // A thread the system refuses only leaves fewer workers: the + // calling thread claims every tile the others do not. + if std::thread::Builder::new() + .spawn_scoped(scope, work) + .is_err() + { + break; + } } work(); }); diff --git a/crates/j2k/src/view.rs b/crates/j2k/src/view.rs index bcbb0ccf..b546a72a 100644 --- a/crates/j2k/src/view.rs +++ b/crates/j2k/src/view.rs @@ -396,6 +396,7 @@ impl<'a> J2kDecoder<'a> { budget, parallelism: native_context.cpu_decode_parallelism(), components: self.info.components, + coded_bytes: self.bytes.len(), reduced: false, }, ); @@ -509,6 +510,7 @@ impl<'a> J2kDecoder<'a> { budget: self.full_decode_planes_budget(), parallelism: native_context.cpu_decode_parallelism(), components: self.info.components, + coded_bytes: self.bytes.len(), reduced: true, }, ); From e7c5e0fb8c0d63bac945b6b3da44ea5bf6a74cb8 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 05:04:57 -0400 Subject: [PATCH 23/94] perf(j2k): stream tiled classic images tile by tile when their tiles qualify Two refinements of the default route for tiled images, measured on an 18-core ARM64 machine (warm / cold ms, footprint MB; bytes identical to the planes route and to the reference hashes everywhere): - a full-resolution tiled classic RGB image whose tiles do not decode at once (too few tiles for the workers) now streams them one after another when each tile meets the untiled rule as an image of its own (a megapixel or more, 4 coded bits per pixel or more, 2 to 11 pool threads): 4096 x 4096 RGB in 16 tiles of 1024 x 1024 on 6 threads 116.9 vs 124.9 ms on the planes (cold 118.4 vs 146.6), 70 vs 315 MB; smaller tiles keep the planes (one after another was 40% slower on 512 x 512 HTJ2K tiles). - reduced-resolution tiled images keep the planes: one tile after another was 1-13% slower on 6 threads and 1.5-10.6 times on 18, and tiles at once were never measured reduced (the default used them). Tiles at once stay wherever every worker gets a busy tile, measured with the caller + T scoped workers: 4096 x 8192 RGB in 32 tiles 219.1 vs 237.9 ms on the planes with 6 threads and 103.5 vs 134.8 with 18 (cold 218.0 vs 253.7 and 103.6 vs 152.6), 158 vs 564 MB; 4096 x 2048 HTJ2K in 32 tiles 7.59 vs 9.19 ms (6) and 4.61 vs 5.25 ms (18). Every full-image benchmark row is a one-tile image: their routes do not change. --- crates/j2k/src/decode/route.rs | 73 ++++++++++++++-------------- crates/j2k/src/decode/route/tests.rs | 21 ++++++-- 2 files changed, 53 insertions(+), 41 deletions(-) diff --git a/crates/j2k/src/decode/route.rs b/crates/j2k/src/decode/route.rs index 350c25ca..b930df9f 100644 --- a/crates/j2k/src/decode/route.rs +++ b/crates/j2k/src/decode/route.rs @@ -96,7 +96,7 @@ pub(crate) fn full_image_route(image: &Image<'_>, request: RouteRequest) -> Full ) }; if streams { - streamed_route(image, request.fmt, request.parallelism) + streamed_route(image, request) } else { FullImageRoute::Planes } @@ -119,46 +119,51 @@ const MIN_STREAMED_SAMPLES: u64 = 1 << 20; /// pixel images are 10% faster). const MIN_STREAMED_CODED_BITS_PER_SAMPLE: u64 = 4; -/// Whether an image the planes route can decode streams anyway, because -/// streaming was measured at least as fast for its kind: -/// - an RGB image of several tiles, when it has a tile for every thread -/// that decodes tiles at once (see [`tile_workers`]); -/// - an untiled three-component image when it is reduced: classic at any -/// thread count, HTJ2K on two threads or more (one thread decodes a half -/// resolution HTJ2K image 2-9% faster on the planes); -/// - an untiled three-component classic image of a megapixel or more and at -/// least 4 coded bits per pixel on 2 to 11 pool threads. +/// Whether a three-component image the planes route can decode streams +/// anyway, because streaming was measured at least as fast for its kind: +/// - a full-resolution tiled image whose tiles decode at once (see +/// [`tile_workers`]); +/// - an untiled reduced image: classic at any thread count, HTJ2K on two +/// threads or more (one thread decodes a half resolution HTJ2K image 2-9% +/// faster on the planes); +/// - a full-resolution classic image of a megapixel or more and at least 4 +/// coded bits per pixel on 2 to 11 pool threads; a tiled one whose tiles +/// do not decode at once streams them one after another, so each of its +/// tiles must meet that as an image of its own (4096 x 4096 RGB in 16 +/// tiles of 1024 x 1024: 116.9 vs 124.9 ms on the planes with 6 threads). /// -/// Untiled gray images (whose streamed bands wait on a single component), +/// Gray images (whose streamed bands wait on a single component), /// full-resolution HTJ2K images (whose streamed IDWT and pack cost more per -/// sample than the planes' fused last level) and small images keep the -/// planes. +/// sample than the planes' fused last level), small images and reduced +/// tiled images (one tile after another was 1-13% slower on 6 threads, +/// 1.5-10 times on 18) keep the planes. fn streams_by_default(image: &Image<'_>, request: RouteRequest) -> bool { - let threads = pool_threads(request.parallelism); - if image.tile_count() > 1 { - return request.components == 3 - && image - .output_tile_rects() - .is_ok_and(|rects| tile_workers(&rects, threads).is_some()); - } if request.components != 3 { return false; } + let threads = pool_threads(request.parallelism); let htj2k = image.uses_high_throughput_block_coding(); + let tiled = image.tile_count() > 1; if request.reduced { - return !htj2k || threads >= 2; + return !tiled && (!htj2k || threads >= 2); } - if htj2k { - return false; + let mut samples = u64::from(image.width()) * u64::from(image.height()); + let mut coded_bytes = request.coded_bytes; + if tiled { + let Ok(rects) = image.output_tile_rects() else { + return false; + }; + if tile_workers(&rects, threads).is_some() { + return true; + } + let tiles = rects.len().max(1); + samples /= u64::try_from(tiles).unwrap_or(u64::MAX); + coded_bytes /= tiles; } - full_resolution_streams( - u64::from(image.width()) * u64::from(image.height()), - request.coded_bytes, - threads, - ) + !htj2k && full_resolution_streams(samples, coded_bytes, threads) } -/// Whether a full-resolution untiled three-component classic image of +/// Whether a full-resolution three-component classic image (or tile) of /// `samples` pixels coded in `coded_bytes` streams on `threads` pool threads. fn full_resolution_streams(samples: u64, coded_bytes: usize, threads: usize) -> bool { let coded_bits = u64::try_from(coded_bytes) @@ -216,15 +221,11 @@ const EST_STREAMED_BYTES_PER_TILE_SAMPLE: usize = 16; /// The streamed route for `image`: its tiles at once when it has a tile for /// every worker and the workers' tiles fit in the native decode budget, else /// one region decode of the whole image. -fn streamed_route( - image: &Image<'_>, - fmt: PixelFormat, - parallelism: CpuDecodeParallelism, -) -> FullImageRoute { +fn streamed_route(image: &Image<'_>, request: RouteRequest) -> FullImageRoute { let Ok(rects) = image.output_tile_rects() else { return FullImageRoute::StreamedRegion; }; - let Some(workers) = tile_workers(&rects, pool_threads(parallelism)) else { + let Some(workers) = tile_workers(&rects, pool_threads(request.parallelism)) else { return FullImageRoute::StreamedRegion; }; let largest = rects @@ -233,7 +234,7 @@ fn streamed_route( .max() .unwrap_or(0); let per_worker = largest - .saturating_mul(EST_STREAMED_BYTES_PER_TILE_SAMPLE + fmt.bytes_per_pixel()) + .saturating_mul(EST_STREAMED_BYTES_PER_TILE_SAMPLE + request.fmt.bytes_per_pixel()) .max(1); let workers = workers.min(j2k_native::DEFAULT_MAX_DECODE_BYTES / per_worker); if workers < 2 { diff --git a/crates/j2k/src/decode/route/tests.rs b/crates/j2k/src/decode/route/tests.rs index 239ebc0c..291a4a9a 100644 --- a/crates/j2k/src/decode/route/tests.rs +++ b/crates/j2k/src/decode/route/tests.rs @@ -216,9 +216,9 @@ fn tiles_at_once_need_a_busy_tile_for_every_worker() { assert_eq!(workers(2048, 6), None); } -/// Without a fixed budget, reduced three-component images stream (HTJ2K -/// ones on a pool), while gray and small full-resolution images keep the -/// planes. +/// Without a fixed budget, reduced untiled three-component images stream +/// (HTJ2K ones on a pool), while gray, reduced tiled and small +/// full-resolution images keep the planes. #[test] fn the_default_streams_where_streaming_measured_faster() { let options = |use_ht_block_coding| EncodeOptions { @@ -230,6 +230,13 @@ fn the_default_streams_where_streaming_measured_faster() { let classic = codestream(3, &options(false)); let ht = codestream(3, &options(true)); let gray = codestream(1, &options(false)); + let tiled = codestream( + 3, + &EncodeOptions { + tile_size: Some((48, 40)), + ..options(false) + }, + ); let default = |bytes: &[u8], scale, fmt, components, parallelism| { let image = image_at_scale(bytes, DecodeSettings::default(), scale).expect("image"); full_image_route( @@ -263,9 +270,13 @@ fn the_default_streams_where_streaming_measured_faster() { default(&gray, scale, PixelFormat::Gray8, 1, parallelism), FullImageRoute::Planes ); + assert_eq!( + default(&tiled, scale, PixelFormat::Rgb8, 3, parallelism), + FullImageRoute::Planes + ); } - // 96 x 80 samples are far below a megapixel. - for bytes in [classic.as_slice(), ht.as_slice()] { + // 96 x 80 samples are far below a megapixel, tiled or not. + for bytes in [classic.as_slice(), ht.as_slice(), tiled.as_slice()] { assert_eq!( default(bytes, Downscale::None, PixelFormat::Rgb8, 3, parallelism), FullImageRoute::Planes From a31a60377a84b39344db3213a4e30a5ad1fba0ee Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 05:48:40 -0400 Subject: [PATCH 24/94] perf(j2k): route small images by their measured streamed speed The default route treated every reduced untiled classic decode alike and kept every full-resolution image under a megapixel on the planes. Measured on small images (an 18-core ARM64 machine, warm ms ratio streamed / planes, bytes identical), both are wrong in places: - small reduced classic decodes on 12 pool threads or more run on the pool's bands as slowly as on one thread: a 384 x 256 output of the 768 x 512 lossless images 4.86 times as long as on the planes on 18 threads, 512 x 512 images at 1/2 3.6-3.8 times; on 6 threads they are 1-12% faster. Outputs from 1024 x 512 up are 6-53% faster on 18 threads. Small HTJ2K outputs are up to 59% slower streamed. Reduced decodes now stream from 2^19 output samples as before, and below it only classic ones from 256 x 256 on 2 to 11 pool threads (256 x 256 and 384 x 256 outputs 3-12% faster on 6 threads, a 192 x 128 one 1-3% slower). - full-resolution classic images of 4 coded bits per pixel or more stream faster below a megapixel too: 768 x 512 (9.6 bpp) 0.90 on 6 threads and 0.91 on 18, 512 x 512 (6.6 bpp) 0.95 and 0.86-0.97, while a 4096 x 1024 one is 6-7% slower on 18. They now stream from 512 x 512 on 2 to 11 pool threads, and below 2^19 samples on larger pools. Tiles decoded one after another still need a megapixel each (measured on 1024 x 1024 tiles only). Full-image benchmark rows (4096 x 2048 and larger, outputs of 1024 x 512 and more) keep their routes. --- crates/j2k/src/decode/route.rs | 77 ++++++++++++++++++++++------ crates/j2k/src/decode/route/tests.rs | 68 +++++++++++++++++++----- 2 files changed, 116 insertions(+), 29 deletions(-) diff --git a/crates/j2k/src/decode/route.rs b/crates/j2k/src/decode/route.rs index b930df9f..06161bf8 100644 --- a/crates/j2k/src/decode/route.rs +++ b/crates/j2k/src/decode/route.rs @@ -108,9 +108,34 @@ pub(crate) fn full_image_route(image: &Image<'_>, request: RouteRequest) -> Full /// untiled image decodes 3-6% faster on the planes. const CLASSIC_SYNTHESIS_THREADS: usize = 12; -/// Whole-image samples from which a full-resolution untiled image streams: -/// smaller images spend more of their time setting up the pool bands. -const MIN_STREAMED_SAMPLES: u64 = 1 << 20; +/// Samples from which a full-resolution classic image streams on 2 to 11 +/// pool threads (a 512 x 512 image at 6.6 coded bits per pixel and 768 x 512 +/// ones at 9.6 decode 5-10% faster streamed on 6 threads); smaller images +/// spend more of their time setting up the pool bands. +const MIN_STREAMED_SAMPLES: u64 = 1 << 18; + +/// Samples below which a full-resolution classic image also streams from +/// [`CLASSIC_SYNTHESIS_THREADS`] pool threads: the 512 x 512 and 768 x 512 +/// images decode 3-14% faster streamed on 18 threads, a 4096 x 1024 one +/// 6-7% slower. +const MAX_STREAMED_SAMPLES_ON_A_LARGE_POOL: u64 = 1 << 19; + +/// Samples a tile needs to stream one after another with the pool behind +/// it (measured on 1024 x 1024 tiles only). +const MIN_STREAMED_TILE_SAMPLES: u64 = 1 << 20; + +/// Output samples from which a reduced-resolution decode streams on any +/// pool (a 1024 x 512 output and larger: 6-53% faster on 18 threads). On 12 +/// pool threads or more a smaller classic output decodes on the pool's +/// bands as slowly as on one thread (a 384 x 256 output of a 768 x 512 +/// image 4.9 times as long as on the planes), and smaller HTJ2K outputs are +/// up to 59% slower streamed on any pool. +const MIN_REDUCED_STREAMED_SAMPLES: u64 = 1 << 19; + +/// Output samples from which a smaller reduced classic decode streams on 2 +/// to 11 pool threads: 256 x 256 and 384 x 256 outputs decode 3-12% faster +/// streamed on 6 threads, a 192 x 128 one 1-3% slower. +const MIN_SMALL_REDUCED_STREAMED_SAMPLES: u64 = 1 << 16; /// Coded bits per sample from which a full-resolution untiled classic image /// streams: below it Tier-1 is light, the decode is bound by the inverse @@ -123,14 +148,17 @@ const MIN_STREAMED_CODED_BITS_PER_SAMPLE: u64 = 4; /// anyway, because streaming was measured at least as fast for its kind: /// - a full-resolution tiled image whose tiles decode at once (see /// [`tile_workers`]); -/// - an untiled reduced image: classic at any thread count, HTJ2K on two -/// threads or more (one thread decodes a half resolution HTJ2K image 2-9% -/// faster on the planes); -/// - a full-resolution classic image of a megapixel or more and at least 4 -/// coded bits per pixel on 2 to 11 pool threads; a tiled one whose tiles -/// do not decode at once streams them one after another, so each of its -/// tiles must meet that as an image of its own (4096 x 4096 RGB in 16 -/// tiles of 1024 x 1024: 116.9 vs 124.9 ms on the planes with 6 threads). +/// - an untiled reduced image of [`MIN_REDUCED_STREAMED_SAMPLES`] output +/// samples or more: classic at any thread count, HTJ2K on two threads or +/// more (one thread decodes a half resolution HTJ2K image 2-9% faster on +/// the planes); a smaller classic one of +/// [`MIN_SMALL_REDUCED_STREAMED_SAMPLES`] or more on 2 to 11 pool threads; +/// - a full-resolution classic image of at least 4 coded bits per pixel (see +/// [`full_resolution_streams`]); a tiled one whose tiles do not decode at +/// once streams them one after another, so each of its tiles must meet +/// that as an image of its own, of [`MIN_STREAMED_TILE_SAMPLES`] or more +/// (4096 x 4096 RGB in 16 tiles of 1024 x 1024: 116.9 vs 124.9 ms on the +/// planes with 6 threads). /// /// Gray images (whose streamed bands wait on a single component), /// full-resolution HTJ2K images (whose streamed IDWT and pack cost more per @@ -144,10 +172,10 @@ fn streams_by_default(image: &Image<'_>, request: RouteRequest) -> bool { let threads = pool_threads(request.parallelism); let htj2k = image.uses_high_throughput_block_coding(); let tiled = image.tile_count() > 1; + let mut samples = u64::from(image.width()) * u64::from(image.height()); if request.reduced { - return !tiled && (!htj2k || threads >= 2); + return !tiled && reduced_streams(samples, htj2k, threads); } - let mut samples = u64::from(image.width()) * u64::from(image.height()); let mut coded_bytes = request.coded_bytes; if tiled { let Ok(rects) = image.output_tile_rects() else { @@ -159,19 +187,38 @@ fn streams_by_default(image: &Image<'_>, request: RouteRequest) -> bool { let tiles = rects.len().max(1); samples /= u64::try_from(tiles).unwrap_or(u64::MAX); coded_bytes /= tiles; + if samples < MIN_STREAMED_TILE_SAMPLES { + return false; + } } !htj2k && full_resolution_streams(samples, coded_bytes, threads) } +/// Whether a reduced-resolution decode of an untiled three-component image +/// to `samples` output samples streams on `threads` pool threads. +fn reduced_streams(samples: u64, htj2k: bool, threads: usize) -> bool { + if samples >= MIN_REDUCED_STREAMED_SAMPLES { + !htj2k || threads >= 2 + } else { + !htj2k + && samples >= MIN_SMALL_REDUCED_STREAMED_SAMPLES + && (2..CLASSIC_SYNTHESIS_THREADS).contains(&threads) + } +} + /// Whether a full-resolution three-component classic image (or tile) of -/// `samples` pixels coded in `coded_bytes` streams on `threads` pool threads. +/// `samples` pixels coded in `coded_bytes` streams on `threads` pool threads: +/// from [`MIN_STREAMED_SAMPLES`] and [`MIN_STREAMED_CODED_BITS_PER_SAMPLE`], +/// on 2 to 11 pool threads, or on more when it is below +/// [`MAX_STREAMED_SAMPLES_ON_A_LARGE_POOL`]. fn full_resolution_streams(samples: u64, coded_bytes: usize, threads: usize) -> bool { let coded_bits = u64::try_from(coded_bytes) .unwrap_or(u64::MAX) .saturating_mul(8); samples >= MIN_STREAMED_SAMPLES && coded_bits >= samples.saturating_mul(MIN_STREAMED_CODED_BITS_PER_SAMPLE) - && (2..CLASSIC_SYNTHESIS_THREADS).contains(&threads) + && threads >= 2 + && (threads < CLASSIC_SYNTHESIS_THREADS || samples < MAX_STREAMED_SAMPLES_ON_A_LARGE_POOL) } /// The threads a decode with `parallelism` runs on. diff --git a/crates/j2k/src/decode/route/tests.rs b/crates/j2k/src/decode/route/tests.rs index 291a4a9a..a1a04cee 100644 --- a/crates/j2k/src/decode/route/tests.rs +++ b/crates/j2k/src/decode/route/tests.rs @@ -216,9 +216,8 @@ fn tiles_at_once_need_a_busy_tile_for_every_worker() { assert_eq!(workers(2048, 6), None); } -/// Without a fixed budget, reduced untiled three-component images stream -/// (HTJ2K ones on a pool), while gray, reduced tiled and small -/// full-resolution images keep the planes. +/// Without a fixed budget, tiny reduced images, gray, reduced tiled and +/// small full-resolution images keep the planes. #[test] fn the_default_streams_where_streaming_measured_faster() { let options = |use_ht_block_coding| EncodeOptions { @@ -254,15 +253,15 @@ fn the_default_streams_where_streaming_measured_faster() { for parallelism in [CpuDecodeParallelism::Serial, CpuDecodeParallelism::Auto] { for scale in [Downscale::Half, Downscale::Quarter] { for fmt in [PixelFormat::Rgb8, PixelFormat::Rgba8, PixelFormat::Gray8] { - assert!( - default(&classic, scale, fmt, 3, parallelism).streams(), + // 48 x 40 and 24 x 20 outputs are below 256 x 256. + assert_eq!( + default(&classic, scale, fmt, 3, parallelism), + FullImageRoute::Planes, "{scale:?} {fmt:?} {parallelism:?}" ); - let pool = parallelism == CpuDecodeParallelism::Auto - && j2k_native::cpu_decode_pool_threads() >= 2; assert_eq!( - default(&ht, scale, fmt, 3, parallelism).streams(), - pool, + default(&ht, scale, fmt, 3, parallelism), + FullImageRoute::Planes, "{scale:?} {fmt:?} {parallelism:?}" ); } @@ -384,10 +383,11 @@ fn tiles_at_once_report_the_first_failing_tile() { } } -/// Full-resolution untiled classic images stream from a megapixel and 4 coded -/// bits per pixel, on 2 to 11 pool threads. +/// Full-resolution classic images stream from 4 coded bits per pixel on 2 +/// to 11 pool threads from 512 x 512 samples, and on larger pools too below +/// 2^19 samples. #[test] -fn full_resolution_images_stream_when_tier1_bound_on_a_mid_sized_pool() { +fn full_resolution_images_stream_when_tier1_bound() { use super::full_resolution_streams; let mosaic = 4096 * 2048; // 11 MB lossless and 2.2 MB at 2 bits per pixel. @@ -401,6 +401,46 @@ fn full_resolution_images_stream_when_tier1_bound_on_a_mid_sized_pool() { } assert!(full_resolution_streams(mosaic, 11_104_128, 2)); assert!(full_resolution_streams(mosaic, 11_104_128, 11)); - // Below a megapixel. - assert!(!full_resolution_streams(1000 * 1000, 2_000_000, 6)); + // 768 x 512 at 9.6 and 512 x 512 at 6.6 bits per pixel stream on any + // pool, 4096 x 1024 at 5.9 only below 12 threads. + for threads in [2, 6, 12, 18] { + assert!( + full_resolution_streams(768 * 512, 472_804, threads), + "{threads}" + ); + assert!( + full_resolution_streams(512 * 512, 217_619, threads), + "{threads}" + ); + } + assert!(full_resolution_streams(4096 * 1024, 3_114_650, 6)); + assert!(!full_resolution_streams(4096 * 1024, 3_114_650, 12)); + assert!(!full_resolution_streams(768 * 512, 472_804, 1)); + // Below 512 x 512, and 512 x 512 tiles at 2 bits per pixel. + assert!(!full_resolution_streams(500 * 500, 250_000, 6)); + assert!(!full_resolution_streams(512 * 512, 64_947, 6)); +} + +/// Reduced decodes stream from a 1024 x 512 output on any pool (HTJ2K on +/// two threads or more); smaller classic outputs from 256 x 256 on 2 to 11 +/// pool threads. +#[test] +fn reduced_decodes_stream_when_large_or_on_a_mid_sized_pool() { + use super::reduced_streams; + let (large, small) = (1024 * 512, 384 * 256); + for threads in [1, 6, 18] { + assert!(reduced_streams(large, false, threads), "{threads}"); + assert!(!reduced_streams(small, true, threads), "{threads}"); + } + assert!(!reduced_streams(large, true, 1)); + for threads in [2, 6, 11] { + assert!(reduced_streams(large, true, threads), "{threads}"); + assert!(reduced_streams(small, false, threads), "{threads}"); + } + for threads in [1, 12, 18] { + assert!(!reduced_streams(small, false, threads), "{threads}"); + } + // 256 x 256 streams on 6 threads, 192 x 128 keeps the planes. + assert!(reduced_streams(256 * 256, false, 6)); + assert!(!reduced_streams(192 * 128, false, 6)); } From 5f36f234e73b8a89754e98d57e1f3d43a4eb5ccb Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 08:01:57 -0400 Subject: [PATCH 25/94] perf(j2k): stream large gray images and one-thread decodes on the merged pool bands Re-picks the default routes now that classic pool bands decode in parts and tiles with deep band rings hold half a row batch, streaming wherever the streamed route was at least as fast as the planes (an 18-core ARM64 machine, one interleaved run, warm ratio streamed / planes, cold in brackets; bytes identical and equal to the reference hashes): - untiled gray classic images: 2896 x 2896 / 4096 x 4096 / 5792 x 5792 on 6 threads 0.88 / 0.87 / 0.85 (0.80 / 0.80 / 0.78), on 1 thread 1.00; on 18 threads and the default pool 5792 x 5792 is 1.00 (0.76- 0.79) while the smaller ones average 4-5% slower. They now stream from 2896 x 2896 below 12 pool threads and from 5792 x 5792 above; 5792 x 5792 footprint 459 -> 68 MB. - one thread: untiled classic RGB of a megapixel or more at 4 bits per pixel or more (0.99-1.00, 144 -> 46 MB) and tiled RGB images, whose tiles stream one after another (4096 x 4096 in 1024 x 1024 tiles 1.00, 262 -> 63 MB; HTJ2K in 512 x 512 tiles 0.93-0.99, 136 -> 38 MB). HTJ2K full-resolution images stay on the planes (streamed 1.04-1.45x), as do the rows listed in the change log. --- crates/j2k/src/decode/route.rs | 66 ++++++++++++++++++++++------ crates/j2k/src/decode/route/tests.rs | 31 +++++++++++-- crates/j2k/tests/resource_limits.rs | 20 +++++++-- 3 files changed, 95 insertions(+), 22 deletions(-) diff --git a/crates/j2k/src/decode/route.rs b/crates/j2k/src/decode/route.rs index 06161bf8..29cd12f4 100644 --- a/crates/j2k/src/decode/route.rs +++ b/crates/j2k/src/decode/route.rs @@ -121,7 +121,10 @@ const MIN_STREAMED_SAMPLES: u64 = 1 << 18; const MAX_STREAMED_SAMPLES_ON_A_LARGE_POOL: u64 = 1 << 19; /// Samples a tile needs to stream one after another with the pool behind -/// it (measured on 1024 x 1024 tiles only). +/// it (measured on 1024 x 1024 tiles only), and an untiled classic image to +/// stream on one thread: 2048 x 2048 to 4096 x 4096 RGB images at 5-11 +/// coded bits per pixel decode as fast streamed on one thread (0.99-1.01), +/// in a third of the memory. const MIN_STREAMED_TILE_SAMPLES: u64 = 1 << 20; /// Output samples from which a reduced-resolution decode streams on any @@ -137,6 +140,19 @@ const MIN_REDUCED_STREAMED_SAMPLES: u64 = 1 << 19; /// streamed on 6 threads, a 192 x 128 one 1-3% slower. const MIN_SMALL_REDUCED_STREAMED_SAMPLES: u64 = 1 << 16; +/// Samples from which an untiled one-component classic image streams at +/// full resolution below [`CLASSIC_SYNTHESIS_THREADS`] pool threads: the +/// smallest gray image measured. With the pool bands decoding a band in +/// parts when the rings hold too few bands, 2896 x 2896 to 5792 x 5792 gray +/// images decode 12-15% faster streamed on 6 threads and as fast on one, in +/// a fifth to a seventh of the memory (5792 x 5792: 459 vs 68 MB). +const MIN_STREAMED_GRAY_SAMPLES: u64 = 2896 * 2896; + +/// Samples from which a gray image streams on larger pools too: 5792 x 5792 +/// decodes as fast streamed on 18 threads (0.96-1.02), 2896 x 2896 and +/// 4096 x 4096 are 3-14% slower. +const MIN_STREAMED_GRAY_SAMPLES_ON_A_LARGE_POOL: u64 = 5792 * 5792; + /// Coded bits per sample from which a full-resolution untiled classic image /// streams: below it Tier-1 is light, the decode is bound by the inverse /// transforms and the pack, and the planes' fused last level does those @@ -144,8 +160,9 @@ const MIN_SMALL_REDUCED_STREAMED_SAMPLES: u64 = 1 << 16; /// pixel images are 10% faster). const MIN_STREAMED_CODED_BITS_PER_SAMPLE: u64 = 4; -/// Whether a three-component image the planes route can decode streams -/// anyway, because streaming was measured at least as fast for its kind: +/// Whether an image the planes route can decode streams anyway, because +/// streaming was measured at least as fast for its kind; for three +/// components: /// - a full-resolution tiled image whose tiles decode at once (see /// [`tile_workers`]); /// - an untiled reduced image of [`MIN_REDUCED_STREAMED_SAMPLES`] output @@ -160,19 +177,25 @@ const MIN_STREAMED_CODED_BITS_PER_SAMPLE: u64 = 4; /// (4096 x 4096 RGB in 16 tiles of 1024 x 1024: 116.9 vs 124.9 ms on the /// planes with 6 threads). /// -/// Gray images (whose streamed bands wait on a single component), +/// An untiled one-component classic image streams at full resolution as +/// [`gray_streams`] says; a tiled three-component one on one thread streams +/// its tiles one after another from 512 x 512 tiles (as fast as the planes: +/// 0.93-1.00, in a quarter of the memory). Reduced and tiled gray images, /// full-resolution HTJ2K images (whose streamed IDWT and pack cost more per /// sample than the planes' fused last level), small images and reduced /// tiled images (one tile after another was 1-13% slower on 6 threads, /// 1.5-10 times on 18) keep the planes. fn streams_by_default(image: &Image<'_>, request: RouteRequest) -> bool { - if request.components != 3 { - return false; - } let threads = pool_threads(request.parallelism); let htj2k = image.uses_high_throughput_block_coding(); let tiled = image.tile_count() > 1; let mut samples = u64::from(image.width()) * u64::from(image.height()); + if request.components == 1 { + return !request.reduced && !tiled && !htj2k && gray_streams(samples, threads); + } + if request.components != 3 { + return false; + } if request.reduced { return !tiled && reduced_streams(samples, htj2k, threads); } @@ -184,9 +207,12 @@ fn streams_by_default(image: &Image<'_>, request: RouteRequest) -> bool { if tile_workers(&rects, threads).is_some() { return true; } - let tiles = rects.len().max(1); - samples /= u64::try_from(tiles).unwrap_or(u64::MAX); - coded_bytes /= tiles; + let tile_count = rects.len().max(1); + samples /= u64::try_from(tile_count).unwrap_or(u64::MAX); + coded_bytes /= tile_count; + if threads == 1 { + return samples >= MIN_STREAMED_SAMPLES; + } if samples < MIN_STREAMED_TILE_SAMPLES { return false; } @@ -194,6 +220,14 @@ fn streams_by_default(image: &Image<'_>, request: RouteRequest) -> bool { !htj2k && full_resolution_streams(samples, coded_bytes, threads) } +/// Whether an untiled one-component classic image of `samples` streams at +/// full resolution on `threads` pool threads. +fn gray_streams(samples: u64, threads: usize) -> bool { + samples >= MIN_STREAMED_GRAY_SAMPLES + && (threads < CLASSIC_SYNTHESIS_THREADS + || samples >= MIN_STREAMED_GRAY_SAMPLES_ON_A_LARGE_POOL) +} + /// Whether a reduced-resolution decode of an untiled three-component image /// to `samples` output samples streams on `threads` pool threads. fn reduced_streams(samples: u64, htj2k: bool, threads: usize) -> bool { @@ -209,16 +243,20 @@ fn reduced_streams(samples: u64, htj2k: bool, threads: usize) -> bool { /// Whether a full-resolution three-component classic image (or tile) of /// `samples` pixels coded in `coded_bytes` streams on `threads` pool threads: /// from [`MIN_STREAMED_SAMPLES`] and [`MIN_STREAMED_CODED_BITS_PER_SAMPLE`], -/// on 2 to 11 pool threads, or on more when it is below -/// [`MAX_STREAMED_SAMPLES_ON_A_LARGE_POOL`]. +/// on 2 to 11 pool threads, on more when it is below +/// [`MAX_STREAMED_SAMPLES_ON_A_LARGE_POOL`], and on one from +/// [`MIN_STREAMED_TILE_SAMPLES`]. fn full_resolution_streams(samples: u64, coded_bytes: usize, threads: usize) -> bool { let coded_bits = u64::try_from(coded_bytes) .unwrap_or(u64::MAX) .saturating_mul(8); samples >= MIN_STREAMED_SAMPLES && coded_bits >= samples.saturating_mul(MIN_STREAMED_CODED_BITS_PER_SAMPLE) - && threads >= 2 - && (threads < CLASSIC_SYNTHESIS_THREADS || samples < MAX_STREAMED_SAMPLES_ON_A_LARGE_POOL) + && match threads { + 0 | 1 => samples >= MIN_STREAMED_TILE_SAMPLES, + 2..CLASSIC_SYNTHESIS_THREADS => true, + _ => samples < MAX_STREAMED_SAMPLES_ON_A_LARGE_POOL, + } } /// The threads a decode with `parallelism` runs on. diff --git a/crates/j2k/src/decode/route/tests.rs b/crates/j2k/src/decode/route/tests.rs index a1a04cee..e8c66870 100644 --- a/crates/j2k/src/decode/route/tests.rs +++ b/crates/j2k/src/decode/route/tests.rs @@ -274,13 +274,18 @@ fn the_default_streams_where_streaming_measured_faster() { FullImageRoute::Planes ); } - // 96 x 80 samples are far below a megapixel, tiled or not. + // 96 x 80 samples are far below a megapixel, tiled or not, and below + // the smallest gray image that streams. for bytes in [classic.as_slice(), ht.as_slice(), tiled.as_slice()] { assert_eq!( default(bytes, Downscale::None, PixelFormat::Rgb8, 3, parallelism), FullImageRoute::Planes ); } + assert_eq!( + default(&gray, Downscale::None, PixelFormat::Gray8, 1, parallelism), + FullImageRoute::Planes + ); assert_eq!( default( &classic, @@ -384,8 +389,8 @@ fn tiles_at_once_report_the_first_failing_tile() { } /// Full-resolution classic images stream from 4 coded bits per pixel on 2 -/// to 11 pool threads from 512 x 512 samples, and on larger pools too below -/// 2^19 samples. +/// to 11 pool threads from 512 x 512 samples, on larger pools too below 2^19 +/// samples, and on one thread from a megapixel. #[test] fn full_resolution_images_stream_when_tier1_bound() { use super::full_resolution_streams; @@ -393,12 +398,15 @@ fn full_resolution_images_stream_when_tier1_bound() { // 11 MB lossless and 2.2 MB at 2 bits per pixel. assert!(full_resolution_streams(mosaic, 11_104_128, 6)); assert!(!full_resolution_streams(mosaic, 2_246_803, 6)); - for threads in [1, 12, 18] { + for threads in [12, 18] { assert!( !full_resolution_streams(mosaic, 11_104_128, threads), "{threads}" ); } + // One thread: from a megapixel. + assert!(full_resolution_streams(mosaic, 11_104_128, 1)); + assert!(!full_resolution_streams(mosaic, 2_246_803, 1)); assert!(full_resolution_streams(mosaic, 11_104_128, 2)); assert!(full_resolution_streams(mosaic, 11_104_128, 11)); // 768 x 512 at 9.6 and 512 x 512 at 6.6 bits per pixel stream on any @@ -444,3 +452,18 @@ fn reduced_decodes_stream_when_large_or_on_a_mid_sized_pool() { assert!(reduced_streams(256 * 256, false, 6)); assert!(!reduced_streams(192 * 128, false, 6)); } + +/// Gray images stream from 2896 x 2896 below 12 pool threads, and on larger +/// pools from 5792 x 5792. +#[test] +fn large_gray_images_stream() { + use super::gray_streams; + for threads in [1, 6, 11] { + assert!(gray_streams(2896 * 2896, threads), "{threads}"); + assert!(!gray_streams(2048 * 2048, threads), "{threads}"); + } + for threads in [12, 18] { + assert!(gray_streams(5792 * 5792, threads), "{threads}"); + assert!(!gray_streams(4096 * 4096, threads), "{threads}"); + } +} diff --git a/crates/j2k/tests/resource_limits.rs b/crates/j2k/tests/resource_limits.rs index d7af1d1c..9a69c795 100644 --- a/crates/j2k/tests/resource_limits.rs +++ b/crates/j2k/tests/resource_limits.rs @@ -58,8 +58,18 @@ const TINY_ROI: Rect = Rect { #[test] fn decoder_component_planes_over_the_cap_report_requested_and_cap_bytes() { let codestream = large_siz_codestream(65_537); - let mut decoder = J2kDecoder::new(&codestream).expect("large SIZ parses"); + // Past the budget, native component planes stream as a region decode of + // the whole image, which refuses the claimed tile as too large without + // byte counts. + let error = J2kDecoder::new(&codestream) + .expect("large SIZ parses") + .decode_native_components() + .expect_err("a 65537x65537 tile exceeds the decode limits"); + resource_limit(&error); + // The planes route reports the bytes its planes need and the cap. + let mut decoder = J2kDecoder::new(&codestream).expect("large SIZ parses"); + decoder.set_full_decode_planes_budget(usize::MAX); let error = decoder .decode_native_components() .expect_err("65537x65537 planes exceed the decode cap"); @@ -71,12 +81,14 @@ fn decoder_component_planes_over_the_cap_report_requested_and_cap_bytes() { #[test] fn decode_into_over_the_cap_reports_requested_and_cap_bytes() { // 11600x11600 gray needs 538 MB of f32 planes, over the 512 MiB decode - // cap. The decode is rejected before it writes the caller's buffer. An - // 8-bit decode past the cap streams as a region of the whole image - // instead; 16-bit output has no streamed route and keeps the planes. + // cap. The decode is rejected before it writes the caller's buffer. By + // default, 8- and 16-bit decodes past the cap stream as a region of the + // whole image instead (and fail on this codestream's packets), so the + // planes route is pinned. let dimension = 11_600_u32; let codestream = large_siz_codestream(dimension); let mut decoder = J2kDecoder::new(&codestream).expect("large SIZ parses"); + decoder.set_full_decode_planes_budget(usize::MAX); let stride = dimension as usize * 2; let mut out = vec![0_u8; stride * dimension as usize]; From 1039132ab2fb64f64156a62d3f50958bff1a31cf Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 08:01:57 -0400 Subject: [PATCH 26/94] docs(changelog): note the plan-time full-image route --- CHANGELOG.md | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5136c6d9..3ff862de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,49 @@ Staged workspace version: `0.13.2`. full-image decodes do. Before, 8-bit region decodes of such tiles failed with `IncompletePlaneStore` or `CodeBlockDecodeFailure`, and 16-bit and component-plane region decodes returned mid-gray samples for them. +- An 8-bit full-image decode (`J2kDecoder::decode_into`, `decode_scaled_into` + and the decoders behind `decode_tiles_into`) whose component planes and + largest tile's workspace would pass the native decode budget now runs as a + region decode of the whole image, which writes rows as the streamed IDWT + finishes them, instead of failing with a resource-limit error. The native + decoder plans the planes route's charge from the codestream's tiles before + decoding (no planes or coefficients are allocated for it), so an image the + planes route decodes keeps that route, its output and its speed. A + 5661 x 8287 RGB image now decodes at full resolution with about 100 MiB + peak heap on the default thread pool (26 MiB on one thread), where the + planes route with its budget raised to fit needs about 980 MiB. Untiled RGB + images decode within 8% of that route's time, 1024 x 1024-tiled RGB takes + up to 45% longer and grayscale up to 2.8 times as long. 16-bit outputs, and + images whose tiles have empty coarse resolutions, keep the planes route + and its error. +- Full-image 8-bit decodes also stream when streaming was measured to be the + faster route for the image, chosen before decoding from the codestream's + header and tiles: reduced-resolution decodes (`decode_scaled_into`) of + untiled three-component images with an output of 1024 x 512 or more, classic + at any thread count and HTJ2K on two threads or more (a 5661 x 8287 RGB + image at 1/2 to 1/8 takes 5-47% less time on 6 threads or more, the same on + one, in 4 to 13 times less memory; HTJ2K images at 1/2 and 1/4 9-53% less), + smaller classic outputs from 256 x 256 on 2 to 11 pool threads; + full-resolution classic images coded at 4 bits per pixel or more from + 512 x 512 on 2 to 11 pool threads (5-17% faster on 6 threads) and, below + 2^19 samples, on larger pools too; and full-resolution images with a tile + for every thread. Those now decode their tiles at once on the calling thread + and the pool's threads, each tile on one thread, instead of one tile after + another: a 4096 x 8192 RGB image of 1024 x 1024 tiles decodes 8% faster than + on the planes with 6 threads and 23% faster with 18, in less than half the + memory. Tiled classic images with fewer tiles stream them one after another + on 2 to 11 pool threads when each tile is a megapixel or more. Tiled images + past the decode budget decode their tiles at once too (an 8192 x 8192 RGB + image of 1024 x 1024 tiles 22% faster with 18 threads, a 16384 x 16384 gray + one 2.9 times as fast). Untiled gray classic images stream at full + resolution from 2896 x 2896 below 12 pool threads and from 5792 x 5792 on + larger pools (12-15% faster on 6 threads, as fast elsewhere, in a fifth to a + seventh of the memory), and on one thread tiled RGB images stream their + tiles one after another and untiled classic RGB images of a megapixel or + more at 4 bits per pixel or more stream too (as fast, in a quarter to a + third of the memory). Reduced and tiled gray images, full-resolution HTJ2K + images, reduced tiled images, small images and images with few tiles on a + large pool keep the planes route. Output is unchanged. - x86-64 RGB8 decodes of 9/7 images round and pack the inverse ICT output with the runtime SIMD level (AVX2 where present) instead of a scalar library call per sample: the centered rounding of the inverse MCT runs eight From 6024e44b46dbf50073df337c1e564fc2e2e8ee23 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 03:17:29 -0400 Subject: [PATCH 27/94] feat(j2k-native): pack native component planes from a region decode's streamed rows A region decode that hands its rows to a sink holds no region-sized f32 planes. Image::decode_native_region_components_rows_with_context (hidden) runs that decode and packs each finished rectangle into the returned native-bit-depth planes with the planes route's own conversions (round_clamped_native up to 16 bits, push_native_sample_bytes above), so the planes hold the same bytes. Image::streamed_rows_native_precision (hidden) says when the rows carry the samples the native decodes pack: no palette, channel definition, sYCC or CIELab, unsubsampled components of one precision up to 24 bits. The planes are the decode's output and, like a caller's buffer, are not charged to its budget; each keeps the per-allocation cap. --- crates/j2k-native/src/image/native.rs | 1 + crates/j2k-native/src/image/native/rows.rs | 279 ++++++++++++++++++ ...able-api-1.0.implementation-public-api.txt | 2 + 3 files changed, 282 insertions(+) create mode 100644 crates/j2k-native/src/image/native/rows.rs diff --git a/crates/j2k-native/src/image/native.rs b/crates/j2k-native/src/image/native.rs index 84f48964..91d4c5b8 100644 --- a/crates/j2k-native/src/image/native.rs +++ b/crates/j2k-native/src/image/native.rs @@ -15,6 +15,7 @@ use crate::{ use super::Image; mod allocation; +mod rows; pub(super) use allocation::{try_clone_color_space, NativeOutputBudget}; impl<'a> Image<'a> { diff --git a/crates/j2k-native/src/image/native/rows.rs b/crates/j2k-native/src/image/native/rows.rs new file mode 100644 index 00000000..bbab1d5f --- /dev/null +++ b/crates/j2k-native/src/image/native/rows.rs @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Native-bit-depth component planes packed from a region decode's +//! streamed rows instead of from region-sized `f32` planes. + +use alloc::vec::Vec; + +use super::{ + round_clamped_native, signed_native_bounds, try_clone_color_space, unsigned_native_max, +}; +use crate::color::{ + ComponentPlane, ComponentRowSink, DecodedNativeComponents, NativeComponentPlane, +}; +use crate::error::{bail, DecodingError, Result, ValidationError}; +use crate::j2c::DecoderContext; +use crate::roi::validate_roi; +use crate::{ + checked_decode_byte_len2, checked_decode_sample_count, native_bytes_per_sample, + try_reserve_decode_elements, try_resize_decode_elements, +}; + +use super::Image; + +impl<'a> Image<'a> { + /// The precision of every component when a region decode's rows (see + /// [`Self::decode_region_rows_for_integer_output_with_context`]) carry + /// the samples the native-bit-depth decodes pack: the image has no + /// palette, channel definition, sYCC or `CIELab`, which only those + /// decodes' planes resolve or convert, and its components are at full + /// sampling with one precision of at most 24 bits. `None` otherwise. + #[doc(hidden)] + #[must_use] + pub fn streamed_rows_native_precision(&self) -> Option { + if !self.permits_line_region_idwt() { + return None; + } + let precision = self.header.component_infos.first()?.size_info.precision; + let uniform = self.header.component_infos.iter().all(|component| { + component.size_info.precision == precision + && component.size_info.horizontal_resolution == 1 + && component.size_info.vertical_resolution == 1 + }); + (uniform && (1..=24).contains(&precision)).then_some(precision) + } + + /// [`Self::decode_native_region_components_with_context`] on the + /// region decode that hands its rows to a sink (see + /// [`Self::decode_region_rows_for_integer_output_with_context`]), which + /// packs them into the returned planes as they are finished: the decode + /// holds no region-sized `f32` planes. The planes hold the same bytes. + /// Only for an image with a [`Self::streamed_rows_native_precision`]. + /// The region of the whole image gives + /// [`Self::decode_native_components_with_context`]'s planes. + /// + /// The returned planes are the decode's output: like a caller's buffer + /// they are not charged to the decode budget, and each is held to the + /// per-allocation cap. + /// + /// # Errors + /// + /// Returns an error when the region is invalid, the image has no + /// streamed rows precision, a plane cannot be allocated, or decoding + /// fails. + #[doc(hidden)] + pub fn decode_native_region_components_rows_with_context( + &self, + roi: (u32, u32, u32, u32), + decoder_context: &mut DecoderContext<'a>, + ) -> Result { + validate_roi((self.width(), self.height()), roi)?; + if self.streamed_rows_native_precision().is_none() { + bail!(DecodingError::UnsupportedFeature( + "streamed native component planes need unsubsampled components of one precision up to 24 bits without palette, channel definition, sYCC or CIELab" + )); + } + let dimensions = (roi.2, roi.3); + let samples = checked_decode_sample_count(dimensions.0, dimensions.1)?; + let mut planes = Vec::new(); + try_reserve_decode_elements(&mut planes, self.header.component_infos.len())?; + for component in &self.header.component_infos { + let bit_depth = component.size_info.precision; + let bytes_per_sample = native_bytes_per_sample(bit_depth)?; + let mut data = Vec::new(); + try_resize_decode_elements( + &mut data, + checked_decode_byte_len2(samples, bytes_per_sample)?, + 0, + )?; + planes.push(NativeComponentPlane { + data, + dimensions, + bit_depth, + signed: component.size_info.signed, + sampling: (1, 1), + bytes_per_sample: u8::try_from(bytes_per_sample) + .map_err(|_| ValidationError::ImageTooLarge)?, + }); + } + let color_space = try_clone_color_space(&self.color_space)?; + let mut sink = NativePlanesSink { + planes: &mut planes, + scratch: Vec::new(), + failed: false, + }; + self.decode_region_rows_for_integer_output_with_context(roi, decoder_context, &mut sink)?; + if sink.failed { + bail!(DecodingError::CodeBlockDecodeFailure); + } + Ok(DecodedNativeComponents { + dimensions, + color_space, + has_alpha: self.has_alpha, + planes, + }) + } +} + +/// Packs each rectangle of a region decode into the native planes it +/// covers. A rectangle that does not fit them fails the decode. +struct NativePlanesSink<'p> { + planes: &'p mut [NativeComponentPlane], + /// One row of samples wider than 16 bits, packed before it is copied. + scratch: Vec, + failed: bool, +} + +impl ComponentRowSink for NativePlanesSink<'_> { + fn write_rows(&mut self, x: u32, y: u32, planes: &[ComponentPlane<'_>]) { + if !self.failed && self.try_write_rows(x, y, planes).is_none() { + self.failed = true; + } + } +} + +impl NativePlanesSink<'_> { + fn try_write_rows(&mut self, x: u32, y: u32, sources: &[ComponentPlane<'_>]) -> Option<()> { + if sources.len() != self.planes.len() { + return None; + } + for (source, target) in sources.iter().zip(self.planes.iter_mut()) { + if source.bit_depth != target.bit_depth || source.signed != target.signed { + return None; + } + let width = source.dimensions.0 as usize; + let height = source.dimensions.1 as usize; + if width == 0 || height == 0 { + continue; + } + let bytes_per_sample = usize::from(target.bytes_per_sample); + let row_bytes = width.checked_mul(bytes_per_sample)?; + let stride = (target.dimensions.0 as usize).checked_mul(bytes_per_sample)?; + let mut start = (y as usize) + .checked_mul(stride)? + .checked_add((x as usize).checked_mul(bytes_per_sample)?)?; + let rows = source.samples.get(..width.checked_mul(height)?)?; + for samples in rows.chunks_exact(width) { + let end = start.checked_add(row_bytes)?; + let destination = target.data.get_mut(start..end)?; + pack_native_samples( + destination, + samples, + target.bit_depth, + target.signed, + &mut self.scratch, + ) + .ok()?; + start = start.checked_add(stride)?; + } + } + Some(()) + } +} + +/// Write `samples` to `destination` as the packed little-endian native +/// samples `Image::extend_component_native_sample_bytes` appends for an +/// `f32` component of `bit_depth`. Samples wider than 16 bits are packed +/// in `scratch` first. +#[expect( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + reason = "samples are rounded and clamped to the component's 8- or 16-bit range before packing" +)] +pub(super) fn pack_native_samples( + destination: &mut [u8], + samples: &[f32], + bit_depth: u8, + signed: bool, + scratch: &mut Vec, +) -> Result<()> { + match (bit_depth, signed) { + (1..=8, true) => { + let (min, max) = signed_native_bounds(bit_depth); + for (destination, &sample) in destination.iter_mut().zip(samples) { + *destination = (round_clamped_native(sample, min, max) as i8) as u8; + } + } + (1..=8, false) => { + let max = unsigned_native_max(bit_depth); + for (destination, &sample) in destination.iter_mut().zip(samples) { + *destination = round_clamped_native(sample, 0.0, max) as u8; + } + } + (9..=16, true) => { + let (min, max) = signed_native_bounds(bit_depth); + for (destination, &sample) in destination.as_chunks_mut::<2>().0.iter_mut().zip(samples) + { + *destination = + ((round_clamped_native(sample, min, max) as i16) as u16).to_le_bytes(); + } + } + (9..=16, false) => { + let max = unsigned_native_max(bit_depth); + for (destination, &sample) in destination.as_chunks_mut::<2>().0.iter_mut().zip(samples) + { + *destination = (round_clamped_native(sample, 0.0, max) as u16).to_le_bytes(); + } + } + _ => { + scratch.clear(); + try_reserve_decode_elements(scratch, destination.len())?; + for &sample in samples { + Image::push_native_sample_bytes(scratch, sample, bit_depth, signed); + } + let packed = scratch + .get(..destination.len()) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + destination.copy_from_slice(packed); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use alloc::vec::Vec; + + use super::pack_native_samples; + use crate::image::Image; + use crate::native_bytes_per_sample; + + /// Samples around every bound and rounding tie of each precision, plus + /// values far outside it and NaN. + #[expect(clippy::cast_precision_loss, reason = "bounds of at most 24 bits")] + fn samples(bit_depth: u8) -> Vec { + let mut samples = Vec::from([f32::NAN, f32::INFINITY, f32::NEG_INFINITY, 0.0, -0.0]); + for bound in [ + -(1_i64 << (bit_depth - 1)), + (1_i64 << (bit_depth - 1)) - 1, + (1_i64 << bit_depth) - 1, + 0, + 1, + ] { + for offset in [-1.5, -1.0, -0.5, -0.49, 0.0, 0.49, 0.5, 1.0, 1.5] { + samples.push(bound as f32 + offset); + } + } + samples.extend([1.0e9, -1.0e9, 0.499_999_97, 2.5, -2.5, 3.5]); + samples + } + + #[test] + fn packed_rows_match_the_planes_pack_at_every_precision() { + let mut scratch = Vec::new(); + for bit_depth in 1..=24 { + for signed in [false, true] { + let samples = samples(bit_depth); + let mut expected = Vec::new(); + for &sample in &samples { + Image::push_native_sample_bytes(&mut expected, sample, bit_depth, signed); + } + let bytes = native_bytes_per_sample(bit_depth).expect("bytes per sample"); + let mut packed = alloc::vec![0xa5; samples.len() * bytes]; + pack_native_samples(&mut packed, &samples, bit_depth, signed, &mut scratch) + .expect("pack"); + assert_eq!(packed, expected, "{bit_depth} bits, signed {signed}"); + } + } + } +} diff --git a/xtask/api/stable-api-1.0.implementation-public-api.txt b/xtask/api/stable-api-1.0.implementation-public-api.txt index e51766e8..0f1fd256 100644 --- a/xtask/api/stable-api-1.0.implementation-public-api.txt +++ b/xtask/api/stable-api-1.0.implementation-public-api.txt @@ -2501,6 +2501,7 @@ pub fn j2k_native::Image<'a>::decode_components_for_integer_output_with_context< pub fn j2k_native::Image<'a>::decode_components_for_integer_output_with_ht_decoder<'ctx>(&self, &'ctx mut j2k_native::DecoderContext<'a>, &mut dyn j2k_native::HtCodeBlockDecoder) -> j2k_native::Result> pub fn j2k_native::Image<'a>::decode_components_with_ht_decoder<'ctx>(&self, &'ctx mut j2k_native::DecoderContext<'a>, &mut dyn j2k_native::HtCodeBlockDecoder) -> j2k_native::Result> pub fn j2k_native::Image<'a>::decode_native_components_with_retained_capacity(&self, usize) -> j2k_native::Result +pub fn j2k_native::Image<'a>::decode_native_region_components_rows_with_context(&self, (u32, u32, u32, u32), &mut j2k_native::DecoderContext<'a>) -> j2k_native::Result pub fn j2k_native::Image<'a>::decode_native_with_retained_capacity(&self, usize) -> j2k_native::Result pub fn j2k_native::Image<'a>::decode_region_components_for_integer_output_with_context<'ctx>(&self, (u32, u32, u32, u32), &'ctx mut j2k_native::DecoderContext<'a>) -> j2k_native::Result> pub fn j2k_native::Image<'a>::decode_region_components_for_integer_output_with_ht_decoder<'ctx>(&self, &'ctx mut j2k_native::DecoderContext<'a>, (u32, u32, u32, u32), &mut dyn j2k_native::HtCodeBlockDecoder) -> j2k_native::Result> @@ -2515,6 +2516,7 @@ pub fn j2k_native::Image<'a>::output_tile_rects(&self) -> j2k_native::Result::prepare_region_decoder_with_context<'image, 'context>(&'image self, &'context mut j2k_native::DecoderContext<'a>) -> j2k_native::Result> pub fn j2k_native::Image<'a>::primary_icc_profile(&self) -> core::option::Option<&[u8]> pub fn j2k_native::Image<'a>::retained_allocation_bytes(&self) -> j2k_native::Result +pub fn j2k_native::Image<'a>::streamed_rows_native_precision(&self) -> core::option::Option pub fn j2k_native::Image<'a>::supports_direct_device_plane_reuse(&self) -> bool pub fn j2k_native::Image<'a>::tile_count(&self) -> u32 pub fn j2k_native::Image<'a>::uses_high_throughput_block_coding(&self) -> bool From 18db9e237ae590bdd4671cd73125c69409eb7214 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 03:18:15 -0400 Subject: [PATCH 28/94] fix(j2k): stream 16-bit region decodes and over-budget native component planes A 16-bit region decode of an image whose rows carry its native samples (no palette, channel definition, sYCC or CIELab; unsubsampled components of one precision up to 16 bits; an RGB or gray layout) now streams its rows through a U16RowSink instead of holding the region's f32 planes and a RawBitmap of its native samples. The sink converts each finished rectangle as write_u16_output converts the packed native samples. Other images keep the RawBitmap path, with its output and errors. The 16-bit component-plane writers convert a plane at a time (the per-plane choices made once, a 256-entry table for samples of up to 8 bits instead of a u32 division per sample, 8-bit samples as byte * 257, RGB one pixel at a time), with the per-sample conversion kept in the tests as the reference. J2kDecoder::decode_native_components runs as a region decode of the whole image that packs its rows into the native planes when the planes route would be refused, by its decode (the native planes floor) or by its pack, which holds every f32 plane besides the packed samples. decode_native_region_components streams whenever the rows carry the native samples, as 8- and 16-bit region decodes do. Tests: forced streamed routes write the planes route's bytes for 4/8/12/16-bit, signed and unsigned, 5/3 and 9/7, tiled, HT and RGBA images, and region decodes equal crops of the full image. --- crates/j2k/src/decode.rs | 26 +- crates/j2k/src/decode/output.rs | 10 +- crates/j2k/src/decode/output/native.rs | 85 ++++ crates/j2k/src/decode/output/u16.rs | 450 +++++++++++++++--- crates/j2k/src/view.rs | 58 ++- .../tests/streamed_sixteen_bit_and_native.rs | 363 ++++++++++++++ 6 files changed, 905 insertions(+), 87 deletions(-) create mode 100644 crates/j2k/src/decode/output/native.rs create mode 100644 crates/j2k/tests/streamed_sixteen_bit_and_native.rs diff --git a/crates/j2k/src/decode.rs b/crates/j2k/src/decode.rs index d9d35e00..5f74b9e4 100644 --- a/crates/j2k/src/decode.rs +++ b/crates/j2k/src/decode.rs @@ -12,12 +12,17 @@ pub use component_handoff::{ J2kNativeComponentPlane, }; mod output; +pub(crate) use output::{ + decode_native_components_with_native_context, native_components_route, + native_region_components_route, +}; mod route; mod settings; mod srgb8; use output::{ - can_decode_u8_directly, expand_rgb8_to_rgba8, write_components_u16_output, - write_components_u8_output, write_u16_output, write_u8_output, U8RowSink, + can_decode_u8_directly, expand_rgb8_to_rgba8, rows_match_native_u16, + write_components_u16_output, write_components_u8_output, write_u16_output, write_u8_output, + U16RowSink, U8RowSink, }; pub(crate) use route::{full_image_route, FullImageRoute, PlanesBudget, RouteRequest}; pub use settings::DecodeSettings; @@ -183,6 +188,23 @@ pub(crate) fn decode_image_region_into_with_native_context<'a>( .map_err(J2kError::from_native_decode_error)?; sink.finish() } + PixelFormat::Rgb16 | PixelFormat::Rgba16 | PixelFormat::Gray16 + if rows_match_native_u16(image, fmt) => + { + // As at 8 bits, the rows go to `out` as they are finished; the + // sink converts them as `write_u16_output` converts the native + // samples of the same region. + let mut sink = + U16RowSink::new(image.color_space(), image.has_alpha(), out, stride, fmt); + image + .decode_region_rows_for_integer_output_with_context( + (roi.x, roi.y, roi.w, roi.h), + native_context, + &mut sink, + ) + .map_err(J2kError::from_native_decode_error)?; + sink.finish() + } PixelFormat::Rgb16 | PixelFormat::Rgba16 | PixelFormat::Gray16 => { let raw = image .decode_native_region_with_context((roi.x, roi.y, roi.w, roi.h), native_context) diff --git a/crates/j2k/src/decode/output.rs b/crates/j2k/src/decode/output.rs index fae0c8b9..873dd9d4 100644 --- a/crates/j2k/src/decode/output.rs +++ b/crates/j2k/src/decode/output.rs @@ -2,10 +2,18 @@ //! Pixel-layout conversion for decoded native output. +mod native; mod u16; mod u8; -pub(super) use u16::{write_components_u16_output, write_u16_output}; +pub(crate) use native::{ + decode_native_components_with_native_context, native_components_route, + native_region_components_route, +}; + +pub(super) use u16::{ + rows_match_native_u16, write_components_u16_output, write_u16_output, U16RowSink, +}; pub(super) use u8::{ can_decode_u8_directly, expand_rgb8_to_rgba8, write_components_u8_output, write_u8_output, U8RowSink, diff --git a/crates/j2k/src/decode/output/native.rs b/crates/j2k/src/decode/output/native.rs new file mode 100644 index 00000000..f6234fb8 --- /dev/null +++ b/crates/j2k/src/decode/output/native.rs @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Plan-time choice of how a full-image decode into owned native-bit-depth +//! component planes runs, and the planes route's charge for packing them. + +use crate::backend::Image; +use crate::decode::FullImageRoute; +use crate::J2kError; +use j2k_core::Rect; + +/// The bytes the planes route charges at least when it packs `image`'s +/// output from its component planes at `bytes_per_sample` native bytes per +/// sample: every `f32` plane and the packed samples. It holds both at once, +/// after the decode's other buffers are released, and refuses a pack past +/// the decode budget. Counts the components of `image`'s color space and +/// alpha only, so it is at most the real charge. +pub(in crate::decode) fn planes_pack_floor(image: &Image<'_>, bytes_per_sample: usize) -> usize { + let components = usize::from(image.color_space().num_channels()) + .saturating_add(usize::from(image.has_alpha())); + (image.width() as usize) + .saturating_mul(image.height() as usize) + .saturating_mul(components) + .saturating_mul(bytes_per_sample.saturating_add(size_of::())) +} + +/// How a full-image decode of `image` into owned native component planes +/// runs when its planes may charge `planes_budget` bytes: as a region decode +/// of the whole image that packs its rows into the planes as they are +/// finished, when its rows carry the native samples (see +/// `Image::streamed_rows_native_precision`) and the planes route would be +/// refused, by its decode (see `Image::full_decode_planes_floor`) or by its +/// pack (see [`planes_pack_floor`]). +/// +/// A planning error keeps the planes route, which meets the same error and +/// reports it. +pub(crate) fn native_components_route(image: &Image<'_>, planes_budget: usize) -> FullImageRoute { + let Some(precision) = image.streamed_rows_native_precision() else { + return FullImageRoute::Planes; + }; + let pack_floor = planes_pack_floor(image, usize::from(precision).div_ceil(8)); + match image.full_decode_planes_floor(planes_budget) { + Ok(Some(floor)) if floor.max(pack_floor) > planes_budget => FullImageRoute::StreamedRegion, + _ => FullImageRoute::Planes, + } +} + +/// How a region decode of `image` into owned native component planes runs: +/// as one that packs its rows into the planes as they are finished whenever +/// its rows carry the native samples, as 8- and 16-bit region decodes +/// stream theirs, so that it holds no region-sized `f32` planes. +pub(crate) fn native_region_components_route(image: &Image<'_>) -> FullImageRoute { + if image.streamed_rows_native_precision().is_some() { + FullImageRoute::StreamedRegion + } else { + FullImageRoute::Planes + } +} + +/// Decode `roi` of `image`, or the whole image, into owned native component +/// planes on `route`, which [`native_components_route`] or +/// [`native_region_components_route`] chose for it. +pub(crate) fn decode_native_components_with_native_context<'a>( + image: &Image<'a>, + native_context: &mut j2k_native::DecoderContext<'a>, + roi: Option, + route: FullImageRoute, +) -> Result { + match (roi, route) { + (None, FullImageRoute::Planes) => { + image.decode_native_components_with_context(native_context) + } + (Some(roi), FullImageRoute::Planes) => image.decode_native_region_components_with_context( + (roi.x, roi.y, roi.w, roi.h), + native_context, + ), + (roi, FullImageRoute::StreamedRegion) => { + let roi = roi.unwrap_or_else(|| Rect::full((image.width(), image.height()))); + image.decode_native_region_components_rows_with_context( + (roi.x, roi.y, roi.w, roi.h), + native_context, + ) + } + } + .map_err(J2kError::from_native_decode_error) +} diff --git a/crates/j2k/src/decode/output/u16.rs b/crates/j2k/src/decode/output/u16.rs index f76306fe..03dd2874 100644 --- a/crates/j2k/src/decode/output/u16.rs +++ b/crates/j2k/src/decode/output/u16.rs @@ -2,7 +2,7 @@ //! Sixteen-bit channel-layout conversion for native decoded samples. -use crate::backend::{ColorSpace, DecodedComponents as NativeDecodedComponents, RawBitmap}; +use crate::backend::{ColorSpace, DecodedComponents as NativeDecodedComponents, Image, RawBitmap}; use crate::J2kError; use j2k_core::{PixelFormat, Unsupported}; @@ -14,17 +14,49 @@ pub(in crate::decode) fn write_components_u16_output( stride: usize, fmt: PixelFormat, ) -> Result<(), J2kError> { - let dims = components.dimensions(); - let expected_samples = component_sample_count(dims)?; - let width = dims.0 as usize; - let height = dims.1 as usize; - let planes = components.planes(); - match ( + write_planes_u16( components.color_space(), components.has_alpha(), - planes.len(), + components.planes(), + components.dimensions(), + out, + stride, fmt, - ) { + ) +} + +/// Whether a region decode of `image` to `fmt` may stream its rows through +/// [`U16RowSink`]: they carry the samples `decode_native_region` packs (see +/// `Image::streamed_rows_native_precision`), every component has one +/// precision of at most 16 bits, which `U16Conversion` converts to the +/// bytes [`write_u16_output`] writes from those packed samples, and +/// `fmt` is a layout the image's color space maps to. Images that fail any +/// of these keep the native-bit-depth planes, with their output or error. +pub(in crate::decode) fn rows_match_native_u16(image: &Image<'_>, fmt: PixelFormat) -> bool { + image + .streamed_rows_native_precision() + .is_some_and(|precision| precision <= 16) + && matches!( + (image.color_space(), image.has_alpha(), fmt), + (ColorSpace::Gray, false, PixelFormat::Gray16) + | (ColorSpace::RGB, _, PixelFormat::Rgb16 | PixelFormat::Rgba16) + ) +} + +/// Write `planes`, each `dims` in size, to `out` from its first byte. +fn write_planes_u16( + color_space: &ColorSpace, + has_alpha: bool, + planes: &[j2k_native::ComponentPlane<'_>], + dims: (u32, u32), + out: &mut [u8], + stride: usize, + fmt: PixelFormat, +) -> Result<(), J2kError> { + let expected_samples = component_sample_count(dims)?; + let width = dims.0 as usize; + let height = dims.1 as usize; + match (color_space, has_alpha, planes.len(), fmt) { (ColorSpace::Gray, false, 1, PixelFormat::Gray16) => { validate_component_planes(&planes[..1], expected_samples)?; write_component_rows_u16(&planes[0], out, stride, width, height); @@ -53,6 +85,74 @@ pub(in crate::decode) fn write_components_u16_output( } } +/// Writes a region decode's rectangles of component rows to a 16-bit +/// output as [`write_components_u16_output`] writes whole planes, which for +/// an image that [`rows_match_native_u16`] accepts is what +/// [`write_u16_output`] writes from its native samples. The first error +/// stops every later write and is returned by [`Self::finish`]. +pub(in crate::decode) struct U16RowSink<'o> { + color_space: &'o ColorSpace, + has_alpha: bool, + out: &'o mut [u8], + stride: usize, + fmt: PixelFormat, + error: Option, +} + +impl<'o> U16RowSink<'o> { + /// `out` must already hold the region at `stride` in `fmt`. + pub(in crate::decode) fn new( + color_space: &'o ColorSpace, + has_alpha: bool, + out: &'o mut [u8], + stride: usize, + fmt: PixelFormat, + ) -> Self { + Self { + color_space, + has_alpha, + out, + stride, + fmt, + error: None, + } + } + + pub(in crate::decode) fn finish(self) -> Result<(), J2kError> { + self.error.map_or(Ok(()), Err) + } +} + +impl j2k_native::ComponentRowSink for U16RowSink<'_> { + fn write_rows(&mut self, x: u32, y: u32, planes: &[j2k_native::ComponentPlane<'_>]) { + if self.error.is_some() { + return; + } + let dims = planes + .first() + .map_or((0, 0), j2k_native::ComponentPlane::dimensions); + let offset = (y as usize) + .checked_mul(self.stride) + .and_then(|row| row.checked_add((x as usize).checked_mul(self.fmt.bytes_per_pixel())?)); + let result = match offset.and_then(|offset| self.out.get_mut(offset..)) { + Some(out) => write_planes_u16( + self.color_space, + self.has_alpha, + planes, + dims, + out, + self.stride, + self.fmt, + ), + None => Err(J2kError::DimensionOverflow { + width: x, + height: y, + }), + }; + self.error = result.err(); + } +} + fn write_component_rows_u16( plane: &j2k_native::ComponentPlane<'_>, out: &mut [u8], @@ -60,13 +160,14 @@ fn write_component_rows_u16( width: usize, height: usize, ) { + let conversion = U16Conversion::new(plane.bit_depth(), plane.signed()); for y in 0..height { - let src = &plane.samples()[y * width..(y + 1) * width]; - let dst = &mut out[y * stride..y * stride + width * 2]; - for (sample, destination) in src.iter().zip(dst.as_chunks_mut::<2>().0) { - *destination = - component_sample_as_u16(*sample, plane.bit_depth(), plane.signed()).to_le_bytes(); - } + conversion.visit(ChannelWriter { + src: &plane.samples()[y * width..(y + 1) * width], + dst: &mut out[y * stride..y * stride + width * 2], + channels: 1, + channel: 0, + }); } } @@ -100,65 +201,204 @@ fn write_color_component_rows_u16( output_channels: usize, synthesize_alpha: bool, ) { + let color = [0, 1, 2] + .map(|channel| U16Conversion::new(planes[channel].bit_depth(), planes[channel].signed())); + let fused = color[1] == color[0] && color[2] == color[0]; + let alpha = (output_channels == 4 && !synthesize_alpha) + .then(|| U16Conversion::new(planes[3].bit_depth(), planes[3].signed())); + let opaque = opaque_alpha_u16( + usize::from(planes[0].bit_depth()).div_ceil(8), + planes[0].bit_depth(), + ) + .to_le_bytes(); for y in 0..height { - let row = y * width; + let row = y * width..(y + 1) * width; let destination = &mut out[y * stride..y * stride + width * output_channels * 2]; - for x in 0..width { - let pixel = &mut destination[x * output_channels * 2..(x + 1) * output_channels * 2]; - for channel in 0..3 { - let sample = component_sample_as_u16( - planes[channel].samples()[row + x], - planes[channel].bit_depth(), - planes[channel].signed(), - ); - pixel[channel * 2..channel * 2 + 2].copy_from_slice(&sample.to_le_bytes()); + if fused { + color[0].visit(PixelWriter { + src: [0, 1, 2].map(|channel| &planes[channel].samples()[row.clone()]), + dst: destination, + channels: output_channels, + }); + } else { + for (channel, conversion) in color.iter().enumerate() { + conversion.visit(ChannelWriter { + src: &planes[channel].samples()[row.clone()], + dst: destination, + channels: output_channels, + channel, + }); } - if output_channels == 4 { - let alpha = if synthesize_alpha { - opaque_alpha_u16( - usize::from(planes[0].bit_depth()).div_ceil(8), - planes[0].bit_depth(), - ) - } else { - component_sample_as_u16( - planes[3].samples()[row + x], - planes[3].bit_depth(), - planes[3].signed(), - ) - }; - pixel[6..8].copy_from_slice(&alpha.to_le_bytes()); + } + match &alpha { + Some(conversion) => conversion.visit(ChannelWriter { + src: &planes[3].samples()[row], + dst: destination, + channels: 4, + channel: 3, + }), + None if output_channels == 4 => { + for pixel in destination.as_chunks_mut::<8>().0 { + pixel[6..8].copy_from_slice(&opaque); + } } + None => {} } } } -#[expect( - clippy::cast_possible_truncation, - clippy::cast_sign_loss, - reason = "decoded samples are rounded and clamped to the component's declared integer representation" -)] -fn component_sample_as_u16(sample: f32, bit_depth: u8, signed: bool) -> u16 { - let rounded = sample.round(); - if signed { +/// How [`U16Conversion`] turns a rounded sample into its output. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum U16Kind { + /// Unsigned 8 bits: the byte times 257, which is its widened value. + Byte, + /// Fewer unsigned bits: the byte's widened value from the table. + ByteTable, + /// Signed, at most 8 bits: the two's-complement byte's widened value. + SignedByteTable, + /// Unsigned, 9 to 16 bits: the sample itself. + Unsigned, + /// Signed, 9 to 16 bits: the sample's two's complement. + Signed, +} + +/// Converts the decoded samples of one component of `bit_depth` bits to +/// 16-bit output: rounded, clamped to the component's signed or unsigned +/// range (at most 16 bits) and, for at most 8 bits, clamped to a byte and +/// widened across the 16-bit range as [`write_u16_output`] widens native +/// 8-bit samples. The choices are made once per plane. +#[derive(Debug, PartialEq)] +struct U16Conversion { + kind: U16Kind, + /// The component's range, or the byte's for at most 8 unsigned bits. + bounds: (f32, f32), + /// The widened value of each byte for the table kinds. + widened: [u16; 256], +} + +impl U16Conversion { + fn new(bit_depth: u8, signed: bool) -> Self { if bit_depth <= 8 { - let magnitude_bits = u32::from(bit_depth.saturating_sub(1)); - let min = -(1_i16 << magnitude_bits); - let max = (1_i16 << magnitude_bits) - 1; - return widen_u8_sample_to_u16( - (rounded.clamp(f32::from(min), f32::from(max)) as i8) as u8, - bit_depth, - ); + let (kind, bounds) = if signed { + let magnitude = 1_i16 << bit_depth.saturating_sub(1); + ( + U16Kind::SignedByteTable, + (f32::from(-magnitude), f32::from(magnitude - 1)), + ) + } else if bit_depth == 8 { + (U16Kind::Byte, (0.0, f32::from(u8::MAX))) + } else { + (U16Kind::ByteTable, (0.0, f32::from(u8::MAX))) + }; + return Self { + kind, + bounds, + widened: core::array::from_fn(|byte| { + widen_u8_sample_to_u16(u8::try_from(byte).unwrap_or(u8::MAX), bit_depth) + }), + }; + } + let (kind, bounds) = if signed { + let magnitude = 1_i32 << (bit_depth.min(16) - 1); + ( + U16Kind::Signed, + ( + f32::from(i16::try_from(-magnitude).unwrap_or(i16::MIN)), + f32::from(i16::try_from(magnitude - 1).unwrap_or(i16::MAX)), + ), + ) + } else { + let max = (1_u32 << bit_depth.min(16)) - 1; + ( + U16Kind::Unsigned, + (0.0, f32::from(u16::try_from(max).unwrap_or(u16::MAX))), + ) + }; + Self { + kind, + bounds, + widened: [0; 256], } - let magnitude_bits = u32::from(bit_depth.min(16).saturating_sub(1)); - let min = i16::try_from(-(1_i32 << magnitude_bits)).unwrap_or(i16::MIN); - let max = i16::try_from((1_i32 << magnitude_bits) - 1).unwrap_or(i16::MAX); - return (rounded.clamp(f32::from(min), f32::from(max)) as i16) as u16; } - if bit_depth <= 8 { - return widen_u8_sample_to_u16(rounded.clamp(0.0, f32::from(u8::MAX)) as u8, bit_depth); + + /// Hand `writer` this conversion as a closure of its kind. + #[expect( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + reason = "decoded samples are rounded and clamped to the component's declared integer representation" + )] + #[inline(always)] + fn visit(&self, writer: impl U16Writer) { + let (min, max) = self.bounds; + let widened = &self.widened; + match self.kind { + U16Kind::Byte => { + writer.write(|sample: f32| u16::from(sample.round().clamp(min, max) as u8) * 257); + } + U16Kind::ByteTable => writer + .write(|sample: f32| widened[usize::from(sample.round().clamp(min, max) as u8)]), + U16Kind::SignedByteTable => writer.write(|sample: f32| { + widened[usize::from((sample.round().clamp(min, max) as i8) as u8)] + }), + U16Kind::Unsigned => writer.write(|sample: f32| sample.round().clamp(min, max) as u16), + U16Kind::Signed => { + writer.write(|sample: f32| (sample.round().clamp(min, max) as i16) as u16); + } + } + } +} + +/// Writes rows of samples with a [`U16Conversion`]'s closure. +trait U16Writer { + fn write(self, convert: impl Fn(f32) -> u16); +} + +/// One component's samples into one channel of interleaved pixels. +struct ChannelWriter<'s, 'd> { + src: &'s [f32], + dst: &'d mut [u8], + channels: usize, + channel: usize, +} + +impl U16Writer for ChannelWriter<'_, '_> { + #[inline(always)] + fn write(self, convert: impl Fn(f32) -> u16) { + let Self { + src, + dst, + channels, + channel, + } = self; + for (pixel, &sample) in dst.chunks_exact_mut(channels * 2).zip(src) { + pixel[channel * 2..channel * 2 + 2].copy_from_slice(&convert(sample).to_le_bytes()); + } + } +} + +/// Three components' samples into the first three channels of interleaved +/// pixels, one pixel at a time. +struct PixelWriter<'s, 'd> { + src: [&'s [f32]; 3], + dst: &'d mut [u8], + channels: usize, +} + +impl U16Writer for PixelWriter<'_, '_> { + #[inline(always)] + fn write(self, convert: impl Fn(f32) -> u16) { + let Self { + src: [red, green, blue], + dst, + channels, + } = self; + let samples = red.iter().zip(green).zip(blue); + for (pixel, ((&red, &green), &blue)) in dst.chunks_exact_mut(channels * 2).zip(samples) { + pixel[..2].copy_from_slice(&convert(red).to_le_bytes()); + pixel[2..4].copy_from_slice(&convert(green).to_le_bytes()); + pixel[4..6].copy_from_slice(&convert(blue).to_le_bytes()); + } } - let max = u16::try_from((1_u32 << u32::from(bit_depth.min(16))) - 1).unwrap_or(u16::MAX); - rounded.clamp(0.0, f32::from(max)) as u16 } pub(in crate::decode) fn write_u16_output( @@ -357,7 +597,91 @@ fn convert_or_copy_u16( #[cfg(test)] mod tests { - use super::{opaque_alpha_u16, widen_u8_sample_to_u16}; + use super::{ + opaque_alpha_u16, widen_u8_sample_to_u16, ChannelWriter, PixelWriter, U16Conversion, + }; + + /// The per-sample conversion the component-plane writers used before + /// they converted a plane at a time. + #[expect( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + reason = "decoded samples are rounded and clamped to the component's declared integer representation" + )] + fn component_sample_as_u16(sample: f32, bit_depth: u8, signed: bool) -> u16 { + let rounded = sample.round(); + if signed { + if bit_depth <= 8 { + let magnitude_bits = u32::from(bit_depth.saturating_sub(1)); + let min = -(1_i16 << magnitude_bits); + let max = (1_i16 << magnitude_bits) - 1; + return widen_u8_sample_to_u16( + (rounded.clamp(f32::from(min), f32::from(max)) as i8) as u8, + bit_depth, + ); + } + let magnitude_bits = u32::from(bit_depth.min(16).saturating_sub(1)); + let min = i16::try_from(-(1_i32 << magnitude_bits)).unwrap_or(i16::MIN); + let max = i16::try_from((1_i32 << magnitude_bits) - 1).unwrap_or(i16::MAX); + return (rounded.clamp(f32::from(min), f32::from(max)) as i16) as u16; + } + if bit_depth <= 8 { + return widen_u8_sample_to_u16(rounded.clamp(0.0, f32::from(u8::MAX)) as u8, bit_depth); + } + let max = u16::try_from((1_u32 << u32::from(bit_depth.min(16))) - 1).unwrap_or(u16::MAX); + rounded.clamp(0.0, f32::from(max)) as u16 + } + + #[test] + #[expect(clippy::cast_precision_loss, reason = "test values fit f32 exactly")] + fn plane_conversion_matches_the_sample_conversion() { + let mut samples = vec![ + f32::NAN, + f32::INFINITY, + f32::NEG_INFINITY, + -0.0, + 1.0e9, + -1.0e9, + ]; + for value in -70_000_i32..70_000 { + for offset in [-0.5, 0.0, 0.25, 0.5] { + samples.push(value as f32 + offset); + } + } + for bit_depth in 1..=16 { + for signed in [false, true] { + let conversion = U16Conversion::new(bit_depth, signed); + let mut out = vec![0; samples.len() * 2]; + conversion.visit(ChannelWriter { + src: &samples, + dst: &mut out, + channels: 1, + channel: 0, + }); + let mut pixels = vec![0; samples.len() * 8]; + conversion.visit(PixelWriter { + src: [&samples; 3], + dst: &mut pixels, + channels: 4, + }); + for (pixel, sample) in pixels.as_chunks::<8>().0.iter().zip(out.as_chunks::<2>().0) + { + assert_eq!( + pixel[..6], + [*sample; 3].concat(), + "{bit_depth} bits, signed {signed}" + ); + } + for (sample, bytes) in samples.iter().zip(out.as_chunks::<2>().0) { + assert_eq!( + u16::from_le_bytes(*bytes), + component_sample_as_u16(*sample, bit_depth, signed), + "{sample} at {bit_depth} bits, signed {signed}" + ); + } + } + } + } #[test] fn eight_bit_samples_widen_across_the_complete_u16_domain() { diff --git a/crates/j2k/src/view.rs b/crates/j2k/src/view.rs index b546a72a..bfb14b75 100644 --- a/crates/j2k/src/view.rs +++ b/crates/j2k/src/view.rs @@ -8,9 +8,10 @@ use crate::{ }, decode::{ decode_full_image_into_with_native_context, decode_image_region_into_with_native_context, - decode_warnings_for_image, full_image_route, validate_buffer, validate_region, - J2kDecodeOutcome, J2kDecodedComponents, J2kDecodedNativeComponents, PlanesBudget, - RouteRequest, + decode_native_components_with_native_context, decode_warnings_for_image, full_image_route, + native_components_route, native_region_components_route, validate_buffer, validate_region, + FullImageRoute, J2kDecodeOutcome, J2kDecodedComponents, J2kDecodedNativeComponents, + PlanesBudget, RouteRequest, }, parse::{parse_image_info, parse_info}, scratch::J2kScratchPool, @@ -249,19 +250,30 @@ impl<'a> J2kDecoder<'a> { /// /// This preserves per-component bit depth, signedness, sampling, and byte /// width for callers that cannot use a single interleaved packed bitmap. + /// A decode whose `f32` component planes would pass the native decode + /// budget runs as a region decode of the whole image, which packs rows + /// into the returned planes as they are finished. /// /// # Errors /// Returns [`J2kError`] when the codestream cannot be decoded. pub fn decode_native_components(&mut self) -> Result { self.ensure_image()?; + let planes_budget = self.full_decode_planes_budget(); let (Some(image), native_context) = (self.image.as_ref(), &mut self.native_context) else { return Err(J2kError::internal_backend("internal image cache missing")); }; - let decoded = image - .decode_native_components_with_context(native_context) - .map_err(J2kError::from_native_decode_error)?; - let retained_image_bytes = component_handoff_image_bytes(image)?; - J2kDecodedNativeComponents::try_from_native(decoded, retained_image_bytes) + let planes_budget = match planes_budget { + PlanesBudget::Default => j2k_native::DEFAULT_MAX_DECODE_BYTES, + PlanesBudget::Fixed(bytes) => bytes, + }; + let route = native_components_route(image, planes_budget); + let decoded = + decode_native_components_with_native_context(image, native_context, None, route); + let retained_image_bytes = component_handoff_image_bytes(image); + if route == FullImageRoute::StreamedRegion { + self.native_context = self.fresh_native_context(); + } + J2kDecodedNativeComponents::try_from_native(decoded?, retained_image_bytes?) } /// Extract selected codestream wavelet bands on the CPU, without inverse @@ -286,6 +298,10 @@ impl<'a> J2kDecoder<'a> { /// Decode a source-coordinate region into owned native-bit-depth component /// planes. /// + /// Like 8- and 16-bit region decodes, it packs rows into the returned + /// planes as they are finished when the image has no palette, channel + /// definition, sYCC or CIELab and its components share one precision. + /// /// # Errors /// Returns [`J2kError`] when the region is invalid or the codestream cannot /// be decoded. @@ -298,14 +314,14 @@ impl<'a> J2kDecoder<'a> { let (Some(image), native_context) = (self.image.as_ref(), &mut self.native_context) else { return Err(J2kError::internal_backend("internal image cache missing")); }; - let decoded = image - .decode_native_region_components_with_context( - (roi.x, roi.y, roi.w, roi.h), - native_context, - ) - .map_err(J2kError::from_native_decode_error)?; - let retained_image_bytes = component_handoff_image_bytes(image)?; - J2kDecodedNativeComponents::try_from_native(decoded, retained_image_bytes) + let route = native_region_components_route(image); + let decoded = + decode_native_components_with_native_context(image, native_context, Some(roi), route); + let retained_image_bytes = component_handoff_image_bytes(image); + if route == FullImageRoute::StreamedRegion { + self.native_context = self.fresh_native_context(); + } + J2kDecodedNativeComponents::try_from_native(decoded?, retained_image_bytes?) } /// Return the CPU decode parallelism policy for this decoder. @@ -597,11 +613,11 @@ impl<'a> J2kDecoder<'a> { .map_or(PlanesBudget::Default, PlanesBudget::Fixed) } - /// Test hook: hold full-image 8-bit decodes' component planes to - /// `bytes` and drop the preference for streaming, so that a decode - /// takes the streamed region route exactly when its planes would pass - /// `bytes`: 0 streams every decode that can stream, `usize::MAX` keeps - /// every decode the planes can finish on the planes. The native decode + /// Test hook: hold full-image decodes' component planes to `bytes` and + /// drop the preference for streaming, so that a decode takes the + /// streamed region route exactly when its planes would pass `bytes`: 0 + /// streams every decode that can stream, `usize::MAX` keeps every decode + /// the planes can finish on the planes. The native decode /// budget itself is unchanged. #[doc(hidden)] pub fn set_full_decode_planes_budget(&mut self, bytes: usize) { diff --git a/crates/j2k/tests/streamed_sixteen_bit_and_native.rs b/crates/j2k/tests/streamed_sixteen_bit_and_native.rs new file mode 100644 index 00000000..df9c7998 --- /dev/null +++ b/crates/j2k/tests/streamed_sixteen_bit_and_native.rs @@ -0,0 +1,363 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! A 16-bit full-image decode and a decode into owned native component +//! planes whose `f32` planes would pass the decode budget run as a region +//! decode of the whole image that streams its rows, as 8-bit decodes do, and +//! 16-bit region decodes stream their rows always. A decoder whose planes +//! budget is lowered takes the streamed route for small images, and must +//! write the bytes the planes route writes for every precision and +//! signedness, at every scale, stride and parallelism, and fail where it +//! fails. + +use j2k::{CpuDecodeParallelism, J2kDecodedNativeComponents, J2kDecoder, J2kScratchPool}; +use j2k_core::{Downscale, PixelFormat, Rect}; +use j2k_native::{encode, EncodeOptions}; +use j2k_test_support::wrap_jp2_rgba_codestream; + +const WIDTH: u32 = 203; +const HEIGHT: u32 = 157; +const PADDING: u8 = 0xab; + +const RGB_FORMATS: &[PixelFormat] = &[ + PixelFormat::Rgb16, + PixelFormat::Rgba16, + PixelFormat::Gray16, + PixelFormat::Rgb8, +]; +const GRAY_FORMATS: &[PixelFormat] = &[PixelFormat::Gray16, PixelFormat::Rgb16, PixelFormat::Gray8]; +const RGBA_FORMATS: &[PixelFormat] = &[PixelFormat::Rgba16, PixelFormat::Rgb16, PixelFormat::Rgba8]; + +/// Noise samples of `bit_depth` bits, one or two little-endian bytes each, +/// which the encoder masks to the precision (and sign-extends when signed). +fn noisy_samples(components: u16, bit_depth: u8) -> Vec { + let bytes = usize::from(bit_depth).div_ceil(8); + let mut state = 0x2545_f491_u32 ^ u32::from(bit_depth); + (0..WIDTH as usize * HEIGHT as usize * usize::from(components) * bytes) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + (state >> 24) as u8 + }) + .collect() +} + +fn codestream(components: u16, bit_depth: u8, signed: bool, options: &EncodeOptions) -> Vec { + let samples = noisy_samples(components, bit_depth); + encode( + &samples, WIDTH, HEIGHT, components, bit_depth, signed, options, + ) + .expect("encode fixture") +} + +/// Images of 4, 8, 12 and 16 bits, signed and unsigned, reversible and +/// irreversible, tiled and untiled, classic and HT, with the output formats +/// each is decoded to (8-bit ones included: they stream past the decode +/// budget too). +fn fixtures() -> Vec<(&'static str, Vec, &'static [PixelFormat])> { + let levels = EncodeOptions { + num_decomposition_levels: 4, + ..EncodeOptions::default() + }; + let irreversible = EncodeOptions { + reversible: false, + ..levels.clone() + }; + let reversible = EncodeOptions { + reversible: true, + ..levels.clone() + }; + vec![ + ( + "rgb 8-bit 9/7", + codestream(3, 8, false, &irreversible), + RGB_FORMATS, + ), + ( + "rgb 8-bit tiled HT 5/3", + codestream( + 3, + 8, + false, + &EncodeOptions { + tile_size: Some((64, 64)), + use_ht_block_coding: true, + ..reversible.clone() + }, + ), + RGB_FORMATS, + ), + ( + "rgb 12-bit signed 9/7", + codestream(3, 12, true, &irreversible), + RGB_FORMATS, + ), + ( + "rgb 16-bit 5/3", + codestream(3, 16, false, &reversible), + RGB_FORMATS, + ), + ( + "gray 12-bit 5/3", + codestream(1, 12, false, &reversible), + GRAY_FORMATS, + ), + ( + "gray 16-bit tiled 9/7", + codestream( + 1, + 16, + false, + &EncodeOptions { + tile_size: Some((96, 64)), + ..irreversible.clone() + }, + ), + GRAY_FORMATS, + ), + ( + "gray 16-bit signed 9/7", + codestream(1, 16, true, &irreversible), + GRAY_FORMATS, + ), + ( + "gray 8-bit signed 5/3", + codestream(1, 8, true, &reversible), + GRAY_FORMATS, + ), + ( + "gray 4-bit 5/3", + codestream(1, 4, false, &reversible), + GRAY_FORMATS, + ), + ( + "rgba 8-bit 9/7", + wrap_jp2_rgba_codestream(&codestream(4, 8, false, &irreversible), WIDTH, HEIGHT, 8), + RGBA_FORMATS, + ), + ] +} + +fn decoder( + bytes: &[u8], + planes_budget: Option, + parallelism: CpuDecodeParallelism, +) -> J2kDecoder<'_> { + let mut decoder = J2kDecoder::new(bytes).expect("decoder"); + decoder.set_cpu_decode_parallelism(parallelism); + if let Some(bytes) = planes_budget { + decoder.set_full_decode_planes_budget(bytes); + } + decoder +} + +/// The image at `scale` in a buffer padded past each row, or `None` when the +/// decode fails, from a decoder whose planes budget is `planes_budget` when +/// one is given. +fn decode( + bytes: &[u8], + planes_budget: Option, + parallelism: CpuDecodeParallelism, + fmt: PixelFormat, + scale: Downscale, + padding: usize, +) -> Option> { + let mut decoder = decoder(bytes, planes_budget, parallelism); + let divisor = scale.denominator(); + let (width, height) = (WIDTH.div_ceil(divisor), HEIGHT.div_ceil(divisor)); + let row = width as usize * fmt.bytes_per_pixel(); + let stride = row + padding; + let mut out = vec![PADDING; stride * height as usize]; + let pool = &mut J2kScratchPool::new(); + let result = match scale { + Downscale::None => decoder.decode_into(&mut out, stride, fmt), + _ => decoder.decode_scaled_into(pool, &mut out, stride, fmt, scale), + }; + result.ok()?; + for line in out.chunks(stride) { + assert!( + line[row..].iter().all(|&byte| byte == PADDING), + "stride padding untouched" + ); + } + Some(out) +} + +#[test] +fn streamed_full_image_outputs_match_planes() { + let mut decoded = 0; + for (label, bytes, formats) in &fixtures() { + for &fmt in *formats { + for scale in [Downscale::None, Downscale::Half, Downscale::Eighth] { + for (parallelism, padding) in [ + (CpuDecodeParallelism::Serial, 6), + (CpuDecodeParallelism::Auto, 0), + ] { + let planes = decode(bytes, None, parallelism, fmt, scale, padding); + let streamed = decode(bytes, Some(0), parallelism, fmt, scale, padding); + assert!( + planes == streamed, + "{label} {fmt:?} {scale:?} {parallelism:?}: planes {} streamed {}", + planes.is_some(), + streamed.is_some() + ); + decoded += usize::from(planes.is_some()); + } + } + } + } + // Gray to RGB and RGB to gray are rejected; the rest must decode. + assert!( + decoded >= (4 * 3 + 5 * 2 + 3) * 3 * 2, + "{decoded} decodes succeeded" + ); +} + +fn native_components( + bytes: &[u8], + planes_budget: Option, + parallelism: CpuDecodeParallelism, +) -> J2kDecodedNativeComponents { + decoder(bytes, planes_budget, parallelism) + .decode_native_components() + .expect("native components") +} + +#[test] +fn streamed_native_components_match_planes() { + for (label, bytes, _) in &fixtures() { + for parallelism in [CpuDecodeParallelism::Serial, CpuDecodeParallelism::Auto] { + let planes = native_components(bytes, None, parallelism); + let streamed = native_components(bytes, Some(0), parallelism); + assert_eq!(planes, streamed, "{label} {parallelism:?}"); + } + } +} + +/// Native component region decodes stream their rows into the planes, +/// which must hold the full image's planes cropped to the region. +#[test] +fn native_region_components_match_the_full_image() { + for (label, bytes, _) in &fixtures() { + for parallelism in [CpuDecodeParallelism::Serial, CpuDecodeParallelism::Auto] { + let full = native_components(bytes, None, parallelism); + for roi in regions() { + let region = decoder(bytes, None, parallelism) + .decode_native_region_components(roi) + .unwrap_or_else(|error| panic!("{label} {roi:?}: {error:?}")); + assert_eq!(region.dimensions(), (roi.w, roi.h), "{label} {roi:?}"); + assert_eq!(region.color_space(), full.color_space(), "{label}"); + assert_eq!(region.planes().len(), full.planes().len(), "{label}"); + for (plane, whole) in region.planes().iter().zip(full.planes()) { + assert_eq!(plane.bit_depth(), whole.bit_depth(), "{label}"); + assert_eq!(plane.signed(), whole.signed(), "{label}"); + let bytes = usize::from(whole.bytes_per_sample()); + let row = roi.w as usize * bytes; + for (y, line) in plane.data().chunks(row).enumerate() { + let start = + ((roi.y as usize + y) * WIDTH as usize + roi.x as usize) * bytes; + assert!( + line == &whole.data()[start..start + row], + "{label} {parallelism:?} {roi:?} row {y}" + ); + } + } + } + } + } +} + +fn regions() -> [Rect; 4] { + [ + Rect::full((WIDTH, HEIGHT)), + Rect { + x: 17, + y: 9, + w: 64, + h: 50, + }, + Rect { + x: WIDTH - 33, + y: HEIGHT - 21, + w: 33, + h: 21, + }, + Rect { + x: 100, + y: 0, + w: 1, + h: HEIGHT, + }, + ] +} + +/// The 16-bit region decodes stream their rows, which must be the rows of +/// the full image the planes decode. +#[test] +fn sixteen_bit_regions_match_the_full_image() { + let pool = &mut J2kScratchPool::new(); + for (label, bytes, formats) in &fixtures() { + for &fmt in formats.iter().filter(|fmt| fmt.bytes_per_pixel() % 2 == 0) { + for parallelism in [CpuDecodeParallelism::Serial, CpuDecodeParallelism::Auto] { + let Some(full) = decode(bytes, None, parallelism, fmt, Downscale::None, 0) else { + continue; + }; + let pixel = fmt.bytes_per_pixel(); + let full_stride = WIDTH as usize * pixel; + for roi in regions() { + let stride = roi.w as usize * pixel + 2; + let mut out = vec![PADDING; stride * roi.h as usize]; + decoder(bytes, None, parallelism) + .decode_region_into(pool, &mut out, stride, fmt, roi) + .unwrap_or_else(|error| panic!("{label} {fmt:?} {roi:?}: {error:?}")); + for (row, line) in out.chunks(stride).enumerate() { + let start = (roi.y as usize + row) * full_stride + roi.x as usize * pixel; + assert!( + line[..roi.w as usize * pixel] + == full[start..start + roi.w as usize * pixel], + "{label} {fmt:?} {parallelism:?} {roi:?} row {row}" + ); + assert_eq!(line[roi.w as usize * pixel..], [PADDING; 2]); + } + } + } + } + } +} + +/// Tiles whose coarse resolutions are empty while finer ones are not keep +/// the planes for 16-bit and native component output too. +#[test] +fn tiles_with_empty_coarse_levels_keep_the_planes() { + let gray: &[u8] = include_bytes!( + "../../j2k-test-support/fixtures/empty_coarse_levels/openjpeg_gray_8x6_tile8x5_53.j2k" + ); + let rgb: &[u8] = include_bytes!( + "../../j2k-test-support/fixtures/empty_coarse_levels/openjpeg_rgb_6x6_tile5x5_97.j2k" + ); + for (label, bytes, fmt) in [ + ("gray 5/3", gray, PixelFormat::Gray16), + ("rgb 9/7", rgb, PixelFormat::Rgb16), + ] { + for parallelism in [CpuDecodeParallelism::Serial, CpuDecodeParallelism::Auto] { + let mut planes = decoder(bytes, None, parallelism); + let mut lowered = decoder(bytes, Some(0), parallelism); + let (width, height) = planes.info().dimensions; + let stride = width as usize * fmt.bytes_per_pixel(); + let mut expected = vec![0; stride * height as usize]; + let mut out = vec![0; stride * height as usize]; + planes + .decode_into(&mut expected, stride, fmt) + .unwrap_or_else(|error| panic!("{label}: {error:?}")); + lowered + .decode_into(&mut out, stride, fmt) + .unwrap_or_else(|error| panic!("{label}: {error:?}")); + assert_eq!(out, expected, "{label} {parallelism:?}"); + assert_eq!( + lowered.decode_native_components().expect("native"), + planes.decode_native_components().expect("native"), + "{label} {parallelism:?}" + ); + } + } +} From 866af5bdaf1dcb4567246f975561f8bc7c92143a Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 03:18:29 -0400 Subject: [PATCH 29/94] fix(j2k): decode 16-bit full images past the decode budget as a streamed region The plan-time router (full_image_route) also streams Rgb16/Rgba16/Gray16 full-image decodes (decode_into, decode_scaled_into and the decoders behind decode_tiles_into) of images whose rows carry their native samples, when the planes route would be refused: by its decode (the native planes floor, as at 8 bits) or by its pack, which holds every f32 plane besides the RawBitmap of native samples (u16_rows_pack_floor, a lower bound of that charge). The streamed route's 16-bit arm is the U16RowSink region decode, so the output is the planes route's. Other 16-bit decodes keep the planes and their errors. The router edit is the 16-bit arm of the format match and the floor.max(pack) comparison. --- crates/j2k/src/decode/output.rs | 4 +- crates/j2k/src/decode/output/u16.rs | 10 +++ crates/j2k/src/decode/route.rs | 28 ++++---- crates/j2k/src/decode/route/tests.rs | 95 ++++++++++++++++++++++++++-- crates/j2k/src/view.rs | 11 ++-- 5 files changed, 127 insertions(+), 21 deletions(-) diff --git a/crates/j2k/src/decode/output.rs b/crates/j2k/src/decode/output.rs index 873dd9d4..3025458c 100644 --- a/crates/j2k/src/decode/output.rs +++ b/crates/j2k/src/decode/output.rs @@ -6,13 +6,15 @@ mod native; mod u16; mod u8; +use native::planes_pack_floor; pub(crate) use native::{ decode_native_components_with_native_context, native_components_route, native_region_components_route, }; pub(super) use u16::{ - rows_match_native_u16, write_components_u16_output, write_u16_output, U16RowSink, + rows_match_native_u16, u16_rows_pack_floor, write_components_u16_output, write_u16_output, + U16RowSink, }; pub(super) use u8::{ can_decode_u8_directly, expand_rgb8_to_rgba8, write_components_u8_output, write_u8_output, diff --git a/crates/j2k/src/decode/output/u16.rs b/crates/j2k/src/decode/output/u16.rs index 03dd2874..1b969bf4 100644 --- a/crates/j2k/src/decode/output/u16.rs +++ b/crates/j2k/src/decode/output/u16.rs @@ -43,6 +43,16 @@ pub(in crate::decode) fn rows_match_native_u16(image: &Image<'_>, fmt: PixelForm ) } +/// The bytes the planes route charges at least when it packs `image`'s +/// native samples for a 16-bit `fmt` (see [`super::planes_pack_floor`]), +/// when a region decode of `image` may stream its rows to `fmt` instead +/// (see [`rows_match_native_u16`]). +pub(in crate::decode) fn u16_rows_pack_floor(image: &Image<'_>, fmt: PixelFormat) -> Option { + let precision = image.streamed_rows_native_precision()?; + rows_match_native_u16(image, fmt) + .then(|| super::planes_pack_floor(image, usize::from(precision).div_ceil(8))) +} + /// Write `planes`, each `dims` in size, to `out` from its first byte. fn write_planes_u16( color_space: &ColorSpace, diff --git a/crates/j2k/src/decode/route.rs b/crates/j2k/src/decode/route.rs index 29cd12f4..e1b40a4c 100644 --- a/crates/j2k/src/decode/route.rs +++ b/crates/j2k/src/decode/route.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -//! Plan-time choice of how a full-image 8-bit decode runs. +//! Plan-time choice of how a full-image 8- or 16-bit decode runs. //! //! The planes route decodes every component into an image-sized plane of //! `f32` samples and packs the output from them. Its working memory grows @@ -18,6 +18,7 @@ use j2k_core::{PixelFormat, Rect}; use j2k_native::CpuDecodeParallelism; +use super::output::u16_rows_pack_floor; use crate::backend::Image; pub(crate) mod tiles; @@ -67,19 +68,24 @@ pub(crate) struct RouteRequest { } /// How a full-image decode of `image` runs for `request`: streamed when the -/// output has 8 bits per sample, the region decode streams, and either the -/// planes would pass the budget or the streamed route is the faster one -/// for this image. +/// output has 8 bits per sample, or 16 bits whose rows carry the native +/// samples the planes route packs (see `u16_rows_pack_floor`), the region +/// decode streams, and either the planes (or their 16-bit pack) would pass +/// the budget or the streamed route is the faster one for this image. /// /// A planning error keeps the planes route, which meets the same error and /// reports it. pub(crate) fn full_image_route(image: &Image<'_>, request: RouteRequest) -> FullImageRoute { - if !matches!( - request.fmt, - PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Gray8 - ) { - return FullImageRoute::Planes; - } + let pack_floor = match request.fmt { + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Gray8 => 0, + PixelFormat::Rgb16 | PixelFormat::Rgba16 | PixelFormat::Gray16 => { + match u16_rows_pack_floor(image, request.fmt) { + Some(floor) => floor, + None => return FullImageRoute::Planes, + } + } + _ => return FullImageRoute::Planes, + }; let planes_budget = match request.budget { PlanesBudget::Default => j2k_native::DEFAULT_MAX_DECODE_BYTES, PlanesBudget::Fixed(bytes) => bytes, @@ -92,7 +98,7 @@ pub(crate) fn full_image_route(image: &Image<'_>, request: RouteRequest) -> Full } else { matches!( image.full_decode_planes_floor(planes_budget), - Ok(Some(floor)) if floor > planes_budget + Ok(Some(floor)) if floor.max(pack_floor) > planes_budget ) }; if streams { diff --git a/crates/j2k/src/decode/route/tests.rs b/crates/j2k/src/decode/route/tests.rs index e8c66870..57540a89 100644 --- a/crates/j2k/src/decode/route/tests.rs +++ b/crates/j2k/src/decode/route/tests.rs @@ -2,6 +2,7 @@ use super::{full_image_route, FullImageRoute, PlanesBudget, RouteRequest}; use crate::backend::image_at_scale; +use crate::decode::native_components_route; use crate::DecodeSettings; use j2k_core::{Downscale, PixelFormat, Rect}; use j2k_native::DecoderContext; @@ -97,15 +98,45 @@ fn reduced_decodes_plan_full_resolution_coefficients() { assert!(half > 96 * 80 * 4, "{half}"); } +/// A 16-bit decode streams when the planes decode or the pack of its native +/// samples, which holds every `f32` plane besides them, would pass the +/// budget. With small tiles and 16-bit samples the pack charges more than +/// the decode. #[test] -fn sixteen_bit_output_keeps_the_planes() { - let bytes = codestream(3, &EncodeOptions::default()); - for fmt in [PixelFormat::Rgb16, PixelFormat::Rgba16, PixelFormat::Gray16] { +fn sixteen_bit_output_streams_when_its_planes_or_their_pack_pass_the_budget() { + let (width, height) = (96_u32, 80_u32); + let samples: Vec = (0..width * height * 3 * 2) + .map(|index| (index.wrapping_mul(2_654_435_761) >> 24) as u8) + .collect(); + let options = EncodeOptions { + num_decomposition_levels: 2, + tile_size: Some((16, 16)), + ..EncodeOptions::default() + }; + let bytes = encode(&samples, width, height, 3, 16, false, &options).expect("encode fixture"); + let floor = floor(&bytes, Downscale::None); + let pack = 96 * 80 * 3 * (4 + 2); + assert!(pack > floor, "{pack} > {floor}"); + for fmt in [PixelFormat::Rgb16, PixelFormat::Rgba16] { + assert_eq!( + route(&bytes, Downscale::None, fmt, pack - 1), + FullImageRoute::StreamedRegion + ); assert_eq!( - route(&bytes, Downscale::None, fmt, 0), + route(&bytes, Downscale::None, fmt, pack), FullImageRoute::Planes ); } + // The 8-bit planes write the output directly, without a pack. + assert_eq!( + route(&bytes, Downscale::None, PixelFormat::Rgb8, pack - 1), + FullImageRoute::Planes + ); + // An RGB image has no 16-bit gray layout; the planes report that. + assert_eq!( + route(&bytes, Downscale::None, PixelFormat::Gray16, 0), + FullImageRoute::Planes + ); } /// A tile whose coarse resolutions are empty while finer ones are not keeps @@ -467,3 +498,59 @@ fn large_gray_images_stream() { assert!(!gray_streams(4096 * 4096, threads), "{threads}"); } } + +/// Images of more than 8 bits and signed images stream at 16 bits and into +/// native component planes once their planes pass the budget. +#[test] +fn deep_and_signed_images_stream_past_the_budget() { + let (width, height) = (64_u32, 48_u32); + for (components, bit_depth, signed, reversible) in [ + (3, 12, true, false), + (3, 16, false, true), + (1, 12, false, true), + (1, 16, true, false), + (1, 4, false, true), + ] { + let samples: Vec = (0..width * height * u32::from(components) * 2) + .map(|index| (index.wrapping_mul(2_654_435_761) >> 24) as u8) + .collect(); + let samples = &samples[..samples.len() / if bit_depth > 8 { 1 } else { 2 }]; + let options = EncodeOptions { + num_decomposition_levels: 3, + reversible, + ..EncodeOptions::default() + }; + let bytes = encode( + samples, width, height, components, bit_depth, signed, &options, + ) + .expect("encode fixture"); + let image = + image_at_scale(&bytes, DecodeSettings::default(), Downscale::None).expect("image"); + assert_eq!(image.streamed_rows_native_precision(), Some(bit_depth)); + let fmt = if components == 1 { + PixelFormat::Gray16 + } else { + PixelFormat::Rgb16 + }; + let label = format!("{components} x {bit_depth} bits, signed {signed}"); + assert_eq!( + full_image_route(&image, fmt, 0), + FullImageRoute::StreamedRegion, + "{label}" + ); + assert_eq!( + native_components_route(&image, 0), + FullImageRoute::StreamedRegion, + "{label}" + ); + let budget = j2k_native::DEFAULT_MAX_DECODE_BYTES; + assert_eq!( + full_image_route(&image, fmt, budget), + FullImageRoute::Planes + ); + assert_eq!( + native_components_route(&image, budget), + FullImageRoute::Planes + ); + } +} diff --git a/crates/j2k/src/view.rs b/crates/j2k/src/view.rs index bfb14b75..0f8725a6 100644 --- a/crates/j2k/src/view.rs +++ b/crates/j2k/src/view.rs @@ -337,9 +337,10 @@ impl<'a> J2kDecoder<'a> { /// Decode the full image into `out` using `stride` bytes per output row. /// - /// An 8-bit decode whose component planes would pass the native decode - /// budget runs as a region decode of the whole image, which writes rows - /// as they are finished instead of holding the planes. + /// A decode whose component planes would pass the native decode budget + /// runs as a region decode of the whole image, which writes rows as they + /// are finished instead of holding the planes: at 8 bits, and at 16 bits + /// for images whose components share one precision of at most 16 bits. /// /// # Errors /// Returns [`J2kError`] when the format is unsupported, the output buffer @@ -494,8 +495,8 @@ impl<'a> J2kDecoder<'a> { /// /// `scale` uses the shared [`Downscale`] contract; `Downscale::None` /// delegates to full-resolution decode. As with [`Self::decode_into`], - /// an 8-bit decode whose component planes would pass the native decode - /// budget runs as a region decode of the whole reduced image. + /// a decode whose component planes would pass the native decode budget + /// runs as a region decode of the whole reduced image. /// /// # Errors /// Returns [`J2kError`] when the format or scale request is unsupported, From cea9ab6da0a4b23cadb180f29e31e2c80fe44ffc Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 03:25:57 -0400 Subject: [PATCH 30/94] perf(j2k): convert 16-bit rows in pixels of a constant size The 16-bit component-plane writers walk the output in 2-, 6- or 8-byte pixels chosen once per row, so the per-sample conversion and interleave compile to fixed-size stores. --- crates/j2k/src/decode/output/u16.rs | 47 +++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/crates/j2k/src/decode/output/u16.rs b/crates/j2k/src/decode/output/u16.rs index 1b969bf4..ddf1f46b 100644 --- a/crates/j2k/src/decode/output/u16.rs +++ b/crates/j2k/src/decode/output/u16.rs @@ -380,12 +380,27 @@ impl U16Writer for ChannelWriter<'_, '_> { channels, channel, } = self; - for (pixel, &sample) in dst.chunks_exact_mut(channels * 2).zip(src) { - pixel[channel * 2..channel * 2 + 2].copy_from_slice(&convert(sample).to_le_bytes()); + // Whole pixels of a constant size, which the compiler vectorizes. + match channels { + 1 => write_channel::<2>(src, dst, channel, convert), + 3 => write_channel::<6>(src, dst, channel, convert), + _ => write_channel::<8>(src, dst, channel, convert), } } } +#[inline(always)] +fn write_channel( + src: &[f32], + dst: &mut [u8], + channel: usize, + convert: impl Fn(f32) -> u16, +) { + for (pixel, &sample) in dst.as_chunks_mut::().0.iter_mut().zip(src) { + pixel[channel * 2..channel * 2 + 2].copy_from_slice(&convert(sample).to_le_bytes()); + } +} + /// Three components' samples into the first three channels of interleaved /// pixels, one pixel at a time. struct PixelWriter<'s, 'd> { @@ -397,20 +412,28 @@ struct PixelWriter<'s, 'd> { impl U16Writer for PixelWriter<'_, '_> { #[inline(always)] fn write(self, convert: impl Fn(f32) -> u16) { - let Self { - src: [red, green, blue], - dst, - channels, - } = self; - let samples = red.iter().zip(green).zip(blue); - for (pixel, ((&red, &green), &blue)) in dst.chunks_exact_mut(channels * 2).zip(samples) { - pixel[..2].copy_from_slice(&convert(red).to_le_bytes()); - pixel[2..4].copy_from_slice(&convert(green).to_le_bytes()); - pixel[4..6].copy_from_slice(&convert(blue).to_le_bytes()); + let Self { src, dst, channels } = self; + match channels { + 3 => write_pixels::<6>(src, dst, convert), + _ => write_pixels::<8>(src, dst, convert), } } } +#[inline(always)] +fn write_pixels( + [red, green, blue]: [&[f32]; 3], + dst: &mut [u8], + convert: impl Fn(f32) -> u16, +) { + let samples = red.iter().zip(green).zip(blue); + for (pixel, ((&red, &green), &blue)) in dst.as_chunks_mut::().0.iter_mut().zip(samples) { + pixel[..2].copy_from_slice(&convert(red).to_le_bytes()); + pixel[2..4].copy_from_slice(&convert(green).to_le_bytes()); + pixel[4..6].copy_from_slice(&convert(blue).to_le_bytes()); + } +} + pub(in crate::decode) fn write_u16_output( color_space: &ColorSpace, has_alpha: bool, From 1a7f159fbcd0fa4333476c5d7006261b14cdf84f Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 03:42:38 -0400 Subject: [PATCH 31/94] perf(j2k-native): pack interleaved native samples a block of pixels at a time decode_native and decode_native_region packed two or more components one sample at a time through push_component_native_sample_bytes (f64 rounding and a Vec push per byte), serially. Two to four f32 components of one signedness and at most 16 bits now pack with the converters extend_component_native_sample_bytes uses for one component (round_clamped_native, equal to the f64 rounding for these bounds), a block of 16384 pixels at a time, the blocks split across the pool for large planes. Other components keep the per-sample pack. Bytes are unchanged (unit test against the per-sample pack, 2-5 components, 1-17 bits, signed and unsigned, serial and parallel). Rgb16 decode_into on the planes route (10 threads, with the next commit): 4096 x 4096 RGB 276 -> 135 ms, its HTJ2K version 180 -> 45 ms, 4096 x 2048 HT mosaic 80 -> 20 ms. --- crates/j2k-native/src/image/native.rs | 1 + .../j2k-native/src/image/native/interleave.rs | 208 ++++++++++++++++++ crates/j2k-native/src/image/output_api.rs | 28 +-- 3 files changed, 223 insertions(+), 14 deletions(-) create mode 100644 crates/j2k-native/src/image/native/interleave.rs diff --git a/crates/j2k-native/src/image/native.rs b/crates/j2k-native/src/image/native.rs index 91d4c5b8..f3a8f863 100644 --- a/crates/j2k-native/src/image/native.rs +++ b/crates/j2k-native/src/image/native.rs @@ -15,6 +15,7 @@ use crate::{ use super::Image; mod allocation; +mod interleave; mod rows; pub(super) use allocation::{try_clone_color_space, NativeOutputBudget}; diff --git a/crates/j2k-native/src/image/native/interleave.rs b/crates/j2k-native/src/image/native/interleave.rs new file mode 100644 index 00000000..90b97daa --- /dev/null +++ b/crates/j2k-native/src/image/native/interleave.rs @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Interleaved native samples of several components, packed a block of +//! pixels at a time. + +use alloc::vec::Vec; + +use super::{round_clamped_native, signed_native_bounds, unsigned_native_max}; +use crate::j2c::ComponentData; + +use super::Image; + +/// Pixels packed per block, and so per task when the pack is split across +/// threads. +const BLOCK_PIXELS: usize = 16 * 1024; + +impl Image<'_> { + /// Append samples `0..pixel_count` of every component, pixel by pixel, + /// as packed little-endian native samples of `bit_depth`. Produces the + /// same bytes as [`Self::push_component_native_sample_bytes`] for every + /// index and component, but two to four `f32` components of one + /// signedness and at most 16 bits are converted as + /// [`Self::extend_component_native_sample_bytes`] converts one, a block + /// of pixels at a time, the blocks split across threads when `parallel`. + #[expect( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + reason = "samples are rounded and clamped to the component's 8- or 16-bit range before packing" + )] + pub(crate) fn extend_interleaved_native_sample_bytes( + out: &mut Vec, + components: &[ComponentData], + pixel_count: usize, + bit_depth: u8, + parallel: bool, + ) { + let Some(planes) = float_planes(components, pixel_count, bit_depth) else { + for index in 0..pixel_count { + for component in components { + Self::push_component_native_sample_bytes(out, component, index, bit_depth); + } + } + return; + }; + let planes = &planes[..components.len()]; + let start = out.len(); + let bytes = if bit_depth <= 8 { 1 } else { 2 }; + // Callers reserve the full output first, so this does not reallocate. + out.resize(start + pixel_count * components.len() * bytes, 0); + let destination = &mut out[start..]; + match (bit_depth, components[0].signed) { + (1..=8, true) => { + let (min, max) = signed_native_bounds(bit_depth); + interleave_converted(destination, planes, parallel, move |sample| { + [(round_clamped_native(sample, min, max) as i8) as u8] + }); + } + (1..=8, false) => { + let max = unsigned_native_max(bit_depth); + interleave_converted(destination, planes, parallel, move |sample| { + [round_clamped_native(sample, 0.0, max) as u8] + }); + } + (_, true) => { + let (min, max) = signed_native_bounds(bit_depth); + interleave_converted(destination, planes, parallel, move |sample| { + ((round_clamped_native(sample, min, max) as i16) as u16).to_le_bytes() + }); + } + (_, false) => { + let max = unsigned_native_max(bit_depth); + interleave_converted(destination, planes, parallel, move |sample| { + (round_clamped_native(sample, 0.0, max) as u16).to_le_bytes() + }); + } + } + } +} + +/// The first `pixel_count` samples of each component, when there are two +/// to four `f32` components of one signedness and `bit_depth` is 1 to 16. +fn float_planes( + components: &[ComponentData], + pixel_count: usize, + bit_depth: u8, +) -> Option<[&[f32]; 4]> { + let first = components.first()?; + if !(2..=4).contains(&components.len()) || !(1..=16).contains(&bit_depth) { + return None; + } + let mut planes: [&[f32]; 4] = [&[]; 4]; + for (plane, component) in planes.iter_mut().zip(components) { + if component.integer_container.is_some() || component.signed != first.signed { + return None; + } + *plane = component.container.truncated().get(..pixel_count)?; + } + Some(planes) +} + +/// Write `convert(sample)` of every plane's sample, pixel by pixel, to +/// `destination`, a block of pixels at a time, the blocks split across +/// threads when `parallel` and the planes are large. +fn interleave_converted( + destination: &mut [u8], + planes: &[&[f32]], + parallel: bool, + convert: impl Fn(f32) -> [u8; N] + Send + Sync, +) { + let pixel_bytes = planes.len() * N; + let block = |(index, destination): (usize, &mut [u8])| { + let first = index * BLOCK_PIXELS; + for (offset, pixel) in destination.chunks_exact_mut(pixel_bytes).enumerate() { + for (samples, bytes) in planes.iter().zip(pixel.as_chunks_mut::().0) { + *bytes = convert(samples[first + offset]); + } + } + }; + #[cfg(feature = "parallel")] + if parallel && planes[0].len() >= super::PARALLEL_PACK_SAMPLES { + use rayon::prelude::*; + destination + .par_chunks_mut(pixel_bytes * BLOCK_PIXELS) + .enumerate() + .for_each(block); + return; + } + #[cfg(not(feature = "parallel"))] + let _ = parallel; + destination + .chunks_mut(pixel_bytes * BLOCK_PIXELS) + .enumerate() + .for_each(block); +} + +#[cfg(test)] +mod tests { + use alloc::vec::Vec; + + use crate::image::Image; + use crate::j2c::ComponentData; + use crate::math::{SimdBuffer, SIMD_WIDTH}; + + /// Samples around the bounds and rounding ties of 1 to 16 bits. + #[expect(clippy::cast_precision_loss, reason = "test values fit f32 exactly")] + fn samples(count: usize) -> Vec { + let mut samples = Vec::from([f32::NAN, f32::INFINITY, f32::NEG_INFINITY, -0.0]); + let mut value = -70_000_i32; + while samples.len() < count { + samples.extend([value as f32, value as f32 + 0.5, value as f32 - 0.5]); + value += 97; + } + samples.truncate(count); + samples + } + + #[test] + fn interleaved_pack_matches_the_per_sample_pack() { + let count = 5000; + let base = samples(count); + for components in 2..=5 { + for bit_depth in [1, 4, 8, 9, 12, 16, 17] { + for signed in [false, true] { + for parallel in [false, true] { + let planes: Vec = (0..components) + .map(|component| { + let mut samples = base.clone(); + samples.rotate_left(component * 7); + ComponentData { + container: SimdBuffer::::new(samples), + integer_container: None, + bit_depth, + signed, + } + }) + .collect(); + let mut expected = Vec::new(); + for index in 0..count { + for component in &planes { + Image::push_component_native_sample_bytes( + &mut expected, + component, + index, + bit_depth, + ); + } + } + let mut packed = Vec::from([7_u8]); + packed.reserve(expected.len()); + Image::extend_interleaved_native_sample_bytes( + &mut packed, + &planes, + count, + bit_depth, + parallel, + ); + assert_eq!( + packed[1..], + expected[..], + "{components} x {bit_depth} bits, signed {signed}, parallel {parallel}" + ); + assert_eq!(packed[0], 7); + } + } + } + } + } +} diff --git a/crates/j2k-native/src/image/output_api.rs b/crates/j2k-native/src/image/output_api.rs index f883c3cb..a1a8930b 100644 --- a/crates/j2k-native/src/image/output_api.rs +++ b/crates/j2k-native/src/image/output_api.rs @@ -198,13 +198,13 @@ impl<'a> Image<'a> { parallel, ); } else { - for index in 0..pixel_count { - for component in components { - Self::push_component_native_sample_bytes( - &mut data, component, index, bit_depth, - ); - } - } + Self::extend_interleaved_native_sample_bytes( + &mut data, + components, + pixel_count, + bit_depth, + parallel, + ); } if data.len() != capacity { bail!(DecodingError::CodeBlockDecodeFailure); @@ -288,13 +288,13 @@ impl<'a> Image<'a> { parallel, ); } else { - for index in 0..pixel_count { - for component in components { - Self::push_component_native_sample_bytes( - &mut data, component, index, bit_depth, - ); - } - } + Self::extend_interleaved_native_sample_bytes( + &mut data, + components, + pixel_count, + bit_depth, + parallel, + ); } if data.len() != capacity { bail!(DecodingError::CodeBlockDecodeFailure); From 7532ee2b3967131e4d5cd16a0c341b44dc3fa779 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 03:42:38 -0400 Subject: [PATCH 32/94] perf(j2k): widen 8-bit native samples to 16 bits from a table write_u16_output widened every 8-bit native sample with a u32 division. It now looks the widened value up in a 256-entry table computed once per call (widened_u8_samples, shared with the row sink's conversion). Output is unchanged. --- crates/j2k/src/decode/output/u16.rs | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/crates/j2k/src/decode/output/u16.rs b/crates/j2k/src/decode/output/u16.rs index ddf1f46b..094e3adb 100644 --- a/crates/j2k/src/decode/output/u16.rs +++ b/crates/j2k/src/decode/output/u16.rs @@ -303,9 +303,7 @@ impl U16Conversion { return Self { kind, bounds, - widened: core::array::from_fn(|byte| { - widen_u8_sample_to_u16(u8::try_from(byte).unwrap_or(u8::MAX), bit_depth) - }), + widened: widened_u8_samples(bit_depth), }; } let (kind, bounds) = if signed { @@ -553,6 +551,7 @@ fn write_u16_channel_rows(job: U16ChannelRows<'_, '_>) { let src_row_bytes = width * source_channels * bytes_per_sample; let dst_row_bytes = width * dst_channels * 2; let alpha = opaque_alpha_u16(bytes_per_sample, bit_depth); + let widened = widened_u8_samples(bit_depth); let dst_rows = output_rows(out, stride, dst_row_bytes, height); for (src_row, dst_row) in src.chunks_exact(src_row_bytes).zip(dst_rows) { @@ -560,7 +559,7 @@ fn write_u16_channel_rows(job: U16ChannelRows<'_, '_>) { let src_pixel = &src_row[x * source_channels * bytes_per_sample..]; let dst_pixel = &mut dst_row[x * dst_channels * 2..(x + 1) * dst_channels * 2]; for channel in 0..3 { - let sample = output_u16_sample(src_pixel, channel, bytes_per_sample, bit_depth); + let sample = output_u16_sample(src_pixel, channel, bytes_per_sample, &widened); dst_pixel[channel * 2..channel * 2 + 2].copy_from_slice(&sample.to_le_bytes()); } match layout { @@ -569,7 +568,7 @@ fn write_u16_channel_rows(job: U16ChannelRows<'_, '_>) { dst_pixel[6..8].copy_from_slice(&alpha.to_le_bytes()); } U16ChannelLayout::Preserve => { - let sample = output_u16_sample(src_pixel, 3, bytes_per_sample, bit_depth); + let sample = output_u16_sample(src_pixel, 3, bytes_per_sample, &widened); dst_pixel[6..8].copy_from_slice(&sample.to_le_bytes()); } } @@ -585,17 +584,18 @@ fn opaque_alpha_u16(bytes_per_sample: usize, bit_depth: u8) -> u16 { } } +/// `widened` is [`widened_u8_samples`] of the samples' bit depth. fn output_u16_sample( src_pixel: &[u8], channel: usize, bytes_per_sample: usize, - bit_depth: u8, + widened: &[u16; 256], ) -> u16 { let offset = channel * bytes_per_sample; if bytes_per_sample == 2 { return u16::from_le_bytes([src_pixel[offset], src_pixel[offset + 1]]); } - widen_u8_sample_to_u16(src_pixel[offset], bit_depth) + widened[usize::from(src_pixel[offset])] } fn widen_u8_sample_to_u16(sample: u8, bit_depth: u8) -> u16 { @@ -604,6 +604,14 @@ fn widen_u8_sample_to_u16(sample: u8, bit_depth: u8) -> u16 { .unwrap_or(u16::MAX) } +/// [`widen_u8_sample_to_u16`] of every byte at `bit_depth`, so that a +/// conversion looks each sample up instead of dividing. +fn widened_u8_samples(bit_depth: u8) -> [u16; 256] { + core::array::from_fn(|byte| { + widen_u8_sample_to_u16(u8::try_from(byte).unwrap_or(u8::MAX), bit_depth) + }) +} + fn convert_or_copy_u16( src: &[u8], bytes_per_sample: u8, @@ -616,6 +624,7 @@ fn convert_or_copy_u16( let (width, height) = dims; let dst_row_bytes = width * channels * 2; let src_row_bytes = width * channels * usize::from(bytes_per_sample); + let widened = widened_u8_samples(bit_depth); let dst_rows = output_rows(out, stride, dst_row_bytes, height); for (src_row, dst_row) in src.chunks_exact(src_row_bytes).zip(dst_rows) { if bytes_per_sample == 2 { @@ -623,7 +632,7 @@ fn convert_or_copy_u16( continue; } for (sample, dst_sample) in src_row.iter().zip(dst_row.as_chunks_mut::<2>().0) { - *dst_sample = widen_u8_sample_to_u16(*sample, bit_depth).to_le_bytes(); + *dst_sample = widened[usize::from(*sample)].to_le_bytes(); } } } From 53aa54b674d18e7caa50eb43bbc6319bd216383b Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 03:55:27 -0400 Subject: [PATCH 33/94] perf(j2k): keep gray region decodes within the budget on their planes 16-bit region decodes and decode_native_region_components stream their rows for RGB images, where the rows finish first, and for other images only when the region's f32 planes and packed native samples would pass the native decode budget. A gray region within the budget keeps the planes: one component gives the streamed rows' Tier-1 too little work per row on a large pool (dev timing, 10 threads: 12-bit 8192^2 gray full region 648 ms streamed vs 605 ms on the planes; RGB 4096^2 full region 127 vs 150 ms). A routed full-image decode always streams, through the new decode_image_region_rows_into_with_native_context. Output is unchanged. --- crates/j2k/src/decode.rs | 86 +++++++++++++++++--------- crates/j2k/src/decode/output.rs | 6 +- crates/j2k/src/decode/output/native.rs | 52 ++++++++++++---- crates/j2k/src/decode/output/u16.rs | 16 +++++ crates/j2k/src/view.rs | 10 +-- 5 files changed, 123 insertions(+), 47 deletions(-) diff --git a/crates/j2k/src/decode.rs b/crates/j2k/src/decode.rs index 5f74b9e4..f6fa1ab8 100644 --- a/crates/j2k/src/decode.rs +++ b/crates/j2k/src/decode.rs @@ -20,7 +20,7 @@ mod route; mod settings; mod srgb8; use output::{ - can_decode_u8_directly, expand_rgb8_to_rgba8, rows_match_native_u16, + can_decode_u8_directly, expand_rgb8_to_rgba8, region_rows_preferred_u16, rows_match_native_u16, write_components_u16_output, write_components_u8_output, write_u16_output, write_u8_output, U16RowSink, U8RowSink, }; @@ -83,7 +83,7 @@ pub(crate) fn decode_full_image_into_with_native_context<'a>( FullImageRoute::Planes => { decode_image_into_with_native_context(image, native_context, out, stride, fmt) } - FullImageRoute::StreamedRegion => decode_image_region_into_with_native_context( + FullImageRoute::StreamedRegion => decode_image_region_rows_into_with_native_context( image, native_context, out, @@ -175,35 +175,26 @@ pub(crate) fn decode_image_region_into_with_native_context<'a>( ) -> Result<(), J2kError> { match fmt { PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Gray8 => { - // A serial decode writes rows as they are finished instead of - // holding the region's component planes. A decode that fails - // may have written some of them. - let mut sink = U8RowSink::new(image.color_space(), image.has_alpha(), out, stride, fmt); - image - .decode_region_rows_for_integer_output_with_context( - (roi.x, roi.y, roi.w, roi.h), - native_context, - &mut sink, - ) - .map_err(J2kError::from_native_decode_error)?; - sink.finish() + decode_image_region_rows_into_with_native_context( + image, + native_context, + out, + stride, + fmt, + roi, + ) } PixelFormat::Rgb16 | PixelFormat::Rgba16 | PixelFormat::Gray16 - if rows_match_native_u16(image, fmt) => + if region_rows_preferred_u16(image, fmt, roi) => { - // As at 8 bits, the rows go to `out` as they are finished; the - // sink converts them as `write_u16_output` converts the native - // samples of the same region. - let mut sink = - U16RowSink::new(image.color_space(), image.has_alpha(), out, stride, fmt); - image - .decode_region_rows_for_integer_output_with_context( - (roi.x, roi.y, roi.w, roi.h), - native_context, - &mut sink, - ) - .map_err(J2kError::from_native_decode_error)?; - sink.finish() + decode_image_region_rows_into_with_native_context( + image, + native_context, + out, + stride, + fmt, + roi, + ) } PixelFormat::Rgb16 | PixelFormat::Rgba16 | PixelFormat::Gray16 => { let raw = image @@ -225,6 +216,45 @@ pub(crate) fn decode_image_region_into_with_native_context<'a>( } } +/// Decode `roi` of `image` into `out` through a row sink, which writes +/// rows as they are finished instead of holding the region's component +/// planes: always at 8 bits, and at 16 bits when the rows carry the native +/// samples (see `rows_match_native_u16`), whose conversion the sink then +/// matches. A decode that fails may have written some rows. +pub(crate) fn decode_image_region_rows_into_with_native_context<'a>( + image: &Image<'a>, + native_context: &mut j2k_native::DecoderContext<'a>, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + roi: Rect, +) -> Result<(), J2kError> { + let roi = (roi.x, roi.y, roi.w, roi.h); + match fmt { + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Gray8 => { + let mut sink = U8RowSink::new(image.color_space(), image.has_alpha(), out, stride, fmt); + image + .decode_region_rows_for_integer_output_with_context(roi, native_context, &mut sink) + .map_err(J2kError::from_native_decode_error)?; + sink.finish() + } + PixelFormat::Rgb16 | PixelFormat::Rgba16 | PixelFormat::Gray16 + if rows_match_native_u16(image, fmt) => + { + let mut sink = + U16RowSink::new(image.color_space(), image.has_alpha(), out, stride, fmt); + image + .decode_region_rows_for_integer_output_with_context(roi, native_context, &mut sink) + .map_err(J2kError::from_native_decode_error)?; + sink.finish() + } + _ => Err(Unsupported { + what: "pixel format cannot stream its rows for this image", + } + .into()), + } +} + pub(crate) fn decode_prepared_image_region_into( decoder: &mut j2k_native::PreparedRegionDecoder<'_, '_, '_>, out: &mut [u8], diff --git a/crates/j2k/src/decode/output.rs b/crates/j2k/src/decode/output.rs index 3025458c..03958570 100644 --- a/crates/j2k/src/decode/output.rs +++ b/crates/j2k/src/decode/output.rs @@ -6,15 +6,15 @@ mod native; mod u16; mod u8; -use native::planes_pack_floor; pub(crate) use native::{ decode_native_components_with_native_context, native_components_route, native_region_components_route, }; +use native::{planes_pack_floor, region_rows_preferred}; pub(super) use u16::{ - rows_match_native_u16, u16_rows_pack_floor, write_components_u16_output, write_u16_output, - U16RowSink, + region_rows_preferred_u16, rows_match_native_u16, u16_rows_pack_floor, + write_components_u16_output, write_u16_output, U16RowSink, }; pub(super) use u8::{ can_decode_u8_directly, expand_rgb8_to_rgba8, write_components_u8_output, write_u8_output, diff --git a/crates/j2k/src/decode/output/native.rs b/crates/j2k/src/decode/output/native.rs index f6234fb8..34505c26 100644 --- a/crates/j2k/src/decode/output/native.rs +++ b/crates/j2k/src/decode/output/native.rs @@ -3,7 +3,7 @@ //! Plan-time choice of how a full-image decode into owned native-bit-depth //! component planes runs, and the planes route's charge for packing them. -use crate::backend::Image; +use crate::backend::{ColorSpace, Image}; use crate::decode::FullImageRoute; use crate::J2kError; use j2k_core::Rect; @@ -15,10 +15,19 @@ use j2k_core::Rect; /// the decode budget. Counts the components of `image`'s color space and /// alpha only, so it is at most the real charge. pub(in crate::decode) fn planes_pack_floor(image: &Image<'_>, bytes_per_sample: usize) -> usize { + region_pack_floor( + image, + Rect::full((image.width(), image.height())), + bytes_per_sample, + ) +} + +/// [`planes_pack_floor`] for the planes of `roi` of `image`. +fn region_pack_floor(image: &Image<'_>, roi: Rect, bytes_per_sample: usize) -> usize { let components = usize::from(image.color_space().num_channels()) .saturating_add(usize::from(image.has_alpha())); - (image.width() as usize) - .saturating_mul(image.height() as usize) + (roi.w as usize) + .saturating_mul(roi.h as usize) .saturating_mul(components) .saturating_mul(bytes_per_sample.saturating_add(size_of::())) } @@ -44,15 +53,34 @@ pub(crate) fn native_components_route(image: &Image<'_>, planes_budget: usize) - } } -/// How a region decode of `image` into owned native component planes runs: -/// as one that packs its rows into the planes as they are finished whenever -/// its rows carry the native samples, as 8- and 16-bit region decodes -/// stream theirs, so that it holds no region-sized `f32` planes. -pub(crate) fn native_region_components_route(image: &Image<'_>) -> FullImageRoute { - if image.streamed_rows_native_precision().is_some() { - FullImageRoute::StreamedRegion - } else { - FullImageRoute::Planes +/// Whether a region decode of `roi` of `image`, whose rows carry its native +/// samples of `bytes_per_sample` bytes, streams them rather than packing +/// region-sized planes: always for an RGB image, whose streamed rows finish +/// first; for others when the planes and their packed samples (see +/// [`planes_pack_floor`]) would pass the native decode budget. One component +/// gives the streamed rows' Tier-1 too little work per row on a large pool, +/// so a gray region within the budget keeps the planes. +pub(in crate::decode) fn region_rows_preferred( + image: &Image<'_>, + roi: Rect, + bytes_per_sample: usize, +) -> bool { + matches!(image.color_space(), ColorSpace::RGB) + || region_pack_floor(image, roi, bytes_per_sample) > j2k_native::DEFAULT_MAX_DECODE_BYTES +} + +/// How a region decode of `roi` of `image` into owned native component +/// planes runs: as one that packs its rows into the planes as they are +/// finished, so that it holds no region-sized `f32` planes, when its rows +/// carry the native samples and [`region_rows_preferred`]. +pub(crate) fn native_region_components_route(image: &Image<'_>, roi: Rect) -> FullImageRoute { + match image.streamed_rows_native_precision() { + Some(precision) + if region_rows_preferred(image, roi, usize::from(precision).div_ceil(8)) => + { + FullImageRoute::StreamedRegion + } + _ => FullImageRoute::Planes, } } diff --git a/crates/j2k/src/decode/output/u16.rs b/crates/j2k/src/decode/output/u16.rs index 094e3adb..69e6d396 100644 --- a/crates/j2k/src/decode/output/u16.rs +++ b/crates/j2k/src/decode/output/u16.rs @@ -53,6 +53,22 @@ pub(in crate::decode) fn u16_rows_pack_floor(image: &Image<'_>, fmt: PixelFormat .then(|| super::planes_pack_floor(image, usize::from(precision).div_ceil(8))) } +/// Whether a 16-bit region decode of `roi` of `image` to `fmt` streams its +/// rows (see [`rows_match_native_u16`] and [`super::region_rows_preferred`]) +/// rather than packing the region's native samples. +pub(in crate::decode) fn region_rows_preferred_u16( + image: &Image<'_>, + fmt: PixelFormat, + roi: j2k_core::Rect, +) -> bool { + image + .streamed_rows_native_precision() + .is_some_and(|precision| { + rows_match_native_u16(image, fmt) + && super::region_rows_preferred(image, roi, usize::from(precision).div_ceil(8)) + }) +} + /// Write `planes`, each `dims` in size, to `out` from its first byte. fn write_planes_u16( color_space: &ColorSpace, diff --git a/crates/j2k/src/view.rs b/crates/j2k/src/view.rs index 0f8725a6..2f30b561 100644 --- a/crates/j2k/src/view.rs +++ b/crates/j2k/src/view.rs @@ -298,9 +298,11 @@ impl<'a> J2kDecoder<'a> { /// Decode a source-coordinate region into owned native-bit-depth component /// planes. /// - /// Like 8- and 16-bit region decodes, it packs rows into the returned - /// planes as they are finished when the image has no palette, channel - /// definition, sYCC or CIELab and its components share one precision. + /// Like 16-bit region decodes, it packs rows into the returned planes as + /// they are finished, without region-sized `f32` planes, for RGB images + /// without palette, channel definition, sYCC or CIELab whose components + /// share one precision, and for other such images when the planes would + /// pass the native decode budget. /// /// # Errors /// Returns [`J2kError`] when the region is invalid or the codestream cannot @@ -314,7 +316,7 @@ impl<'a> J2kDecoder<'a> { let (Some(image), native_context) = (self.image.as_ref(), &mut self.native_context) else { return Err(J2kError::internal_backend("internal image cache missing")); }; - let route = native_region_components_route(image); + let route = native_region_components_route(image, roi); let decoded = decode_native_components_with_native_context(image, native_context, Some(roi), route); let retained_image_bytes = component_handoff_image_bytes(image); From fe8ea227ac1c230a6025ac5548485f8a563eeace Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 04:06:37 -0400 Subject: [PATCH 34/94] refactor(j2k): route native component decodes with their own enum The native component routes (planes, or rows packed into the planes) no longer reuse FullImageRoute, whose variants belong to the full-image router, so that router can grow routes of its own without touching the native component decodes. --- crates/j2k/src/decode.rs | 2 +- crates/j2k/src/decode/output.rs | 2 +- crates/j2k/src/decode/output/native.rs | 35 +++++++++++++++----------- crates/j2k/src/decode/route/tests.rs | 9 +++---- crates/j2k/src/view.rs | 10 ++++---- 5 files changed, 31 insertions(+), 27 deletions(-) diff --git a/crates/j2k/src/decode.rs b/crates/j2k/src/decode.rs index f6fa1ab8..6a3bcb20 100644 --- a/crates/j2k/src/decode.rs +++ b/crates/j2k/src/decode.rs @@ -14,7 +14,7 @@ pub use component_handoff::{ mod output; pub(crate) use output::{ decode_native_components_with_native_context, native_components_route, - native_region_components_route, + native_region_components_route, NativeRoute, }; mod route; mod settings; diff --git a/crates/j2k/src/decode/output.rs b/crates/j2k/src/decode/output.rs index 03958570..0c4c88f9 100644 --- a/crates/j2k/src/decode/output.rs +++ b/crates/j2k/src/decode/output.rs @@ -8,7 +8,7 @@ mod u8; pub(crate) use native::{ decode_native_components_with_native_context, native_components_route, - native_region_components_route, + native_region_components_route, NativeRoute, }; use native::{planes_pack_floor, region_rows_preferred}; diff --git a/crates/j2k/src/decode/output/native.rs b/crates/j2k/src/decode/output/native.rs index 34505c26..9b135ab9 100644 --- a/crates/j2k/src/decode/output/native.rs +++ b/crates/j2k/src/decode/output/native.rs @@ -4,10 +4,19 @@ //! component planes runs, and the planes route's charge for packing them. use crate::backend::{ColorSpace, Image}; -use crate::decode::FullImageRoute; use crate::J2kError; use j2k_core::Rect; +/// How a decode into owned native component planes runs. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum NativeRoute { + /// Component planes of the image or region, packed once decoded. + Planes, + /// A region decode that packs its rows into the planes as they are + /// finished and holds no component planes. + Rows, +} + /// The bytes the planes route charges at least when it packs `image`'s /// output from its component planes at `bytes_per_sample` native bytes per /// sample: every `f32` plane and the packed samples. It holds both at once, @@ -42,14 +51,14 @@ fn region_pack_floor(image: &Image<'_>, roi: Rect, bytes_per_sample: usize) -> u /// /// A planning error keeps the planes route, which meets the same error and /// reports it. -pub(crate) fn native_components_route(image: &Image<'_>, planes_budget: usize) -> FullImageRoute { +pub(crate) fn native_components_route(image: &Image<'_>, planes_budget: usize) -> NativeRoute { let Some(precision) = image.streamed_rows_native_precision() else { - return FullImageRoute::Planes; + return NativeRoute::Planes; }; let pack_floor = planes_pack_floor(image, usize::from(precision).div_ceil(8)); match image.full_decode_planes_floor(planes_budget) { - Ok(Some(floor)) if floor.max(pack_floor) > planes_budget => FullImageRoute::StreamedRegion, - _ => FullImageRoute::Planes, + Ok(Some(floor)) if floor.max(pack_floor) > planes_budget => NativeRoute::Rows, + _ => NativeRoute::Planes, } } @@ -73,14 +82,14 @@ pub(in crate::decode) fn region_rows_preferred( /// planes runs: as one that packs its rows into the planes as they are /// finished, so that it holds no region-sized `f32` planes, when its rows /// carry the native samples and [`region_rows_preferred`]. -pub(crate) fn native_region_components_route(image: &Image<'_>, roi: Rect) -> FullImageRoute { +pub(crate) fn native_region_components_route(image: &Image<'_>, roi: Rect) -> NativeRoute { match image.streamed_rows_native_precision() { Some(precision) if region_rows_preferred(image, roi, usize::from(precision).div_ceil(8)) => { - FullImageRoute::StreamedRegion + NativeRoute::Rows } - _ => FullImageRoute::Planes, + _ => NativeRoute::Planes, } } @@ -91,17 +100,15 @@ pub(crate) fn decode_native_components_with_native_context<'a>( image: &Image<'a>, native_context: &mut j2k_native::DecoderContext<'a>, roi: Option, - route: FullImageRoute, + route: NativeRoute, ) -> Result { match (roi, route) { - (None, FullImageRoute::Planes) => { - image.decode_native_components_with_context(native_context) - } - (Some(roi), FullImageRoute::Planes) => image.decode_native_region_components_with_context( + (None, NativeRoute::Planes) => image.decode_native_components_with_context(native_context), + (Some(roi), NativeRoute::Planes) => image.decode_native_region_components_with_context( (roi.x, roi.y, roi.w, roi.h), native_context, ), - (roi, FullImageRoute::StreamedRegion) => { + (roi, NativeRoute::Rows) => { let roi = roi.unwrap_or_else(|| Rect::full((image.width(), image.height()))); image.decode_native_region_components_rows_with_context( (roi.x, roi.y, roi.w, roi.h), diff --git a/crates/j2k/src/decode/route/tests.rs b/crates/j2k/src/decode/route/tests.rs index 57540a89..ac59d7c2 100644 --- a/crates/j2k/src/decode/route/tests.rs +++ b/crates/j2k/src/decode/route/tests.rs @@ -2,7 +2,7 @@ use super::{full_image_route, FullImageRoute, PlanesBudget, RouteRequest}; use crate::backend::image_at_scale; -use crate::decode::native_components_route; +use crate::decode::{native_components_route, NativeRoute}; use crate::DecodeSettings; use j2k_core::{Downscale, PixelFormat, Rect}; use j2k_native::DecoderContext; @@ -540,7 +540,7 @@ fn deep_and_signed_images_stream_past_the_budget() { ); assert_eq!( native_components_route(&image, 0), - FullImageRoute::StreamedRegion, + NativeRoute::Rows, "{label}" ); let budget = j2k_native::DEFAULT_MAX_DECODE_BYTES; @@ -548,9 +548,6 @@ fn deep_and_signed_images_stream_past_the_budget() { full_image_route(&image, fmt, budget), FullImageRoute::Planes ); - assert_eq!( - native_components_route(&image, budget), - FullImageRoute::Planes - ); + assert_eq!(native_components_route(&image, budget), NativeRoute::Planes); } } diff --git a/crates/j2k/src/view.rs b/crates/j2k/src/view.rs index 2f30b561..93c8fb18 100644 --- a/crates/j2k/src/view.rs +++ b/crates/j2k/src/view.rs @@ -10,7 +10,7 @@ use crate::{ decode_full_image_into_with_native_context, decode_image_region_into_with_native_context, decode_native_components_with_native_context, decode_warnings_for_image, full_image_route, native_components_route, native_region_components_route, validate_buffer, validate_region, - FullImageRoute, J2kDecodeOutcome, J2kDecodedComponents, J2kDecodedNativeComponents, + J2kDecodeOutcome, J2kDecodedComponents, J2kDecodedNativeComponents, NativeRoute, PlanesBudget, RouteRequest, }, parse::{parse_image_info, parse_info}, @@ -270,7 +270,7 @@ impl<'a> J2kDecoder<'a> { let decoded = decode_native_components_with_native_context(image, native_context, None, route); let retained_image_bytes = component_handoff_image_bytes(image); - if route == FullImageRoute::StreamedRegion { + if route == NativeRoute::Rows { self.native_context = self.fresh_native_context(); } J2kDecodedNativeComponents::try_from_native(decoded?, retained_image_bytes?) @@ -320,7 +320,7 @@ impl<'a> J2kDecoder<'a> { let decoded = decode_native_components_with_native_context(image, native_context, Some(roi), route); let retained_image_bytes = component_handoff_image_bytes(image); - if route == FullImageRoute::StreamedRegion { + if route == NativeRoute::Rows { self.native_context = self.fresh_native_context(); } J2kDecodedNativeComponents::try_from_native(decoded?, retained_image_bytes?) @@ -607,8 +607,8 @@ impl<'a> J2kDecoder<'a> { .ok_or_else(|| J2kError::internal_backend("internal image cache missing")) } - /// What a full-image 8-bit decode's component planes may charge: the - /// native decode budget, past which the planes decode is refused, with + /// What a full-image decode's component planes may charge: the native + /// decode budget, past which the planes decode is refused, with /// streaming wherever it is the faster route, unless a test set a fixed /// budget. fn full_decode_planes_budget(&self) -> PlanesBudget { From f5b39c8b8604a4f0bec6163183d015a7fb9f658c Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 04:36:45 -0400 Subject: [PATCH 35/94] fix(j2k): convert samples deeper than 16 bits by value in 8- and 16-bit outputs A 16-bit decode of an image of 17 to 24 bits (3-byte native samples) went through write_u16_output, which widened each byte of a native sample as an 8-bit sample of its own, so the output was a byte-level scramble of the image. Scaled decodes, which convert component planes, already clamped each value to the 16-bit range. Full-image, region and >24-bit row decodes now do the same: 3- to 8-byte native samples are read as their value (sign-extended when signed) and clamped to 0..=65535, or to the i16 range as two's complement. With the conversions equal, 16-bit rows stream for precisions up to 24 bits, as native component rows already did. The 8-bit component-plane writer (scaled decodes, row decodes and the streamed full-image route) quantized samples deeper than 16 bits over 65535 instead of over their component's range, saturating most of them, while the planes route quantizes over 2^bits - 1. It now uses the planes route's formula for those depths, so an over-budget 20-bit image's decode_into writes what a small one's does. Images of at most 16 bits keep every byte: the 1- and 2-byte paths are unchanged; the byte checks over 292 benchmark rows and 62 workload decodes, 8- and 16-bit, pass on this commit. Tests: unit (3-byte native samples against U16Conversion and the previous per-sample reference for 17..=24 bits, signed and unsigned; 4- to 8-byte samples), integration deep_samples_decode_to_their_values_clamped_to_ sixteen_bits (lossless 17/20/24-bit gray and RGB, signed and unsigned, on the planes and streamed routes and in regions, 8-bit included), 20- and 24-bit fixtures in the planes/streamed equality tests and the route test. The integration test fails on the previous head. --- crates/j2k/src/decode/output/u16.rs | 175 ++++++++++++++++-- crates/j2k/src/decode/output/u8.rs | 7 + crates/j2k/src/decode/route/tests.rs | 12 +- crates/j2k/src/view.rs | 5 +- .../tests/streamed_sixteen_bit_and_native.rs | 137 +++++++++++++- 5 files changed, 305 insertions(+), 31 deletions(-) diff --git a/crates/j2k/src/decode/output/u16.rs b/crates/j2k/src/decode/output/u16.rs index 69e6d396..b1640078 100644 --- a/crates/j2k/src/decode/output/u16.rs +++ b/crates/j2k/src/decode/output/u16.rs @@ -27,15 +27,13 @@ pub(in crate::decode) fn write_components_u16_output( /// Whether a region decode of `image` to `fmt` may stream its rows through /// [`U16RowSink`]: they carry the samples `decode_native_region` packs (see -/// `Image::streamed_rows_native_precision`), every component has one -/// precision of at most 16 bits, which `U16Conversion` converts to the +/// `Image::streamed_rows_native_precision`: every component has one +/// precision of at most 24 bits), which `U16Conversion` converts to the /// bytes [`write_u16_output`] writes from those packed samples, and /// `fmt` is a layout the image's color space maps to. Images that fail any /// of these keep the native-bit-depth planes, with their output or error. pub(in crate::decode) fn rows_match_native_u16(image: &Image<'_>, fmt: PixelFormat) -> bool { - image - .streamed_rows_native_precision() - .is_some_and(|precision| precision <= 16) + image.streamed_rows_native_precision().is_some() && matches!( (image.color_space(), image.has_alpha(), fmt), (ColorSpace::Gray, false, PixelFormat::Gray16) @@ -282,17 +280,18 @@ enum U16Kind { ByteTable, /// Signed, at most 8 bits: the two's-complement byte's widened value. SignedByteTable, - /// Unsigned, 9 to 16 bits: the sample itself. + /// Unsigned, 9 bits or more: the sample clamped to 16 bits. Unsigned, - /// Signed, 9 to 16 bits: the sample's two's complement. + /// Signed, 9 bits or more: the two's complement of the sample clamped + /// to 16 bits. Signed, } /// Converts the decoded samples of one component of `bit_depth` bits to /// 16-bit output: rounded, clamped to the component's signed or unsigned -/// range (at most 16 bits) and, for at most 8 bits, clamped to a byte and -/// widened across the 16-bit range as [`write_u16_output`] widens native -/// 8-bit samples. The choices are made once per plane. +/// range, and to 16 bits past 16, and, for at most 8 bits, clamped to a +/// byte and widened across the 16-bit range as [`write_u16_output`] widens +/// native 8-bit samples. The choices are made once per plane. #[derive(Debug, PartialEq)] struct U16Conversion { kind: U16Kind, @@ -458,13 +457,14 @@ pub(in crate::decode) fn write_u16_output( ) -> Result<(), J2kError> { let width = raw.width as usize; let height = raw.height as usize; + let signed = channel_signedness(raw); match (color_space, has_alpha, raw.num_components, fmt) { (ColorSpace::RGB, false, 3, PixelFormat::Rgb16) => { convert_or_copy_u16( &raw.data, raw.bytes_per_sample, raw.bit_depth, - 3, + &signed[..3], out, stride, (width, height), @@ -476,6 +476,7 @@ pub(in crate::decode) fn write_u16_output( src: &raw.data, bytes_per_sample: raw.bytes_per_sample, bit_depth: raw.bit_depth, + signed, source_channels: 4, layout: U16ChannelLayout::Drop, out, @@ -489,6 +490,7 @@ pub(in crate::decode) fn write_u16_output( src: &raw.data, bytes_per_sample: raw.bytes_per_sample, bit_depth: raw.bit_depth, + signed, source_channels: 3, layout: U16ChannelLayout::Synthesize, out, @@ -502,6 +504,7 @@ pub(in crate::decode) fn write_u16_output( src: &raw.data, bytes_per_sample: raw.bytes_per_sample, bit_depth: raw.bit_depth, + signed, source_channels: 4, layout: U16ChannelLayout::Preserve, out, @@ -515,7 +518,7 @@ pub(in crate::decode) fn write_u16_output( &raw.data, raw.bytes_per_sample, raw.bit_depth, - 1, + &signed[..1], out, stride, (width, height), @@ -540,6 +543,8 @@ struct U16ChannelRows<'src, 'out> { src: &'src [u8], bytes_per_sample: u8, bit_depth: u8, + /// Whether each source channel is signed. + signed: [bool; 4], source_channels: usize, layout: U16ChannelLayout, out: &'out mut [u8], @@ -552,6 +557,7 @@ fn write_u16_channel_rows(job: U16ChannelRows<'_, '_>) { src, bytes_per_sample, bit_depth, + signed, source_channels, layout, out, @@ -575,7 +581,13 @@ fn write_u16_channel_rows(job: U16ChannelRows<'_, '_>) { let src_pixel = &src_row[x * source_channels * bytes_per_sample..]; let dst_pixel = &mut dst_row[x * dst_channels * 2..(x + 1) * dst_channels * 2]; for channel in 0..3 { - let sample = output_u16_sample(src_pixel, channel, bytes_per_sample, &widened); + let sample = output_u16_sample( + src_pixel, + channel, + bytes_per_sample, + signed[channel], + &widened, + ); dst_pixel[channel * 2..channel * 2 + 2].copy_from_slice(&sample.to_le_bytes()); } match layout { @@ -584,7 +596,8 @@ fn write_u16_channel_rows(job: U16ChannelRows<'_, '_>) { dst_pixel[6..8].copy_from_slice(&alpha.to_le_bytes()); } U16ChannelLayout::Preserve => { - let sample = output_u16_sample(src_pixel, 3, bytes_per_sample, &widened); + let sample = + output_u16_sample(src_pixel, 3, bytes_per_sample, signed[3], &widened); dst_pixel[6..8].copy_from_slice(&sample.to_le_bytes()); } } @@ -600,18 +613,49 @@ fn opaque_alpha_u16(bytes_per_sample: usize, bit_depth: u8) -> u16 { } } +/// Whether each channel of `raw`'s pixels is signed (the first four). +fn channel_signedness(raw: &RawBitmap) -> [bool; 4] { + core::array::from_fn(|channel| { + raw.component_signed + .get(channel) + .copied() + .unwrap_or(raw.signed) + }) +} + /// `widened` is [`widened_u8_samples`] of the samples' bit depth. fn output_u16_sample( src_pixel: &[u8], channel: usize, bytes_per_sample: usize, + signed: bool, widened: &[u16; 256], ) -> u16 { let offset = channel * bytes_per_sample; - if bytes_per_sample == 2 { - return u16::from_le_bytes([src_pixel[offset], src_pixel[offset + 1]]); + match bytes_per_sample { + 1 => widened[usize::from(src_pixel[offset])], + 2 => u16::from_le_bytes([src_pixel[offset], src_pixel[offset + 1]]), + _ => clamped_wide_sample(&src_pixel[offset..offset + bytes_per_sample], signed), + } +} + +/// The 16-bit output of a native sample of more than two little-endian +/// bytes, from an image deeper than 16 bits: its value clamped to the +/// 16-bit range, as two's complement when `signed`. This is what +/// [`U16Conversion`] writes for the sample's decoded value, and what scaled +/// decodes, which convert component planes, always wrote. +fn clamped_wide_sample(bytes: &[u8], signed: bool) -> u16 { + let len = bytes.len().min(8); + let mut le = [0_u8; 8]; + le[..len].copy_from_slice(&bytes[..len]); + if signed { + // Sign-extend from the sample's top byte. + let unused = 64 - 8 * len; + let value = (i64::from_le_bytes(le) << unused) >> unused; + let clamped = value.clamp(i64::from(i16::MIN), i64::from(i16::MAX)); + return u16::from_le_bytes(i16::try_from(clamped).unwrap_or(i16::MAX).to_le_bytes()); } - widened[usize::from(src_pixel[offset])] + u16::try_from(u64::from_le_bytes(le)).unwrap_or(u16::MAX) } fn widen_u8_sample_to_u16(sample: u8, bit_depth: u8) -> u16 { @@ -628,16 +672,18 @@ fn widened_u8_samples(bit_depth: u8) -> [u16; 256] { }) } +/// `signed` holds whether each of the pixels' channels is signed. fn convert_or_copy_u16( src: &[u8], bytes_per_sample: u8, bit_depth: u8, - channels: usize, + signed: &[bool], out: &mut [u8], stride: usize, dims: (usize, usize), ) { let (width, height) = dims; + let channels = signed.len(); let dst_row_bytes = width * channels * 2; let src_row_bytes = width * channels * usize::from(bytes_per_sample); let widened = widened_u8_samples(bit_depth); @@ -647,6 +693,15 @@ fn convert_or_copy_u16( dst_row.copy_from_slice(src_row); continue; } + if bytes_per_sample > 2 { + let samples = src_row.chunks_exact(usize::from(bytes_per_sample)); + for (index, (sample, dst_sample)) in + samples.zip(dst_row.as_chunks_mut::<2>().0).enumerate() + { + *dst_sample = clamped_wide_sample(sample, signed[index % channels]).to_le_bytes(); + } + continue; + } for (sample, dst_sample) in src_row.iter().zip(dst_row.as_chunks_mut::<2>().0) { *dst_sample = widened[usize::from(*sample)].to_le_bytes(); } @@ -656,7 +711,8 @@ fn convert_or_copy_u16( #[cfg(test)] mod tests { use super::{ - opaque_alpha_u16, widen_u8_sample_to_u16, ChannelWriter, PixelWriter, U16Conversion, + clamped_wide_sample, convert_or_copy_u16, opaque_alpha_u16, widen_u8_sample_to_u16, + ChannelWriter, PixelWriter, U16Conversion, }; /// The per-sample conversion the component-plane writers used before @@ -741,6 +797,86 @@ mod tests { } } + #[test] + #[expect(clippy::cast_precision_loss, reason = "test values fit f32 exactly")] + fn deep_native_samples_convert_as_their_decoded_values_do() { + for bit_depth in 17..=24_u8 { + for signed in [false, true] { + let (min, max) = if signed { + (-(1_i64 << (bit_depth - 1)), (1_i64 << (bit_depth - 1)) - 1) + } else { + (0, (1_i64 << bit_depth) - 1) + }; + let edges = [-32_769, -32_768, -1, 0, 1, 32_767, 32_768, 65_535, 65_536]; + let values: Vec = (min..=max) + .step_by(997) + .chain([min, max]) + .chain( + edges + .into_iter() + .filter(|value| (min..=max).contains(value)), + ) + .collect(); + let samples: Vec = values.iter().map(|value| *value as f32).collect(); + let mut expected = vec![0; samples.len() * 2]; + U16Conversion::new(bit_depth, signed).visit(ChannelWriter { + src: &samples, + dst: &mut expected, + channels: 1, + channel: 0, + }); + // Native samples as `decode_native` packs them: three + // little-endian bytes, two's complement when signed. + let native: Vec = values + .iter() + .flat_map(|value| { + let bytes = value.to_le_bytes(); + [bytes[0], bytes[1], bytes[2]] + }) + .collect(); + let stride = samples.len() * 2; + let mut out = vec![0; stride]; + convert_or_copy_u16( + &native, + 3, + bit_depth, + &[signed], + &mut out, + stride, + (samples.len(), 1), + ); + assert_eq!(out, expected, "{bit_depth} bits, signed {signed}"); + for (sample, bytes) in samples.iter().zip(out.as_chunks::<2>().0) { + assert_eq!( + u16::from_le_bytes(*bytes), + component_sample_as_u16(*sample, bit_depth, signed), + "{sample} at {bit_depth} bits, signed {signed}" + ); + } + } + } + } + + #[test] + fn samples_wider_than_three_bytes_clamp_to_sixteen_bits() { + assert_eq!(clamped_wide_sample(&(-5_i32).to_le_bytes(), true), 0xfffb); + assert_eq!( + clamped_wide_sample(&(-70_000_i32).to_le_bytes(), true), + 0x8000 + ); + assert_eq!(clamped_wide_sample(&70_000_i32.to_le_bytes(), true), 0x7fff); + assert_eq!(clamped_wide_sample(&1234_u32.to_le_bytes(), false), 1234); + assert_eq!( + clamped_wide_sample(&(1_u64 << 36).to_le_bytes()[..5], false), + u16::MAX + ); + assert_eq!(clamped_wide_sample(&i64::MIN.to_le_bytes(), true), 0x8000); + assert_eq!( + clamped_wide_sample(&u64::MAX.to_le_bytes(), false), + u16::MAX + ); + } + #[test] fn eight_bit_samples_widen_across_the_complete_u16_domain() { assert_eq!(widen_u8_sample_to_u16(0, 8), 0); @@ -752,5 +888,6 @@ mod tests { assert_eq!(opaque_alpha_u16(1, 8), u16::MAX); assert_eq!(opaque_alpha_u16(2, 12), 0x0fff); assert_eq!(opaque_alpha_u16(2, 16), u16::MAX); + assert_eq!(opaque_alpha_u16(3, 20), u16::MAX); } } diff --git a/crates/j2k/src/decode/output/u8.rs b/crates/j2k/src/decode/output/u8.rs index 4cacb4d9..47bc0609 100644 --- a/crates/j2k/src/decode/output/u8.rs +++ b/crates/j2k/src/decode/output/u8.rs @@ -298,6 +298,7 @@ fn sample_as_u8_depth8(sample: f32) -> u8 { #[expect( clippy::cast_possible_truncation, clippy::cast_sign_loss, + clippy::cast_precision_loss, reason = "samples are rounded and clamped to the unsigned 8-bit output domain before conversion" )] fn sample_as_u8(sample: f32, bit_depth: u8) -> u8 { @@ -305,6 +306,12 @@ fn sample_as_u8(sample: f32, bit_depth: u8) -> u8 { if bit_depth == 8 { return sample_as_u8_depth8(sample); } + if bit_depth > 16 { + // The sample over the component's range, as the 8-bit planes of a + // full-image decode quantize it. + let source_max = ((1_u64 << bit_depth.min(63)) - 1) as f32; + return ((sample / source_max) * f32::from(u8::MAX)).round() as u8; + } let max_value = if bit_depth >= 16 { f32::from(u16::MAX) } else { diff --git a/crates/j2k/src/decode/route/tests.rs b/crates/j2k/src/decode/route/tests.rs index ac59d7c2..9a62b4f8 100644 --- a/crates/j2k/src/decode/route/tests.rs +++ b/crates/j2k/src/decode/route/tests.rs @@ -510,18 +510,20 @@ fn deep_and_signed_images_stream_past_the_budget() { (1, 12, false, true), (1, 16, true, false), (1, 4, false, true), + (1, 20, false, true), + (3, 24, true, false), ] { - let samples: Vec = (0..width * height * u32::from(components) * 2) + let bytes_per_sample = u32::from(bit_depth).div_ceil(8); + let samples: Vec = (0..width * height * u32::from(components) * bytes_per_sample) .map(|index| (index.wrapping_mul(2_654_435_761) >> 24) as u8) .collect(); - let samples = &samples[..samples.len() / if bit_depth > 8 { 1 } else { 2 }]; let options = EncodeOptions { num_decomposition_levels: 3, reversible, ..EncodeOptions::default() }; let bytes = encode( - samples, width, height, components, bit_depth, signed, &options, + &samples, width, height, components, bit_depth, signed, &options, ) .expect("encode fixture"); let image = @@ -534,7 +536,7 @@ fn deep_and_signed_images_stream_past_the_budget() { }; let label = format!("{components} x {bit_depth} bits, signed {signed}"); assert_eq!( - full_image_route(&image, fmt, 0), + full_image_route(&image, request(fmt, 0, CpuDecodeParallelism::Serial)), FullImageRoute::StreamedRegion, "{label}" ); @@ -545,7 +547,7 @@ fn deep_and_signed_images_stream_past_the_budget() { ); let budget = j2k_native::DEFAULT_MAX_DECODE_BYTES; assert_eq!( - full_image_route(&image, fmt, budget), + full_image_route(&image, request(fmt, budget, CpuDecodeParallelism::Serial)), FullImageRoute::Planes ); assert_eq!(native_components_route(&image, budget), NativeRoute::Planes); diff --git a/crates/j2k/src/view.rs b/crates/j2k/src/view.rs index 93c8fb18..458aeeb9 100644 --- a/crates/j2k/src/view.rs +++ b/crates/j2k/src/view.rs @@ -342,7 +342,10 @@ impl<'a> J2kDecoder<'a> { /// A decode whose component planes would pass the native decode budget /// runs as a region decode of the whole image, which writes rows as they /// are finished instead of holding the planes: at 8 bits, and at 16 bits - /// for images whose components share one precision of at most 16 bits. + /// for images whose components share one precision of at most 24 bits. + /// A 16-bit output holds each sample's value clamped to the 16-bit range + /// (two's complement when signed); samples of at most 8 bits are widened + /// across it. /// /// # Errors /// Returns [`J2kError`] when the format is unsupported, the output buffer diff --git a/crates/j2k/tests/streamed_sixteen_bit_and_native.rs b/crates/j2k/tests/streamed_sixteen_bit_and_native.rs index df9c7998..aeebf530 100644 --- a/crates/j2k/tests/streamed_sixteen_bit_and_native.rs +++ b/crates/j2k/tests/streamed_sixteen_bit_and_native.rs @@ -27,8 +27,9 @@ const RGB_FORMATS: &[PixelFormat] = &[ const GRAY_FORMATS: &[PixelFormat] = &[PixelFormat::Gray16, PixelFormat::Rgb16, PixelFormat::Gray8]; const RGBA_FORMATS: &[PixelFormat] = &[PixelFormat::Rgba16, PixelFormat::Rgb16, PixelFormat::Rgba8]; -/// Noise samples of `bit_depth` bits, one or two little-endian bytes each, -/// which the encoder masks to the precision (and sign-extends when signed). +/// Noise samples of `bit_depth` bits, one to three little-endian bytes +/// each, which the encoder masks to the precision (and sign-extends when +/// signed). fn noisy_samples(components: u16, bit_depth: u8) -> Vec { let bytes = usize::from(bit_depth).div_ceil(8); let mut state = 0x2545_f491_u32 ^ u32::from(bit_depth); @@ -50,10 +51,10 @@ fn codestream(components: u16, bit_depth: u8, signed: bool, options: &EncodeOpti .expect("encode fixture") } -/// Images of 4, 8, 12 and 16 bits, signed and unsigned, reversible and -/// irreversible, tiled and untiled, classic and HT, with the output formats -/// each is decoded to (8-bit ones included: they stream past the decode -/// budget too). +/// Images of 4, 8, 12, 16, 20 and 24 bits, signed and unsigned, reversible +/// and irreversible, tiled and untiled, classic and HT, with the output +/// formats each is decoded to (8-bit ones included: they stream past the +/// decode budget too). fn fixtures() -> Vec<(&'static str, Vec, &'static [PixelFormat])> { let levels = EncodeOptions { num_decomposition_levels: 4, @@ -120,6 +121,16 @@ fn fixtures() -> Vec<(&'static str, Vec, &'static [PixelFormat])> { codestream(1, 16, true, &irreversible), GRAY_FORMATS, ), + ( + "gray 20-bit 5/3", + codestream(1, 20, false, &reversible), + GRAY_FORMATS, + ), + ( + "rgb 24-bit signed 9/7", + codestream(3, 24, true, &irreversible), + RGB_FORMATS, + ), ( "gray 8-bit signed 5/3", codestream(1, 8, true, &reversible), @@ -325,6 +336,120 @@ fn sixteen_bit_regions_match_the_full_image() { } } +/// Samples of `bit_depth` bits, at least a quarter of them within 16 bits +/// (signed: within `i16`). +fn deep_samples(components: u16, bit_depth: u8, signed: bool) -> Vec { + let span = 1_i64 << 18; + let offset = if signed { span / 2 } else { 0 }; + let (min, max) = if signed { + (-(1 << (bit_depth - 1)), (1 << (bit_depth - 1)) - 1) + } else { + (0, (1 << bit_depth) - 1) + }; + let count = WIDTH as usize * HEIGHT as usize * usize::from(components); + (0_i64..) + .take(count) + .map(|index| ((index * 7919 + index / 3) % span - offset).clamp(min, max)) + .collect() +} + +/// A 16-bit output of an image deeper than 16 bits holds each sample's +/// value clamped to the 16-bit range (two's complement when signed), as a +/// scaled decode always wrote it, on the planes route, on the streamed +/// route and in region decodes; an 8-bit output of an unsigned one holds +/// the sample over the component's range, on either route. +#[test] +fn deep_samples_decode_to_their_values_clamped_to_sixteen_bits() { + let reversible = EncodeOptions { + num_decomposition_levels: 3, + reversible: true, + ..EncodeOptions::default() + }; + let pool = &mut J2kScratchPool::new(); + for (components, bit_depth, signed, fmt) in [ + (1_u16, 20_u8, false, PixelFormat::Gray16), + (1, 24, true, PixelFormat::Gray16), + (3, 20, true, PixelFormat::Rgb16), + (3, 17, false, PixelFormat::Rgb16), + ] { + let label = format!("{components} x {bit_depth} bits, signed {signed}"); + let values = deep_samples(components, bit_depth, signed); + let samples: Vec = values + .iter() + .flat_map(|value| { + let bytes = value.to_le_bytes(); + [bytes[0], bytes[1], bytes[2]] + }) + .collect(); + let bytes = encode( + &samples, + WIDTH, + HEIGHT, + components, + bit_depth, + signed, + &reversible, + ) + .expect("encode deep fixture"); + let expected: Vec = values + .iter() + .flat_map(|&value| { + let clamped = if signed { + let value = i16::try_from(value.clamp(-32_768, 32_767)).expect("i16"); + u16::from_le_bytes(value.to_le_bytes()) + } else { + u16::try_from(value.clamp(0, 65_535)).expect("u16") + }; + clamped.to_le_bytes() + }) + .collect(); + for parallelism in [CpuDecodeParallelism::Serial, CpuDecodeParallelism::Auto] { + for budget in [None, Some(0)] { + let decoded = decode(&bytes, budget, parallelism, fmt, Downscale::None, 0) + .unwrap_or_else(|| panic!("{label} {budget:?}: decode failed")); + assert!( + decoded == expected, + "{label} {parallelism:?} planes budget {budget:?}" + ); + } + if !signed { + let fmt = if components == 1 { + PixelFormat::Gray8 + } else { + PixelFormat::Rgb8 + }; + let max = ((1_u64 << bit_depth) - 1) as f32; + let expected: Vec = values + .iter() + .map(|&value| ((value as f32 / max) * 255.0).round() as u8) + .collect(); + for budget in [None, Some(0)] { + let decoded = decode(&bytes, budget, parallelism, fmt, Downscale::None, 0); + assert!( + decoded.as_ref() == Some(&expected), + "{label} {fmt:?} {parallelism:?} planes budget {budget:?}" + ); + } + } + let pixel = fmt.bytes_per_pixel(); + for roi in regions() { + let stride = roi.w as usize * pixel; + let mut out = vec![PADDING; stride * roi.h as usize]; + decoder(&bytes, None, parallelism) + .decode_region_into(pool, &mut out, stride, fmt, roi) + .unwrap_or_else(|error| panic!("{label} {roi:?}: {error:?}")); + for (row, line) in out.chunks(stride).enumerate() { + let start = ((roi.y as usize + row) * WIDTH as usize + roi.x as usize) * pixel; + assert!( + line == &expected[start..start + stride], + "{label} {parallelism:?} {roi:?} row {row}" + ); + } + } + } + } +} + /// Tiles whose coarse resolutions are empty while finer ones are not keep /// the planes for 16-bit and native component output too. #[test] From edbf6e05be6f8c99bcbd309d91965c4e468e93b7 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 04:36:45 -0400 Subject: [PATCH 36/94] test(j2k): check the last row of tight buffers on the streamed routes The last-row test now decodes full images and scaled decodes on the planes (a planes budget of `usize::MAX`), on the streamed rows (a budget of 0) and on the default route, and a 768 x 512 RGB image of six 256 x 256 tiles on a two-thread pool, which streams three tiles at a time, to every color format. Each case compares a decode into a buffer of exactly `strided_output_len` bytes with one into a padded buffer. --- crates/j2k/src/decode/output/u16.rs | 19 +++++- crates/j2k/src/view.rs | 2 +- crates/j2k/tests/decode.rs | 64 +++++++++++++------ .../tests/streamed_sixteen_bit_and_native.rs | 10 +++ 4 files changed, 72 insertions(+), 23 deletions(-) diff --git a/crates/j2k/src/decode/output/u16.rs b/crates/j2k/src/decode/output/u16.rs index b1640078..519e7d53 100644 --- a/crates/j2k/src/decode/output/u16.rs +++ b/crates/j2k/src/decode/output/u16.rs @@ -348,7 +348,8 @@ impl U16Conversion { #[expect( clippy::cast_possible_truncation, clippy::cast_sign_loss, - reason = "decoded samples are rounded and clamped to the component's declared integer representation" + clippy::inline_always, + reason = "decoded samples are rounded and clamped to the component's declared integer representation; the chosen conversion is inlined into the writer so each kind compiles to its own fixed-size pixel loop" )] #[inline(always)] fn visit(&self, writer: impl U16Writer) { @@ -385,6 +386,10 @@ struct ChannelWriter<'s, 'd> { } impl U16Writer for ChannelWriter<'_, '_> { + #[expect( + clippy::inline_always, + reason = "the conversion closure is inlined into the fixed-size pixel loop it runs in" + )] #[inline(always)] fn write(self, convert: impl Fn(f32) -> u16) { let Self { @@ -402,6 +407,10 @@ impl U16Writer for ChannelWriter<'_, '_> { } } +#[expect( + clippy::inline_always, + reason = "the conversion closure is inlined into the fixed-size pixel loop it runs in" +)] #[inline(always)] fn write_channel( src: &[f32], @@ -423,6 +432,10 @@ struct PixelWriter<'s, 'd> { } impl U16Writer for PixelWriter<'_, '_> { + #[expect( + clippy::inline_always, + reason = "the conversion closure is inlined into the fixed-size pixel loop it runs in" + )] #[inline(always)] fn write(self, convert: impl Fn(f32) -> u16) { let Self { src, dst, channels } = self; @@ -433,6 +446,10 @@ impl U16Writer for PixelWriter<'_, '_> { } } +#[expect( + clippy::inline_always, + reason = "the conversion closure is inlined into the fixed-size pixel loop it runs in" +)] #[inline(always)] fn write_pixels( [red, green, blue]: [&[f32]; 3], diff --git a/crates/j2k/src/view.rs b/crates/j2k/src/view.rs index 458aeeb9..1ec0ef36 100644 --- a/crates/j2k/src/view.rs +++ b/crates/j2k/src/view.rs @@ -300,7 +300,7 @@ impl<'a> J2kDecoder<'a> { /// /// Like 16-bit region decodes, it packs rows into the returned planes as /// they are finished, without region-sized `f32` planes, for RGB images - /// without palette, channel definition, sYCC or CIELab whose components + /// without palette, channel definition, sYCC or `CIELab` whose components /// share one precision, and for other such images when the planes would /// pass the native decode budget. /// diff --git a/crates/j2k/tests/decode.rs b/crates/j2k/tests/decode.rs index eeaac395..2cf21d67 100644 --- a/crates/j2k/tests/decode.rs +++ b/crates/j2k/tests/decode.rs @@ -1015,16 +1015,21 @@ enum StridedRoute { /// Decode `bytes` on `route` to `fmt` at a stride three bytes past the /// output rows, into a buffer of exactly [`j2k_core::strided_output_len`] /// bytes when `tight`, which has no padding after its last row, and of one -/// padded row per output row otherwise. Every byte starts as `0xa5`. +/// padded row per output row otherwise. Every byte starts as `0xa5`. A +/// `planes_budget` holds full-image decodes' planes to it: 0 streams every +/// decode that can stream, `usize::MAX` keeps them on the planes. fn decode_to_strided_buffer( bytes: &[u8], - parallelism: CpuDecodeParallelism, + (parallelism, planes_budget): (CpuDecodeParallelism, Option), route: StridedRoute, fmt: PixelFormat, tight: bool, ) -> Vec { let mut decoder = J2kDecoder::new(bytes).expect("decoder"); decoder.set_cpu_decode_parallelism(parallelism); + if let Some(bytes) = planes_budget { + decoder.set_full_decode_planes_budget(bytes); + } let dims = decoder.info().dimensions; let roi = Rect { x: 3, @@ -1062,8 +1067,7 @@ fn decode_to_strided_buffer( #[test] fn decode_writes_the_last_row_of_a_buffer_without_trailing_padding() { - let (width, height) = (45_u32, 31_u32); - let encode_image = |components: u16, bit_depth: u8, reversible: bool| { + let encode_image = |(width, height): (u32, u32), components: u16, bit_depth: u8, tile| { let bytes_per_sample = u32::from(bit_depth).div_ceil(8); let mut pixels: Vec = (0..width * height * u32::from(components) * bytes_per_sample) .map(|i| (i * 37 + i / 11).to_le_bytes()[0]) @@ -1075,8 +1079,9 @@ fn decode_writes_the_last_row_of_a_buffer_without_trailing_padding() { } } let options = EncodeOptions { - reversible, + reversible: components != 3, num_decomposition_levels: 3, + tile_size: tile, ..EncodeOptions::default() }; encode( @@ -1084,11 +1089,14 @@ fn decode_writes_the_last_row_of_a_buffer_without_trailing_padding() { ) .expect("encode") }; - let gray8 = encode_image(1, 8, true); - let gray12 = encode_image(1, 12, false); - let rgb8 = encode_image(3, 8, false); - let rgb12 = encode_image(3, 12, true); - let rgba8_jp2 = wrap_jp2_rgba_codestream(&encode_image(4, 8, true), width, height, 8); + let small = (45, 31); + let gray8 = encode_image(small, 1, 8, None); + let gray12 = encode_image(small, 1, 12, None); + let rgb8 = encode_image(small, 3, 8, None); + let rgb12 = encode_image(small, 3, 12, None); + let rgba8_jp2 = wrap_jp2_rgba_codestream(&encode_image(small, 4, 8, None), 45, 31, 8); + // Six tiles, which a two-thread pool streams three at a time. + let rgb8_tiled = encode_image((768, 512), 3, 8, Some((256, 256))); let gray_formats = [PixelFormat::Gray8, PixelFormat::Gray16]; let color_formats = [ PixelFormat::Rgb8, @@ -1104,24 +1112,38 @@ fn decode_writes_the_last_row_of_a_buffer_without_trailing_padding() { ("rgba8", &rgba8_jp2, &color_formats), ]; let mut failures = Vec::new(); + let mut compare = |name: &str, bytes: &[u8], settings, route, fmt| { + let padded = decode_to_strided_buffer(bytes, settings, route, fmt, false); + let tight = decode_to_strided_buffer(bytes, settings, route, fmt, true); + if tight != padded[..tight.len()] { + failures.push(format!("{name} {fmt:?} {route:?} {settings:?}")); + } + }; for (name, bytes, formats) in cases { for &fmt in formats { - for route in [ - StridedRoute::Full, - StridedRoute::Region, - StridedRoute::Scaled, - StridedRoute::RegionScaled, - ] { - for parallelism in [CpuDecodeParallelism::Auto, CpuDecodeParallelism::Serial] { - let padded = decode_to_strided_buffer(bytes, parallelism, route, fmt, false); - let tight = decode_to_strided_buffer(bytes, parallelism, route, fmt, true); - if tight != padded[..tight.len()] { - failures.push(format!("{name} {fmt:?} {route:?} {parallelism:?}")); + for parallelism in [CpuDecodeParallelism::Auto, CpuDecodeParallelism::Serial] { + // The planes, the streamed rows, and the default route. + for budget in [Some(usize::MAX), Some(0), None] { + for route in [StridedRoute::Full, StridedRoute::Scaled] { + compare(name, bytes, (parallelism, budget), route, fmt); } } + for route in [StridedRoute::Region, StridedRoute::RegionScaled] { + compare(name, bytes, (parallelism, None), route, fmt); + } } } } + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(2) + .build() + .expect("pool"); + pool.install(|| { + for fmt in color_formats { + let settings = (CpuDecodeParallelism::Auto, Some(0)); + compare("rgb8 tiled", &rgb8_tiled, settings, StridedRoute::Full, fmt); + } + }); assert!( failures.is_empty(), "a buffer that ends after the last row's pixels must hold the padded decode's \ diff --git a/crates/j2k/tests/streamed_sixteen_bit_and_native.rs b/crates/j2k/tests/streamed_sixteen_bit_and_native.rs index aeebf530..36ecba9b 100644 --- a/crates/j2k/tests/streamed_sixteen_bit_and_native.rs +++ b/crates/j2k/tests/streamed_sixteen_bit_and_native.rs @@ -418,7 +418,17 @@ fn deep_samples_decode_to_their_values_clamped_to_sixteen_bits() { } else { PixelFormat::Rgb8 }; + #[expect( + clippy::cast_precision_loss, + reason = "the unsigned fixtures are at most 20 bits deep, exact in f32" + )] let max = ((1_u64 << bit_depth) - 1) as f32; + #[expect( + clippy::cast_possible_truncation, + clippy::cast_precision_loss, + clippy::cast_sign_loss, + reason = "unsigned samples below 2^20 are exact in f32 and scale into 0..=255" + )] let expected: Vec = values .iter() .map(|&value| ((value as f32 / max) * 255.0).round() as u8) From bdb3e8374e5c38fe13176b4d3e6f4d726d4e955a Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 04:36:45 -0400 Subject: [PATCH 37/94] docs(changelog): note streamed 16-bit and native component outputs --- CHANGELOG.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ff862de..b0b02ae9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,43 @@ Staged workspace version: `0.13.2`. Staged windows are admitted by planned execution memory, and an admission claim that cannot fit beside the shared plan fails instead of waiting indefinitely. Successful output is unchanged. +- 16-bit full-image decodes (`J2kDecoder::decode_into`, `decode_scaled_into` + and the decoders behind `decode_tiles_into` at `Rgb16`, `Rgba16` and + `Gray16`) and `J2kDecoder::decode_native_components` whose component + planes, or the planes together with their packed native samples, would + pass the native decode budget now run as a region decode of the whole + image that streams its rows, as 8-bit decodes do, instead of failing with + a resource-limit error. `decode_native_components` packs the rows into its + native planes as they are finished. This covers images without palette, + channel definitions, sYCC or CIELab whose unsubsampled components share one + precision of up to 24 bits, signed or unsigned; other images keep the + planes route and its error. Output is unchanged apart from the fix to + images deeper than 16 bits below. A 5661 x 8287 RGB image decodes to `Rgb16` in about 130 ms with + 103 MiB peak heap on an 18-thread pool (1.39 s and 26 MiB on one thread), + where the planes route with its budget raised to fit takes about 196 ms + and 984 MiB (1.49 s and 952 MiB); its native components take about 122 ms + and 96 MiB besides the 141 MB of planes returned. +- 16-bit region decodes (`decode_region_into`, `decode_region_scaled_into` + and their power-of-two forms) and `decode_native_region_components` of RGB + images without palette, channel definitions, sYCC or CIELab stream their + rows instead of holding the region's `f32` planes and native samples, and + so do those of other such images whose planes would pass the native decode + budget. Output is unchanged. +- 16-bit outputs of images deeper than 16 bits hold each sample's value + clamped to the 16-bit range (two's complement when signed), as scaled + decodes already wrote them. Full-image and region decodes (and row decodes + of images deeper than 24 bits) wrote each byte of the wider native samples + as a sample of its own. 8-bit outputs of such images on the streamed + route, in scaled decodes and in `decode_rows_u8_bounded` quantize each + sample over its component's range, as full-image decodes on the planes + route do, instead of over 16 bits, which saturated most samples. Images + of at most 16 bits are unaffected. +- 16-bit output and `decode_native` on the planes route are faster: two to + four components pack into native samples a block of pixels at a time on + the thread pool instead of one sample at a time, and 8-bit native samples + widen to 16 bits from a table instead of a division per sample. `Rgb16` + `decode_into` of a 4096 x 4096 RGB image takes about 40% of the time it + did on an 18-thread pool. Output is unchanged. - Region decodes of a tile whose coarse resolutions are empty while finer ones are not (a one-sample tile row or column at an odd position with two or more levels) reconstruct it from the first level that has samples, as From 23e41003e77ba623c05feb9a5f12f93bed282f89 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 14:56:25 -0400 Subject: [PATCH 38/94] fix(j2k): keep native region decodes that cannot stream on the planes route decode_native_region_components took the rows route for every image whose rows carry its native samples when region_rows_preferred held (always for RGB, otherwise when the region's planes and packed samples would pass the budget). That route allocates every output plane before it decodes, and a region decode that cannot stream (image or tile offsets, or more than four components) falls back to component planes and meets the decode budget only after those allocations. A codestream of 1024 components claiming 1024 x 1024 samples peaked at 1.51 GiB RSS before its resource-limit error (x86-64, default pool), where the planes route fails at 0.50 GiB. The image-level conditions of the streamed decode (no offsets, one to four components) now live in j2c::image_streams_rows, which the decode-time streams_rows calls instead of its own copy. Image::region_decode_streams_rows (doc-hidden, additive; implementation API snapshot updated) adds the conditions the boxes and component sizes settle and is the one check of the full-image planes floor and of the native rows route, whose decode now refuses an image that fails it before allocating. Such images keep the planes route: the 1024-component region decode fails at 0.50 GiB, an RGB image with offsets claiming 12000 x 12000 at 7 MiB instead of 416 MiB, five components at 8192 x 8192 at 260 MiB instead of 582 MiB, with the same error. Images that stream keep the rows route and their bytes. 16-bit region decodes are unchanged: their rows go to the caller's buffer, and a decode that falls back meets the budget before it allocates anything image-sized. --- crates/j2k-native/src/image/native/rows.rs | 15 ++-- crates/j2k-native/src/image/output_api.rs | 41 ++++++--- crates/j2k-native/src/j2c/decode.rs | 29 ++++--- .../j2k-native/src/j2c/decode/planes_floor.rs | 30 +++---- crates/j2k-native/src/j2c/mod.rs | 3 +- crates/j2k/src/decode/output/native.rs | 10 ++- crates/j2k/src/decode/route/tests.rs | 86 ++++++++++++++++++- crates/j2k/src/view.rs | 7 +- ...able-api-1.0.implementation-public-api.txt | 1 + 9 files changed, 167 insertions(+), 55 deletions(-) diff --git a/crates/j2k-native/src/image/native/rows.rs b/crates/j2k-native/src/image/native/rows.rs index bbab1d5f..573eab29 100644 --- a/crates/j2k-native/src/image/native/rows.rs +++ b/crates/j2k-native/src/image/native/rows.rs @@ -48,7 +48,12 @@ impl<'a> Image<'a> { /// [`Self::decode_region_rows_for_integer_output_with_context`]), which /// packs them into the returned planes as they are finished: the decode /// holds no region-sized `f32` planes. The planes hold the same bytes. - /// Only for an image with a [`Self::streamed_rows_native_precision`]. + /// Only for an image with a [`Self::streamed_rows_native_precision`] + /// whose region decodes stream their rows (see + /// [`Self::region_decode_streams_rows`]): the planes are allocated + /// before the decode, so a region decode that fell back to component + /// planes would hold both, and meet the decode budget only after + /// allocating them. /// The region of the whole image gives /// [`Self::decode_native_components_with_context`]'s planes. /// @@ -59,8 +64,8 @@ impl<'a> Image<'a> { /// # Errors /// /// Returns an error when the region is invalid, the image has no - /// streamed rows precision, a plane cannot be allocated, or decoding - /// fails. + /// streamed rows precision or does not stream its rows, a plane cannot + /// be allocated, or decoding fails. #[doc(hidden)] pub fn decode_native_region_components_rows_with_context( &self, @@ -68,9 +73,9 @@ impl<'a> Image<'a> { decoder_context: &mut DecoderContext<'a>, ) -> Result { validate_roi((self.width(), self.height()), roi)?; - if self.streamed_rows_native_precision().is_none() { + if self.streamed_rows_native_precision().is_none() || !self.region_decode_streams_rows() { bail!(DecodingError::UnsupportedFeature( - "streamed native component planes need unsubsampled components of one precision up to 24 bits without palette, channel definition, sYCC or CIELab" + "streamed native component planes need one to four unsubsampled components of one precision up to 24 bits, no image or tile offsets, and no palette, channel definition, sYCC or CIELab" )); } let dimensions = (roi.2, roi.3); diff --git a/crates/j2k-native/src/image/output_api.rs b/crates/j2k-native/src/image/output_api.rs index a1a8930b..a616768e 100644 --- a/crates/j2k-native/src/image/output_api.rs +++ b/crates/j2k-native/src/image/output_api.rs @@ -523,6 +523,29 @@ impl<'a> Image<'a> { Ok(()) } + /// Whether [`Self::decode_region_rows_for_integer_output_with_context`] + /// can stream this image's rows, as far as its main header and boxes + /// say: the image has no palette, channel definition, sYCC or `CIELab` + /// (see `permits_line_region_idwt`), its components are at full sampling + /// with at most 24 bits, and the image and its tiles have no offsets and + /// one to four components (see `j2c::image_streams_rows`, which the + /// decode checks before it streams). Any other region decode falls back + /// to component planes of the region and hands them over at the end. + /// A tile's own coding can still keep it there (see + /// [`Self::full_decode_planes_floor`], which plans the full image only + /// when this holds). + #[doc(hidden)] + #[must_use] + pub fn region_decode_streams_rows(&self) -> bool { + self.permits_line_region_idwt() + && self.header.component_infos.iter().all(|component| { + component.size_info.precision <= 24 + && component.size_info.horizontal_resolution == 1 + && component.size_info.vertical_resolution == 1 + }) + && j2c::image_streams_rows(&self.header) + } + /// The bytes a full-image decode into component planes charges the /// decode budget at least, planned from the codestream's tiles without /// decoding, when [`Self::decode_region_rows_for_integer_output_with_context`] @@ -532,11 +555,11 @@ impl<'a> Image<'a> { /// refused, and an image it can finish never has a floor past the /// budget. /// - /// `None` when the region decode could not stream (a palette, channel - /// definition, sYCC or CIELab image, components of more than 24 bits, - /// subsampling, offsets, or a tile whose coarse resolutions are empty - /// while finer ones are not), or when the header alone puts the planes - /// and one tile's coefficients and IDWT buffers within half of `budget`. + /// `None` when the region decode could not stream (see + /// [`Self::region_decode_streams_rows`], or a tile whose coarse + /// resolutions are empty while finer ones are not), or when the header + /// alone puts the planes and one tile's coefficients and IDWT buffers + /// within half of `budget`. /// /// # Errors /// @@ -544,13 +567,7 @@ impl<'a> Image<'a> { /// meets as well. #[doc(hidden)] pub fn full_decode_planes_floor(&self, budget: usize) -> Result> { - if !self.permits_line_region_idwt() - || self - .header - .component_infos - .iter() - .any(|component| component.size_info.precision > 24) - { + if !self.region_decode_streams_rows() { return Ok(None); } j2c::full_decode_planes_floor( diff --git a/crates/j2k-native/src/j2c/decode.rs b/crates/j2k-native/src/j2c/decode.rs index ab9ba605..13e8d4b5 100644 --- a/crates/j2k-native/src/j2c/decode.rs +++ b/crates/j2k-native/src/j2c/decode.rs @@ -608,10 +608,11 @@ fn choose_region_idwt( /// Whether a region decode on the row-streamed IDWT can hand its samples to /// a row sink instead of component planes: every tile decodes its Tier-1 in /// bands, serially (see [`tile_bands`]) or on the pool (see [`pool_bands`]), -/// so all components can advance together; the -/// image and tiles have no offsets; there are one to four components; and -/// each tile with an inverse MCT has three components or more, the first -/// three with one wavelet, as `apply_inverse_mct_per_tile` requires. +/// so all components can advance together; the image and tiles have no +/// offsets and there are one to four components (see +/// [`image_streams_rows`]); and each tile with an inverse MCT has three +/// components or more, the first three with one wavelet, as +/// `apply_inverse_mct_per_tile` requires. /// `ensure_every_sample_streamed` checks afterwards that the rows covered /// the region. fn streams_rows( @@ -620,13 +621,8 @@ fn streams_rows( tile_ctx: &TileDecodeContext, parallelism: CpuDecodeParallelism, ) -> bool { - let size = &header.size_data; let components = header.component_infos.len(); - size.image_area_x_offset == 0 - && size.image_area_y_offset == 0 - && size.tile_x_offset == 0 - && size.tile_y_offset == 0 - && (1..=4).contains(&components) + image_streams_rows(header) && tiles.iter().all(|tile| { let infos = &tile.component_infos; infos.len() == components @@ -647,6 +643,19 @@ fn streams_rows( }) } +/// The conditions of [`streams_rows`] that the main header settles: the +/// image and its tiles have no offsets, and there are one to four +/// components. The plans that choose a streamed route before decoding +/// check them here too (see `Image::region_decode_streams_rows`). +pub(crate) fn image_streams_rows(header: &Header<'_>) -> bool { + let size = &header.size_data; + size.image_area_x_offset == 0 + && size.image_area_y_offset == 0 + && size.tile_x_offset == 0 + && size.tile_y_offset == 0 + && (1..=4).contains(&header.component_infos.len()) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct OutputRegion { pub(crate) x: u32, diff --git a/crates/j2k-native/src/j2c/decode/planes_floor.rs b/crates/j2k-native/src/j2c/decode/planes_floor.rs index e05bfeb4..a99a03b0 100644 --- a/crates/j2k-native/src/j2c/decode/planes_floor.rs +++ b/crates/j2k-native/src/j2c/decode/planes_floor.rs @@ -31,14 +31,15 @@ use crate::reader::BitReader; /// The bytes a full-image decode of `data` into component planes charges at /// least, when its full-image region decode could stream its rows instead: -/// the image and its tiles have no offsets, one to four components at full -/// sampling, an inverse MCT only over three or more components whose first -/// three share a wavelet (as `streams_rows` requires), and no tile-component -/// whose coarse resolutions are empty while finer ones are not, which the -/// region IDWTs do not reconstruct. `None` when it could not, or when the -/// header alone puts the planes, one tile's coefficients and its IDWT -/// buffers within half of `budget`, which leaves the planes decode far from -/// it unless its metadata is unusually large. +/// the image streams as far as its main header says (the caller checks +/// `Image::region_decode_streams_rows`), and every tile has its components +/// at full sampling, an inverse MCT only over three or more components whose +/// first three share a wavelet (as `streams_rows` requires), and no +/// tile-component whose coarse resolutions are empty while finer ones are +/// not, which the region IDWTs do not reconstruct. `None` when it could not, +/// or when the header alone puts the planes, one tile's coefficients and its +/// IDWT buffers within half of `budget`, which leaves the planes decode far +/// from it unless its metadata is unusually large. /// /// `retained_image_bytes` is what the image holds while it decodes, the /// tile parse's baseline. Errors are the tile parse's, which the planes @@ -50,7 +51,7 @@ pub(crate) fn full_decode_planes_floor( budget: usize, ) -> Result> { let planes = planes_bytes(header)?; - if header_estimate(header, planes).saturating_mul(2) <= budget || !image_streams_rows(header) { + if header_estimate(header, planes).saturating_mul(2) <= budget { return Ok(None); } let mut reader = BitReader::new(data); @@ -120,17 +121,6 @@ fn header_estimate(header: &Header<'_>, planes: usize) -> usize { ) } -/// The image-level conditions of `streams_rows`: no offsets and one to four -/// components. -fn image_streams_rows(header: &Header<'_>) -> bool { - let size = &header.size_data; - size.image_area_x_offset == 0 - && size.image_area_y_offset == 0 - && size.tile_x_offset == 0 - && size.tile_y_offset == 0 - && (1..=4).contains(&header.component_infos.len()) -} - /// The tile-level conditions of `streams_rows` and the region IDWTs that /// `tile`'s header settles. fn tile_streams_rows(tile: &Tile<'_>, components: usize, skipped: u8) -> bool { diff --git a/crates/j2k-native/src/j2c/mod.rs b/crates/j2k-native/src/j2c/mod.rs index 555d684d..9501b6dd 100644 --- a/crates/j2k-native/src/j2c/mod.rs +++ b/crates/j2k-native/src/j2c/mod.rs @@ -59,7 +59,8 @@ pub(crate) use decode::{ build_referenced_classic_rgba_plan, build_referenced_htj2k_color_plan, build_referenced_htj2k_grayscale_plan, build_referenced_htj2k_rgba_plan, decode_preparsed_with_capacity_retry as decode_preparsed, - decode_rows_with_capacity_retry as decode_rows, full_decode_planes_floor, prepare_region_tiles, + decode_rows_with_capacity_retry as decode_rows, full_decode_planes_floor, image_streams_rows, + prepare_region_tiles, }; pub use decode::{CpuDecodeParallelism, DecoderContext, DecoderWorkspace, DecoderWorkspaceStats}; pub use recode::Reversible53CoefficientImage; diff --git a/crates/j2k/src/decode/output/native.rs b/crates/j2k/src/decode/output/native.rs index 9b135ab9..d7500ad7 100644 --- a/crates/j2k/src/decode/output/native.rs +++ b/crates/j2k/src/decode/output/native.rs @@ -81,11 +81,17 @@ pub(in crate::decode) fn region_rows_preferred( /// How a region decode of `roi` of `image` into owned native component /// planes runs: as one that packs its rows into the planes as they are /// finished, so that it holds no region-sized `f32` planes, when its rows -/// carry the native samples and [`region_rows_preferred`]. +/// carry the native samples, the region decode streams them (see +/// `Image::region_decode_streams_rows`, which the full-image floor checks +/// too) and [`region_rows_preferred`]. The rows route allocates the planes +/// before it decodes, so an image whose region decode would fall back to +/// component planes keeps the planes route, which meets the decode budget +/// before allocating them. pub(crate) fn native_region_components_route(image: &Image<'_>, roi: Rect) -> NativeRoute { match image.streamed_rows_native_precision() { Some(precision) - if region_rows_preferred(image, roi, usize::from(precision).div_ceil(8)) => + if image.region_decode_streams_rows() + && region_rows_preferred(image, roi, usize::from(precision).div_ceil(8)) => { NativeRoute::Rows } diff --git a/crates/j2k/src/decode/route/tests.rs b/crates/j2k/src/decode/route/tests.rs index 9a62b4f8..72e791f2 100644 --- a/crates/j2k/src/decode/route/tests.rs +++ b/crates/j2k/src/decode/route/tests.rs @@ -2,11 +2,11 @@ use super::{full_image_route, FullImageRoute, PlanesBudget, RouteRequest}; use crate::backend::image_at_scale; -use crate::decode::{native_components_route, NativeRoute}; +use crate::decode::{native_components_route, native_region_components_route, NativeRoute}; use crate::DecodeSettings; use j2k_core::{Downscale, PixelFormat, Rect}; -use j2k_native::DecoderContext; use j2k_native::{encode, CpuDecodeParallelism, EncodeOptions}; +use j2k_native::{DecodeError, DecoderContext, DecodingError}; fn codestream(components: u16, options: &EncodeOptions) -> Vec { let (width, height) = (96_u32, 80_u32); @@ -553,3 +553,85 @@ fn deep_and_signed_images_stream_past_the_budget() { assert_eq!(native_components_route(&image, budget), NativeRoute::Planes); } } + +/// `bytes` with the SIZ fields at `fields` (byte offsets from the marker) +/// rewritten by `rewrite`. +fn rewrite_siz(bytes: &[u8], fields: &[usize], rewrite: impl Fn(u32) -> u32) -> Vec { + let mut bytes = bytes.to_vec(); + let siz = bytes + .windows(2) + .position(|marker| marker == [0xff, 0x51]) + .expect("SIZ marker"); + for &field in fields { + let field = &mut bytes[siz + field..siz + field + 4]; + let value = u32::from_be_bytes(field.try_into().expect("four-byte SIZ field")); + field.copy_from_slice(&rewrite(value).to_be_bytes()); + } + bytes +} + +/// A region decode into native component planes packs its rows into planes +/// it allocates before decoding only when the region decode streams them, +/// by the check the full-image floor makes: an image of more than four +/// components, or with image and tile offsets, keeps the planes route +/// however large it is, whose decode meets the budget before it allocates +/// image-sized buffers, and the rows decode refuses it. +#[test] +fn native_region_rows_only_for_images_whose_region_decodes_stream() { + let options = EncodeOptions { + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + let rgb = codestream(3, &options); + // The whole image and its tile one tile further on. + let translated = rewrite_siz(&rgb, &[6, 10, 14, 18, 30, 34], |value| value + 128); + // One 8192 x 8192 tile, whose native planes alone pass the budget: an + // image of these components prefers its streamed rows. Headers only; + // nothing here decodes its packets. + let claimed = |components| { + let bytes = codestream(components, &options); + rewrite_siz(&bytes, &[6, 10, 22, 26], |_| 8192) + }; + for (label, bytes, streams) in [ + ("rgb", rgb.clone(), true), + ("rgb with offsets", translated, false), + ("four components", claimed(4), true), + ("five components", claimed(5), false), + ] { + let image = + image_at_scale(&bytes, DecodeSettings::default(), Downscale::None).expect("image"); + let (width, height) = (image.width(), image.height()); + assert_eq!(image.streamed_rows_native_precision(), Some(8), "{label}"); + assert_eq!(image.region_decode_streams_rows(), streams, "{label}"); + assert_eq!( + image.full_decode_planes_floor(0).expect("plan").is_some(), + streams, + "{label}" + ); + let expected = if streams { + NativeRoute::Rows + } else { + NativeRoute::Planes + }; + assert_eq!( + native_region_components_route(&image, Rect::full((width, height))), + expected, + "{label}" + ); + if !streams { + let Err(error) = image.decode_native_region_components_rows_with_context( + (0, 0, width, height), + &mut DecoderContext::default(), + ) else { + panic!("{label}: the rows decode allocated its planes and decoded"); + }; + assert!( + matches!( + error, + DecodeError::Decoding(DecodingError::UnsupportedFeature(_)) + ), + "{label}: {error:?}" + ); + } + } +} diff --git a/crates/j2k/src/view.rs b/crates/j2k/src/view.rs index 1ec0ef36..bd383f66 100644 --- a/crates/j2k/src/view.rs +++ b/crates/j2k/src/view.rs @@ -300,9 +300,10 @@ impl<'a> J2kDecoder<'a> { /// /// Like 16-bit region decodes, it packs rows into the returned planes as /// they are finished, without region-sized `f32` planes, for RGB images - /// without palette, channel definition, sYCC or `CIELab` whose components - /// share one precision, and for other such images when the planes would - /// pass the native decode budget. + /// without palette, channel definition, sYCC, `CIELab` or image and tile + /// offsets whose components share one precision, and for other such + /// images of up to four components when the planes would pass the + /// native decode budget. /// /// # Errors /// Returns [`J2kError`] when the region is invalid or the codestream cannot diff --git a/xtask/api/stable-api-1.0.implementation-public-api.txt b/xtask/api/stable-api-1.0.implementation-public-api.txt index 0f1fd256..8fdefc3a 100644 --- a/xtask/api/stable-api-1.0.implementation-public-api.txt +++ b/xtask/api/stable-api-1.0.implementation-public-api.txt @@ -2515,6 +2515,7 @@ pub fn j2k_native::Image<'a>::new_with_retained_baseline(&'a [u8], &j2k_native:: pub fn j2k_native::Image<'a>::output_tile_rects(&self) -> j2k_native::Result> pub fn j2k_native::Image<'a>::prepare_region_decoder_with_context<'image, 'context>(&'image self, &'context mut j2k_native::DecoderContext<'a>) -> j2k_native::Result> pub fn j2k_native::Image<'a>::primary_icc_profile(&self) -> core::option::Option<&[u8]> +pub fn j2k_native::Image<'a>::region_decode_streams_rows(&self) -> bool pub fn j2k_native::Image<'a>::retained_allocation_bytes(&self) -> j2k_native::Result pub fn j2k_native::Image<'a>::streamed_rows_native_precision(&self) -> core::option::Option pub fn j2k_native::Image<'a>::supports_direct_device_plane_reuse(&self) -> bool From 4ac96d3df16a9edf49f094c048ceb56196947dea Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 15:26:06 -0400 Subject: [PATCH 39/94] fix(j2k-native): stream full images whose tiles have empty coarse levels The full-image planes floor left out every tile whose coarse resolutions are empty while finer ones are not. It was written before both region IDWTs, the whole-window path and the streamed line plan, started such a tile from its first level with samples. Every streamed output (8-bit, 16-bit and native component planes, and each tile's region decode on the tiles route) runs that line plan, so the exclusion only kept such images on the planes past the budget, where they fail. The exclusion and its helper are removed. A 20000 x 7000 gray OpenJPEG image of 2857 x 2333 tiles (a one-sample tile column and row at odd positions) failed decode_into at Gray8 and Gray16 and decode_native_components at the 512 MiB budget. All three now stream, with the bytes of a whole-image region decode and, at 8 bits, the source samples: Gray8 in 3.9 s and 240 MiB RSS on the default pool, 13.9 s and 205 MiB on one thread (x86-64, the 134 MiB output included). Images within the budget take the route the rest of the plan picks for them, with the same bytes. The tests that pinned the planes for such tiles now check that they stream with the planes route's bytes at 8 and 16 bits and into native planes, at full and half resolution. --- crates/j2k-native/src/image/output_api.rs | 8 +- .../j2k-native/src/j2c/decode/planes_floor.rs | 42 ++----- crates/j2k/src/decode/route/tests.rs | 114 +++++++++++++++--- crates/j2k/tests/full_image_streaming.rs | 7 +- .../tests/streamed_sixteen_bit_and_native.rs | 7 +- 5 files changed, 122 insertions(+), 56 deletions(-) diff --git a/crates/j2k-native/src/image/output_api.rs b/crates/j2k-native/src/image/output_api.rs index a616768e..478d9fdc 100644 --- a/crates/j2k-native/src/image/output_api.rs +++ b/crates/j2k-native/src/image/output_api.rs @@ -556,10 +556,10 @@ impl<'a> Image<'a> { /// budget. /// /// `None` when the region decode could not stream (see - /// [`Self::region_decode_streams_rows`], or a tile whose coarse - /// resolutions are empty while finer ones are not), or when the header - /// alone puts the planes and one tile's coefficients and IDWT buffers - /// within half of `budget`. + /// [`Self::region_decode_streams_rows`], or a tile with an inverse MCT + /// over fewer than three components or over wavelets that differ), or + /// when the header alone puts the planes and one tile's coefficients and + /// IDWT buffers within half of `budget`. /// /// # Errors /// diff --git a/crates/j2k-native/src/j2c/decode/planes_floor.rs b/crates/j2k-native/src/j2c/decode/planes_floor.rs index a99a03b0..684a47e6 100644 --- a/crates/j2k-native/src/j2c/decode/planes_floor.rs +++ b/crates/j2k-native/src/j2c/decode/planes_floor.rs @@ -24,7 +24,7 @@ use crate::j2c::codestream::WaveletTransform; use crate::j2c::idwt::planned_workspace_elements; use crate::j2c::rect::IntRect; use crate::j2c::tag_tree::TagNode; -use crate::j2c::tile::{self, ComponentTile, ResolutionTile, Tile}; +use crate::j2c::tile::{self, ResolutionTile, Tile}; use crate::j2c::ComponentData; use crate::math::{SimdBuffer, SIMD_WIDTH}; use crate::reader::BitReader; @@ -33,13 +33,11 @@ use crate::reader::BitReader; /// least, when its full-image region decode could stream its rows instead: /// the image streams as far as its main header says (the caller checks /// `Image::region_decode_streams_rows`), and every tile has its components -/// at full sampling, an inverse MCT only over three or more components whose -/// first three share a wavelet (as `streams_rows` requires), and no -/// tile-component whose coarse resolutions are empty while finer ones are -/// not, which the region IDWTs do not reconstruct. `None` when it could not, -/// or when the header alone puts the planes, one tile's coefficients and its -/// IDWT buffers within half of `budget`, which leaves the planes decode far -/// from it unless its metadata is unusually large. +/// at full sampling and an inverse MCT only over three or more components +/// whose first three share a wavelet (as `streams_rows` requires). `None` +/// when it could not, or when the header alone puts the planes, one tile's +/// coefficients and its IDWT buffers within half of `budget`, which leaves +/// the planes decode far from it unless its metadata is unusually large. /// /// `retained_image_bytes` is what the image holds while it decodes, the /// tile parse's baseline. Errors are the tile parse's, which the planes @@ -62,7 +60,7 @@ pub(crate) fn full_decode_planes_floor( let skipped = header.skipped_resolution_levels; let mut tile_floor = 0_usize; for tile in tiles.iter() { - if !tile_streams_rows(tile, header.component_infos.len(), skipped) { + if !tile_streams_rows(tile, header.component_infos.len()) { return Ok(None); } tile_floor = tile_floor.max(tile_workspace_floor(tile, skipped)); @@ -121,9 +119,9 @@ fn header_estimate(header: &Header<'_>, planes: usize) -> usize { ) } -/// The tile-level conditions of `streams_rows` and the region IDWTs that -/// `tile`'s header settles. -fn tile_streams_rows(tile: &Tile<'_>, components: usize, skipped: u8) -> bool { +/// The tile-level conditions of `streams_rows` that `tile`'s header +/// settles. +fn tile_streams_rows(tile: &Tile<'_>, components: usize) -> bool { let infos = &tile.component_infos; infos.len() == components && infos.iter().all(|component| { @@ -136,26 +134,6 @@ fn tile_streams_rows(tile: &Tile<'_>, components: usize, skipped: u8) -> bool { && infos[1..3].iter().all(|component| { component.wavelet_transform() == infos[0].wavelet_transform() }))) - && !tile - .component_tiles() - .any(|component_tile| has_empty_coarse_resolution(&component_tile, skipped)) -} - -/// Whether a resolution the decode reconstructs, up to its output, is empty -/// while the component tile is not: a one-sample tile row or column at an -/// odd position, say, has no samples at its coarsest levels. -fn has_empty_coarse_resolution(component_tile: &ComponentTile<'_>, skipped: u8) -> bool { - if component_tile.rect.is_empty() { - return false; - } - let output_resolution = component_tile - .component_info - .num_decomposition_levels() - .saturating_sub(skipped); - component_tile - .resolution_tiles() - .take(usize::from(output_resolution) + 1) - .any(|resolution_tile| resolution_tile.rect.is_empty()) } /// What `build` charges for `tile` at least beside the planes: its records, diff --git a/crates/j2k/src/decode/route/tests.rs b/crates/j2k/src/decode/route/tests.rs index 72e791f2..5f25abf8 100644 --- a/crates/j2k/src/decode/route/tests.rs +++ b/crates/j2k/src/decode/route/tests.rs @@ -1,8 +1,11 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use super::{full_image_route, FullImageRoute, PlanesBudget, RouteRequest}; -use crate::backend::image_at_scale; -use crate::decode::{native_components_route, native_region_components_route, NativeRoute}; +use crate::backend::{image_at_scale, Image}; +use crate::decode::{ + decode_full_image_into_with_native_context, decode_native_components_with_native_context, + native_components_route, native_region_components_route, NativeRoute, +}; use crate::DecodeSettings; use j2k_core::{Downscale, PixelFormat, Rect}; use j2k_native::{encode, CpuDecodeParallelism, EncodeOptions}; @@ -139,25 +142,108 @@ fn sixteen_bit_output_streams_when_its_planes_or_their_pack_pass_the_budget() { ); } -/// A tile whose coarse resolutions are empty while finer ones are not keeps -/// the planes, which reconstruct it, whatever the budget. +/// A native decoder context for `parallelism`. +fn context<'a>(parallelism: CpuDecodeParallelism) -> DecoderContext<'a> { + let mut context = DecoderContext::default(); + context.set_cpu_decode_parallelism(parallelism); + context +} + +/// `image` decoded on `route` into a tight `fmt` buffer. +fn decode_on( + image: &Image<'_>, + parallelism: CpuDecodeParallelism, + fmt: PixelFormat, + route: &FullImageRoute, +) -> Vec { + let stride = image.width() as usize * fmt.bytes_per_pixel(); + let mut out = vec![0; stride * image.height() as usize]; + decode_full_image_into_with_native_context( + image, + &mut context(parallelism), + &mut out, + stride, + fmt, + route, + ) + .unwrap_or_else(|error| panic!("{fmt:?} {parallelism:?} {route:?}: {error:?}")); + out +} + +/// A tile whose coarse resolutions are empty while finer ones are not (a +/// one-sample tile row or column at an odd position, with two levels) +/// streams like any other, as the region IDWTs reconstruct it from its first +/// level with samples: the planes floor plans it, a budget of zero puts +/// every output on the streamed route, and that route writes what the planes +/// route writes under a raised budget. (These images are too small for any +/// budget but zero to stream them: their header estimate stays far below.) #[test] -fn tiles_with_empty_coarse_levels_keep_the_planes() { +fn tiles_with_empty_coarse_levels_stream_past_the_budget() { let gray: &[u8] = include_bytes!( "../../../../j2k-test-support/fixtures/empty_coarse_levels/openjpeg_gray_8x6_tile8x5_53.j2k" ); let rgb: &[u8] = include_bytes!( "../../../../j2k-test-support/fixtures/empty_coarse_levels/openjpeg_rgb_6x6_tile5x5_97.j2k" ); - for (bytes, fmt) in [(gray, PixelFormat::Gray8), (rgb, PixelFormat::Rgb8)] { - let image = - image_at_scale(bytes, DecodeSettings::default(), Downscale::None).expect("image"); - assert_eq!(image.full_decode_planes_floor(0).expect("plan"), None); - for parallelism in [CpuDecodeParallelism::Serial, CpuDecodeParallelism::Auto] { - assert_eq!( - full_image_route(&image, request(fmt, 0, parallelism)), - FullImageRoute::Planes - ); + for (bytes, formats) in [ + (gray, [PixelFormat::Gray8, PixelFormat::Gray16]), + (rgb, [PixelFormat::Rgb8, PixelFormat::Rgb16]), + ] { + for scale in [Downscale::None, Downscale::Half] { + let image = image_at_scale(bytes, DecodeSettings::default(), scale).expect("image"); + assert!(floor(bytes, scale) > 0, "{scale:?}"); + for parallelism in [CpuDecodeParallelism::Serial, CpuDecodeParallelism::Auto] { + let label = format!("{:?} {scale:?} {parallelism:?}", image.color_space()); + for fmt in formats { + let streamed = full_image_route(&image, request(fmt, 0, parallelism)); + assert_eq!(streamed, FullImageRoute::StreamedRegion, "{label} {fmt:?}"); + let planes = full_image_route(&image, request(fmt, usize::MAX, parallelism)); + assert_eq!(planes, FullImageRoute::Planes, "{label} {fmt:?}"); + assert_eq!( + decode_on(&image, parallelism, fmt, &streamed), + decode_on(&image, parallelism, fmt, &planes), + "{label} {fmt:?}" + ); + } + let native = |route| { + let decoded = decode_native_components_with_native_context( + &image, + &mut context(parallelism), + None, + route, + ) + .unwrap_or_else(|error| panic!("{label} {route:?}: {error:?}")); + let planes = decoded.into_parts().planes; + planes + .into_iter() + .map(|plane| { + let plane = plane.into_parts(); + ( + plane.data, + plane.dimensions, + plane.bit_depth, + plane.signed, + plane.bytes_per_sample, + ) + }) + .collect::>() + }; + assert_eq!( + native_components_route(&image, 0), + NativeRoute::Rows, + "{label}" + ); + assert_eq!( + native_components_route(&image, usize::MAX), + NativeRoute::Planes, + "{label}" + ); + assert_eq!( + native(NativeRoute::Rows), + native(NativeRoute::Planes), + "{label}" + ); + } } } } diff --git a/crates/j2k/tests/full_image_streaming.rs b/crates/j2k/tests/full_image_streaming.rs index d026d9e2..9a49d7ed 100644 --- a/crates/j2k/tests/full_image_streaming.rs +++ b/crates/j2k/tests/full_image_streaming.rs @@ -169,10 +169,11 @@ fn streamed_full_image_matches_planes() { } /// Tiles whose coarse resolutions are empty while finer ones are not (a -/// one-sample tile row or column at an odd position, with two levels) keep -/// the planes route, which reconstructs them, however low the budget. +/// one-sample tile row or column at an odd position, with two levels) +/// stream past a lowered budget like any other, and the streamed route +/// writes what the planes route writes. #[test] -fn tiles_with_empty_coarse_levels_keep_the_planes() { +fn tiles_with_empty_coarse_levels_stream_as_the_planes_decode() { let gray: &[u8] = include_bytes!( "../../j2k-test-support/fixtures/empty_coarse_levels/openjpeg_gray_8x6_tile8x5_53.j2k" ); diff --git a/crates/j2k/tests/streamed_sixteen_bit_and_native.rs b/crates/j2k/tests/streamed_sixteen_bit_and_native.rs index 36ecba9b..8b76ff16 100644 --- a/crates/j2k/tests/streamed_sixteen_bit_and_native.rs +++ b/crates/j2k/tests/streamed_sixteen_bit_and_native.rs @@ -460,10 +460,11 @@ fn deep_samples_decode_to_their_values_clamped_to_sixteen_bits() { } } -/// Tiles whose coarse resolutions are empty while finer ones are not keep -/// the planes for 16-bit and native component output too. +/// Tiles whose coarse resolutions are empty while finer ones are not stream +/// past a lowered budget for 16-bit and native component output too, with +/// the planes route's bytes. #[test] -fn tiles_with_empty_coarse_levels_keep_the_planes() { +fn tiles_with_empty_coarse_levels_stream_as_the_planes_decode() { let gray: &[u8] = include_bytes!( "../../j2k-test-support/fixtures/empty_coarse_levels/openjpeg_gray_8x6_tile8x5_53.j2k" ); From 94f85a40f3267a0b4e9268ce15128e84ee5c69e0 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 16:25:06 -0400 Subject: [PATCH 40/94] fix(j2k-native): keep native component planes out of the decode budget The rows route treats the native planes it returns as the decode's output: not charged to the decode budget, each held to the per-allocation cap. The planes route (pack_native_component_planes) charged them on top of the f32 planes they are packed from. An image that cannot stream its rows (five components, image offsets), which native region decodes now keep on the planes route, therefore failed with a resource-limit error when its f32 planes fit the budget but not together with its output, though the rows route's planes fallback had decoded some such images before. Both routes now follow one rule. The decode budget charges working memory (the f32 planes, coefficients and workspaces) and checks the f32 planes again before the pack. The returned planes, their owner and the colour space clone are held to an allowance of their own (NativeOutputBudget::for_output), checked before they are allocated, and each plane to the per-allocation cap. j2k's handoff keeps its cap on the retained image and the output together. The 16-bit pack (a RawBitmap the caller never sees) and the exact-integer crop keep charging their intermediates. Route choices are unchanged; the j2k docs now say the pack floor is what the planes route holds, not what it is refused for. Regression test: a 256 x 256 five-component 12-bit image, decoded with a retained baseline that leaves room for its f32 planes but less than its output beside them, decodes to the bytes of a decode with the whole budget (it failed before); with no room for the f32 planes it is still refused. A 5000 x 5000 five-component 8-bit image (f32 planes 477 MiB) now decodes with decode_native_components in 2.7 s and 671 MiB RSS on x86-64 instead of failing at the budget, with the bytes of four quadrant region decodes. packet_directory_output_allocation_retries_successfully pinned a late failure of the owned pack beside the packet cache, which this rule removes. It now checks that a decode with room for its working memory beside the cache keeps the cache, and that one whose working memory fits only without it evicts the cache and retries, with the same planes. Output is unchanged wherever the decode succeeded before. --- crates/j2k-native/src/image/native.rs | 12 ++- .../j2k-native/src/image/native/allocation.rs | 20 +++-- .../src/image/native/allocation/tests.rs | 60 ++++++++++++++- crates/j2k-native/src/image/native/rows.rs | 32 +++++--- .../j2k-native/src/image/packet_directory.rs | 77 +++++++++++-------- crates/j2k/src/decode/output/native.rs | 19 +++-- 6 files changed, 154 insertions(+), 66 deletions(-) diff --git a/crates/j2k-native/src/image/native.rs b/crates/j2k-native/src/image/native.rs index f3a8f863..e3ea7b99 100644 --- a/crates/j2k-native/src/image/native.rs +++ b/crates/j2k-native/src/image/native.rs @@ -28,11 +28,14 @@ impl<'a> Image<'a> { retained_baseline_bytes: usize, parallel: bool, ) -> Result { - let mut budget = NativeOutputBudget::for_decoded_channels( + // The decode budget holds the `f32` planes the pack reads; the packed + // planes are the output and have their own allowance. + NativeOutputBudget::for_decoded_channels( retained_baseline_bytes, components, component_owner_capacity, )?; + let mut budget = NativeOutputBudget::for_output(); budget.include_elements::(components.len())?; for (component_idx, component) in components.iter().enumerate() { let sampling = self.component_plane_sampling_at(component_idx); @@ -95,12 +98,7 @@ impl<'a> Image<'a> { has_alpha: self.has_alpha, planes, }; - NativeOutputBudget::validate_component_pack( - retained_baseline_bytes, - components, - component_owner_capacity, - &packed, - )?; + NativeOutputBudget::validate_component_pack(&packed)?; Ok(packed) } diff --git a/crates/j2k-native/src/image/native/allocation.rs b/crates/j2k-native/src/image/native/allocation.rs index 089bafe1..3df7ebdf 100644 --- a/crates/j2k-native/src/image/native/allocation.rs +++ b/crates/j2k-native/src/image/native/allocation.rs @@ -64,14 +64,18 @@ impl NativeOutputBudget { Ok(budget) } - pub(super) fn validate_component_pack( - retained_image_bytes: usize, - components: &[ComponentData], - component_owner_capacity: usize, - packed: &DecodedNativeComponents, - ) -> Result<()> { - let mut budget = - Self::for_decoded_channels(retained_image_bytes, components, component_owner_capacity)?; + /// The allowance of a decode's returned native component planes. They + /// are its output, like a caller's buffer, so the decode budget, which + /// holds the working memory they are packed from, does not charge them; + /// they are held to this allowance of their own before they are + /// allocated, and each allocation to the per-allocation cap. The planes + /// route and the rows route share it. + pub(super) fn for_output() -> Self { + Self::default() + } + + pub(super) fn validate_component_pack(packed: &DecodedNativeComponents) -> Result<()> { + let mut budget = Self::for_output(); budget.include_elements::(packed.planes.capacity())?; for plane in &packed.planes { budget.include_elements::(plane.data.capacity())?; diff --git a/crates/j2k-native/src/image/native/allocation/tests.rs b/crates/j2k-native/src/image/native/allocation/tests.rs index 41d552fe..9f99a9af 100644 --- a/crates/j2k-native/src/image/native/allocation/tests.rs +++ b/crates/j2k-native/src/image/native/allocation/tests.rs @@ -2,10 +2,13 @@ use super::{bit_capacity_bytes, NativeOutputBudget}; use crate::error::DecodeError; -use crate::j2c::ComponentData; +use crate::j2c::{ComponentData, DecoderContext}; use crate::math::{SimdBuffer, SIMD_WIDTH}; use crate::DEFAULT_MAX_DECODE_BYTES; -use crate::{ColorSpace, ComponentPlane, DecodedComponents, RawBitmap}; +use crate::{ + encode, ColorSpace, ComponentPlane, CpuDecodeParallelism, DecodeErrorClass, DecodeSettings, + DecodedComponents, EncodeOptions, Image, RawBitmap, +}; use alloc::{vec, vec::Vec}; use core::mem::size_of; @@ -169,3 +172,56 @@ fn bit_vector_capacity_is_counted_in_storage_bytes() { if requested > cap && cap == DEFAULT_MAX_DECODE_BYTES )); } + +/// The packed planes of a native component decode are its output, like the +/// rows route's planes: the decode budget holds the `f32` planes they are +/// packed from, not the output beside them. An image that does not stream +/// its rows (five components), given a budget with room for its working +/// memory but not for its `f32` planes and output together, decodes to the +/// bytes a decode with the whole budget gives; one with no room for the +/// `f32` planes is still refused. +#[test] +fn native_component_planes_are_output_beside_the_decode_budget() { + let (width, height, components) = (256_u32, 256_u32, 5_u16); + let samples: Vec = (0..width * height * u32::from(components) * 2) + .map(|index| (index.wrapping_mul(2_654_435_761) >> 24) as u8) + .collect(); + let options = EncodeOptions { + num_decomposition_levels: 2, + tile_size: Some((64, 64)), + ..EncodeOptions::default() + }; + let bytes = encode(&samples, width, height, components, 12, false, &options).expect("encode"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + assert!(!image.region_decode_streams_rows()); + let planes = |retained_baseline_bytes| { + let mut context = DecoderContext::default(); + context.set_cpu_decode_parallelism(CpuDecodeParallelism::Serial); + image + .decode_native_components_with_context_and_retained_baseline( + &mut context, + retained_baseline_bytes, + ) + .map(|decoded| { + decoded + .planes + .into_iter() + .map(|plane| (plane.data, plane.bit_depth, plane.bytes_per_sample)) + .collect::>() + }) + }; + let expected = planes(0).expect("a decode with the whole budget"); + let output: usize = expected.iter().map(|(data, ..)| data.len()).sum(); + assert_eq!(output, 256 * 256 * 5 * 2); + // The `f32` planes without padding or owners: less than the decode holds. + let f32_planes = 256 * 256 * 5 * size_of::(); + let decoded = planes(DEFAULT_MAX_DECODE_BYTES - (f32_planes + output - 1)) + .expect("the output is not charged to the decode budget"); + assert!(decoded == expected, "same planes"); + let error = planes(DEFAULT_MAX_DECODE_BYTES - f32_planes / 2) + .expect_err("the f32 planes pass the budget"); + assert!( + matches!(error.classify(), DecodeErrorClass::ResourceLimit { .. }), + "{error:?}" + ); +} diff --git a/crates/j2k-native/src/image/native/rows.rs b/crates/j2k-native/src/image/native/rows.rs index 573eab29..c4ab0237 100644 --- a/crates/j2k-native/src/image/native/rows.rs +++ b/crates/j2k-native/src/image/native/rows.rs @@ -7,6 +7,7 @@ use alloc::vec::Vec; use super::{ round_clamped_native, signed_native_bounds, try_clone_color_space, unsigned_native_max, + NativeOutputBudget, }; use crate::color::{ ComponentPlane, ComponentRowSink, DecodedNativeComponents, NativeComponentPlane, @@ -57,8 +58,10 @@ impl<'a> Image<'a> { /// The region of the whole image gives /// [`Self::decode_native_components_with_context`]'s planes. /// - /// The returned planes are the decode's output: like a caller's buffer - /// they are not charged to the decode budget, and each is held to the + /// The returned planes are the decode's output: like a caller's buffer, + /// and like the planes the planes route packs, they are not charged to + /// the decode budget but to an allowance of their own (see + /// `NativeOutputBudget::for_output`), and each is held to the /// per-allocation cap. /// /// # Errors @@ -80,17 +83,28 @@ impl<'a> Image<'a> { } let dimensions = (roi.2, roi.3); let samples = checked_decode_sample_count(dimensions.0, dimensions.1)?; + let components = &self.header.component_infos; + let mut allowance = NativeOutputBudget::for_output(); + allowance.include_elements::(components.len())?; + for component in components { + let bytes_per_sample = native_bytes_per_sample(component.size_info.precision)?; + let bytes = checked_decode_byte_len2(samples, bytes_per_sample)?; + allowance.include_elements::(bytes)?; + } + allowance.include_color_space_clone(&self.color_space)?; let mut planes = Vec::new(); - try_reserve_decode_elements(&mut planes, self.header.component_infos.len())?; - for component in &self.header.component_infos { + try_reserve_decode_elements(&mut planes, components.len())?; + allowance.include_capacity_overage::( + components.len(), + planes.capacity(), + )?; + for component in components { let bit_depth = component.size_info.precision; let bytes_per_sample = native_bytes_per_sample(bit_depth)?; + let bytes = checked_decode_byte_len2(samples, bytes_per_sample)?; let mut data = Vec::new(); - try_resize_decode_elements( - &mut data, - checked_decode_byte_len2(samples, bytes_per_sample)?, - 0, - )?; + try_resize_decode_elements(&mut data, bytes, 0)?; + allowance.include_capacity_overage::(bytes, data.capacity())?; planes.push(NativeComponentPlane { data, dimensions, diff --git a/crates/j2k-native/src/image/packet_directory.rs b/crates/j2k-native/src/image/packet_directory.rs index 81736aca..66b1fa1f 100644 --- a/crates/j2k-native/src/image/packet_directory.rs +++ b/crates/j2k-native/src/image/packet_directory.rs @@ -118,21 +118,46 @@ impl PacketDirectoryCache { #[cfg(test)] mod tests { use crate::{ - ColorSpace, CpuDecodeParallelism, DecodeSettings, DecoderContext, Image, - DEFAULT_MAX_DECODE_BYTES, + CpuDecodeParallelism, DecodeSettings, DecoderContext, Image, DEFAULT_MAX_DECODE_BYTES, }; + /// An owned native component decode beside the packet cache: one with + /// room for its working memory and the cache keeps the cache, and one + /// whose working memory fits only without it evicts the cache and + /// retries. (Its planes are output, which the decode budget does not + /// charge, so the cache competes with the working memory only.) #[test] fn packet_directory_output_allocation_retries_successfully() { let bytes = include_bytes!("../../fixtures/packet_directory/openjpeg_rgb_origin5x3_rpcl.j2k"); - let mut image = Image::new(bytes, &DecodeSettings::default()).unwrap(); - // A retained profile is cloned only during output packing. Its size - // makes that phase, rather than coefficient decoding, the peak owner. - image.color_space = ColorSpace::Icc { - profile: vec![0; 1 << 20], - num_channels: 3, + let decode = |image: &Image<'_>, baseline| { + let mut ctx = DecoderContext::default(); + ctx.set_cpu_decode_parallelism(CpuDecodeParallelism::Serial); + image + .decode_native_components_with_context_and_retained_baseline(&mut ctx, baseline) + .map(|decoded| { + decoded + .planes() + .iter() + .map(|plane| plane.data().to_vec()) + .collect::>() + }) }; + let mut control = Image::new(bytes, &DecodeSettings::default()).unwrap(); + control.packet_cache = super::PacketDirectoryCache::disabled(); + let (mut fits, mut fails) = (0, DEFAULT_MAX_DECODE_BYTES); + while fails - fits > 1 { + let candidate = fits + (fails - fits) / 2; + match decode(&control, candidate) { + Ok(_) => fits = candidate, + Err(error) => { + assert!(crate::j2c::is_capacity_error(&error), "{error:?}"); + fails = candidate; + } + } + } + let expected = decode(&control, fits).unwrap(); + let image = Image::new(bytes, &DecodeSettings::default()).unwrap(); let metadata = image.retained_allocation_bytes().unwrap(); let mut ctx = DecoderContext::default(); ctx.set_cpu_decode_parallelism(CpuDecodeParallelism::Serial); @@ -141,31 +166,19 @@ mod tests { .decode_region_components_with_context((60, 40, 16, 16), &mut ctx) .unwrap(); } - let expected = image - .decode_native_components_with_context(&mut ctx) - .unwrap(); - let channels = &ctx.tile_decode_context.channel_data; - let channel_bytes = crate::image::allocation::DecodeOwnerBudget::for_components( - 0, - channels, - channels.capacity(), - ) - .unwrap() - .bytes(); - let baseline = - DEFAULT_MAX_DECODE_BYTES - channel_bytes - expected.allocated_bytes().unwrap(); - // Isolate the late failure: the core already fits with the cache live. - image - .run_decode(&mut ctx, None, None, true, baseline) - .expect("core fits with cache"); - assert!(image.retained_allocation_bytes().unwrap() > metadata); - let actual = image - .decode_native_components_with_context_and_retained_baseline(&mut ctx, baseline) - .expect("owned output must evict and retry"); + let cache = image.retained_allocation_bytes().unwrap() - metadata; + assert!(cache > 0); + // Room for the working memory beside the cache. + assert_eq!( + decode(&image, fits - cache).expect("the cache can stay"), + expected + ); + assert_eq!(image.retained_allocation_bytes().unwrap(), metadata + cache); + assert_eq!( + decode(&image, fits).expect("eviction must restore a fitting request"), + expected + ); assert_eq!(image.retained_allocation_bytes().unwrap(), metadata); - for (actual, expected) in actual.planes().iter().zip(expected.planes()) { - assert_eq!(actual.data(), expected.data()); - } } #[test] diff --git a/crates/j2k/src/decode/output/native.rs b/crates/j2k/src/decode/output/native.rs index d7500ad7..d8c47045 100644 --- a/crates/j2k/src/decode/output/native.rs +++ b/crates/j2k/src/decode/output/native.rs @@ -17,12 +17,14 @@ pub(crate) enum NativeRoute { Rows, } -/// The bytes the planes route charges at least when it packs `image`'s -/// output from its component planes at `bytes_per_sample` native bytes per -/// sample: every `f32` plane and the packed samples. It holds both at once, -/// after the decode's other buffers are released, and refuses a pack past -/// the decode budget. Counts the components of `image`'s color space and -/// alpha only, so it is at most the real charge. +/// The bytes the planes route holds at least when it packs `image`'s output +/// from its component planes at `bytes_per_sample` native bytes per sample: +/// every `f32` plane and the packed samples, both at once, after the +/// decode's other buffers are released. A 16-bit decode's packed samples +/// are working memory, which the decode budget charges and refuses past it; +/// native component planes are the decode's output, which it does not +/// charge. Counts the components of `image`'s color space and alpha only, +/// so it is at most the real amount. pub(in crate::decode) fn planes_pack_floor(image: &Image<'_>, bytes_per_sample: usize) -> usize { region_pack_floor( image, @@ -46,8 +48,9 @@ fn region_pack_floor(image: &Image<'_>, roi: Rect, bytes_per_sample: usize) -> u /// of the whole image that packs its rows into the planes as they are /// finished, when its rows carry the native samples (see /// `Image::streamed_rows_native_precision`) and the planes route would be -/// refused, by its decode (see `Image::full_decode_planes_floor`) or by its -/// pack (see [`planes_pack_floor`]). +/// refused by its decode (see `Image::full_decode_planes_floor`) or would +/// hold more than the budget at once with its packed output (see +/// [`planes_pack_floor`]), which the rows route never holds. /// /// A planning error keeps the planes route, which meets the same error and /// reports it. From 5a1df49456a24df122559a75a1216cb8ebefa469 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 16:25:22 -0400 Subject: [PATCH 41/94] fix(j2k-native): allocate streamed native planes when the first rows arrive decode_native_region_components_rows_with_context allocated every output plane before decoding. A region decode that the plan sends to the rows route but that falls back to component planes at decode time (a component without decomposition levels; classic code-blocks of several segments or mixed HT coding on the pool, which the pool bands do not route) then held the output before it met the decode budget. An RGB image without decomposition levels claiming 12000 x 12000 samples allocated 416 MiB of planes before its resource-limit error (x86-64). The output allowance, and each plane's per-allocation cap, is still checked before the decode, so an output that cannot exist fails first; NativePlanesSink then allocates the planes when the first rectangle arrives, once per decode. An allocation error is kept and returned after the decode, and a decode that hands over no rows fails as before. Any fallback, whatever the eligibility check says, meets the decode budget before output memory exists: the same image now fails at 4.5 MiB. A fallback within the budget hands its planes over in one rectangle and the sink allocates then, as the planes route allocates its pack. Streamed decodes allocate the same planes one rectangle later. Bytes are unchanged, and speed is within noise on x86-64: a 20000 x 7000 gray native decode takes 1.77 s before and after, and an RGB 4096 x 4096 native region 1.010 of before over 15 interleaved rounds, where the A/A spread between rounds is 0.88-1.16. Tests: an RGB image without decomposition levels claiming 12000 x 12000 fails with a resource limit on Serial and Auto before the sink allocated or received anything (test-only counters on the sink); a streamed 96 x 80 RGB image of nine tiles hands over nine or more rectangles, allocates once and gives the planes route's bytes. --- crates/j2k-native/src/image/native/rows.rs | 264 +++++++++++++++++---- crates/j2k/src/decode/output/native.rs | 7 +- 2 files changed, 216 insertions(+), 55 deletions(-) diff --git a/crates/j2k-native/src/image/native/rows.rs b/crates/j2k-native/src/image/native/rows.rs index c4ab0237..711cb8be 100644 --- a/crates/j2k-native/src/image/native/rows.rs +++ b/crates/j2k-native/src/image/native/rows.rs @@ -10,9 +10,10 @@ use super::{ NativeOutputBudget, }; use crate::color::{ - ComponentPlane, ComponentRowSink, DecodedNativeComponents, NativeComponentPlane, + ColorSpace, ComponentPlane, ComponentRowSink, DecodedNativeComponents, NativeComponentPlane, }; -use crate::error::{bail, DecodingError, Result, ValidationError}; +use crate::error::{bail, DecodeError, DecodingError, Result, ValidationError}; +use crate::j2c::codestream::ComponentInfo; use crate::j2c::DecoderContext; use crate::roi::validate_roi; use crate::{ @@ -51,18 +52,17 @@ impl<'a> Image<'a> { /// holds no region-sized `f32` planes. The planes hold the same bytes. /// Only for an image with a [`Self::streamed_rows_native_precision`] /// whose region decodes stream their rows (see - /// [`Self::region_decode_streams_rows`]): the planes are allocated - /// before the decode, so a region decode that fell back to component - /// planes would hold both, and meet the decode budget only after - /// allocating them. + /// [`Self::region_decode_streams_rows`]). /// The region of the whole image gives /// [`Self::decode_native_components_with_context`]'s planes. /// /// The returned planes are the decode's output: like a caller's buffer, /// and like the planes the planes route packs, they are not charged to /// the decode budget but to an allowance of their own (see - /// `NativeOutputBudget::for_output`), and each is held to the - /// per-allocation cap. + /// `NativeOutputBudget::for_output`), checked before the decode, and + /// each is held to the per-allocation cap. They are allocated when the + /// first rows arrive, so a region decode that falls back to component + /// planes meets the decode budget before they exist. /// /// # Errors /// @@ -75,30 +75,126 @@ impl<'a> Image<'a> { roi: (u32, u32, u32, u32), decoder_context: &mut DecoderContext<'a>, ) -> Result { + let dimensions = (roi.2, roi.3); + let mut sink = NativePlanesSink::new(self, dimensions); + self.decode_native_rows(roi, decoder_context, &mut sink)?; + let planes = sink.finish()?; + let color_space = try_clone_color_space(&self.color_space)?; + Ok(DecodedNativeComponents { + dimensions, + color_space, + has_alpha: self.has_alpha, + planes, + }) + } + + /// The region decode of [`Self::decode_native_region_components_rows_with_context`] + /// into `sink`. + fn decode_native_rows( + &self, + roi: (u32, u32, u32, u32), + decoder_context: &mut DecoderContext<'a>, + sink: &mut NativePlanesSink<'_>, + ) -> Result<()> { validate_roi((self.width(), self.height()), roi)?; if self.streamed_rows_native_precision().is_none() || !self.region_decode_streams_rows() { bail!(DecodingError::UnsupportedFeature( "streamed native component planes need one to four unsubsampled components of one precision up to 24 bits, no image or tile offsets, and no palette, channel definition, sYCC or CIELab" )); } - let dimensions = (roi.2, roi.3); - let samples = checked_decode_sample_count(dimensions.0, dimensions.1)?; - let components = &self.header.component_infos; + // The planes are allocated when the first rows arrive, but their + // allowance is known now: an output that cannot exist fails before + // the decode. + sink.output_allowance()?; + self.decode_region_rows_for_integer_output_with_context(roi, decoder_context, sink) + } +} + +/// Packs each rectangle of a region decode into the native planes it +/// covers. The planes are allocated when the first rectangle arrives, once +/// per decode. The first error, an allocation or a rectangle that does not +/// fit the planes, stops every later write and fails the decode. +struct NativePlanesSink<'i> { + components: &'i [ComponentInfo], + color_space: &'i ColorSpace, + dimensions: (u32, u32), + planes: Vec, + /// One row of samples wider than 16 bits, packed before it is copied. + scratch: Vec, + error: Option, + /// Times the planes were allocated. + #[cfg(test)] + allocations: usize, + /// Rectangles handed to the sink. + #[cfg(test)] + writes: usize, +} + +impl ComponentRowSink for NativePlanesSink<'_> { + fn write_rows(&mut self, x: u32, y: u32, planes: &[ComponentPlane<'_>]) { + #[cfg(test)] + { + self.writes += 1; + } + if self.error.is_some() { + return; + } + if self.planes.is_empty() { + if let Err(error) = self.allocate() { + self.error = Some(error); + return; + } + } + if self.try_write_rows(x, y, planes).is_none() { + self.error = Some(DecodingError::CodeBlockDecodeFailure.into()); + } + } +} + +impl<'i> NativePlanesSink<'i> { + /// A sink for the `dimensions` of a region of `image`, with no planes + /// yet. + fn new(image: &'i Image<'_>, dimensions: (u32, u32)) -> Self { + Self { + components: &image.header.component_infos, + color_space: &image.color_space, + dimensions, + planes: Vec::new(), + scratch: Vec::new(), + error: None, + #[cfg(test)] + allocations: 0, + #[cfg(test)] + writes: 0, + } + } + + /// The planes held, as the planes route holds its packed planes, to the + /// output allowance, and each to the per-allocation cap. + fn output_allowance(&self) -> Result { + let samples = checked_decode_sample_count(self.dimensions.0, self.dimensions.1)?; let mut allowance = NativeOutputBudget::for_output(); - allowance.include_elements::(components.len())?; - for component in components { + allowance.include_elements::(self.components.len())?; + for component in self.components { let bytes_per_sample = native_bytes_per_sample(component.size_info.precision)?; let bytes = checked_decode_byte_len2(samples, bytes_per_sample)?; allowance.include_elements::(bytes)?; } - allowance.include_color_space_clone(&self.color_space)?; + allowance.include_color_space_clone(self.color_space)?; + Ok(allowance) + } + + /// Allocate every component's plane within the output allowance. + fn allocate(&mut self) -> Result<()> { + let samples = checked_decode_sample_count(self.dimensions.0, self.dimensions.1)?; + let mut allowance = self.output_allowance()?; let mut planes = Vec::new(); - try_reserve_decode_elements(&mut planes, components.len())?; + try_reserve_decode_elements(&mut planes, self.components.len())?; allowance.include_capacity_overage::( - components.len(), + self.components.len(), planes.capacity(), )?; - for component in components { + for component in self.components { let bit_depth = component.size_info.precision; let bytes_per_sample = native_bytes_per_sample(bit_depth)?; let bytes = checked_decode_byte_len2(samples, bytes_per_sample)?; @@ -107,7 +203,7 @@ impl<'a> Image<'a> { allowance.include_capacity_overage::(bytes, data.capacity())?; planes.push(NativeComponentPlane { data, - dimensions, + dimensions: self.dimensions, bit_depth, signed: component.size_info.signed, sampling: (1, 1), @@ -115,43 +211,26 @@ impl<'a> Image<'a> { .map_err(|_| ValidationError::ImageTooLarge)?, }); } - let color_space = try_clone_color_space(&self.color_space)?; - let mut sink = NativePlanesSink { - planes: &mut planes, - scratch: Vec::new(), - failed: false, - }; - self.decode_region_rows_for_integer_output_with_context(roi, decoder_context, &mut sink)?; - if sink.failed { - bail!(DecodingError::CodeBlockDecodeFailure); + self.planes = planes; + #[cfg(test)] + { + self.allocations += 1; } - Ok(DecodedNativeComponents { - dimensions, - color_space, - has_alpha: self.has_alpha, - planes, - }) + Ok(()) } -} -/// Packs each rectangle of a region decode into the native planes it -/// covers. A rectangle that does not fit them fails the decode. -struct NativePlanesSink<'p> { - planes: &'p mut [NativeComponentPlane], - /// One row of samples wider than 16 bits, packed before it is copied. - scratch: Vec, - failed: bool, -} - -impl ComponentRowSink for NativePlanesSink<'_> { - fn write_rows(&mut self, x: u32, y: u32, planes: &[ComponentPlane<'_>]) { - if !self.failed && self.try_write_rows(x, y, planes).is_none() { - self.failed = true; + /// The packed planes, or the first error a rectangle met. A decode that + /// handed over no rows has no planes. + fn finish(self) -> Result> { + if let Some(error) = self.error { + return Err(error); + } + if self.planes.is_empty() { + bail!(DecodingError::CodeBlockDecodeFailure); } + Ok(self.planes) } -} -impl NativePlanesSink<'_> { fn try_write_rows(&mut self, x: u32, y: u32, sources: &[ComponentPlane<'_>]) -> Option<()> { if sources.len() != self.planes.len() { return None; @@ -253,9 +332,13 @@ pub(super) fn pack_native_samples( mod tests { use alloc::vec::Vec; - use super::pack_native_samples; + use super::{pack_native_samples, NativePlanesSink}; use crate::image::Image; - use crate::native_bytes_per_sample; + use crate::j2c::DecoderContext; + use crate::{ + encode, native_bytes_per_sample, CpuDecodeParallelism, DecodeErrorClass, DecodeSettings, + EncodeOptions, + }; /// Samples around every bound and rounding tie of each precision, plus /// values far outside it and NaN. @@ -295,4 +378,83 @@ mod tests { } } } + + fn context<'a>(parallelism: CpuDecodeParallelism) -> DecoderContext<'a> { + let mut context = DecoderContext::default(); + context.set_cpu_decode_parallelism(parallelism); + context + } + + /// The rows route allocates its planes when the first rows arrive, once + /// per decode. A region decode that falls back to component planes (here + /// an RGB image without decomposition levels, which has no Tier-1 bands + /// to stream) meets the decode budget before any plane exists: past it, + /// the decode fails with a resource limit and the sink allocated + /// nothing. Headers only: its SIZ claims 12000 x 12000 samples, whose + /// `f32` planes pass the budget before any packet is read. + #[test] + fn native_rows_allocate_the_planes_when_the_first_rows_arrive() { + let options = EncodeOptions { + num_decomposition_levels: 0, + ..EncodeOptions::default() + }; + let mut bytes = encode(&[0; 16 * 16 * 3], 16, 16, 3, 8, false, &options).expect("encode"); + let siz = bytes + .windows(2) + .position(|marker| marker == [0xff, 0x51]) + .expect("SIZ marker"); + for field in [6, 10, 22, 26] { + bytes[siz + field..siz + field + 4].copy_from_slice(&12_000_u32.to_be_bytes()); + } + let image = Image::new(&bytes, &DecodeSettings::default()).expect("headers parse"); + assert!(image.region_decode_streams_rows()); + for parallelism in [CpuDecodeParallelism::Serial, CpuDecodeParallelism::Auto] { + let mut sink = NativePlanesSink::new(&image, (12_000, 12_000)); + let error = image + .decode_native_rows((0, 0, 12_000, 12_000), &mut context(parallelism), &mut sink) + .expect_err("the planes pass the budget"); + assert!( + matches!(error.classify(), DecodeErrorClass::ResourceLimit { .. }), + "{parallelism:?}: {error:?}" + ); + assert_eq!((sink.allocations, sink.writes), (0, 0), "{parallelism:?}"); + assert!(sink.planes.is_empty(), "{parallelism:?}"); + } + } + + /// A streamed region decode hands its rows over in many rectangles, one + /// or more per tile, and the sink allocates its planes for the first of + /// them only. The planes hold the planes route's bytes. + #[test] + fn streamed_native_rows_allocate_the_planes_once() { + let (width, height) = (96_u32, 80_u32); + let samples: Vec = (0..width * height * 3) + .map(|index| (index.wrapping_mul(2_654_435_761) >> 24) as u8) + .collect(); + let options = EncodeOptions { + num_decomposition_levels: 2, + tile_size: Some((32, 32)), + ..EncodeOptions::default() + }; + let bytes = encode(&samples, width, height, 3, 8, false, &options).expect("encode"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let roi = (0, 0, width, height); + for parallelism in [CpuDecodeParallelism::Serial, CpuDecodeParallelism::Auto] { + let mut sink = NativePlanesSink::new(&image, (width, height)); + image + .decode_native_rows(roi, &mut context(parallelism), &mut sink) + .expect("streamed decode"); + assert_eq!(sink.allocations, 1, "{parallelism:?}"); + let writes = sink.writes; + assert!(writes >= 9, "{parallelism:?}: {writes} rectangles"); + let streamed = sink.finish().expect("planes"); + let expected = image + .decode_native_region_components_with_context(roi, &mut context(parallelism)) + .expect("planes route"); + assert_eq!(streamed.len(), expected.planes.len(), "{parallelism:?}"); + for (streamed, expected) in streamed.iter().zip(&expected.planes) { + assert!(streamed.data == expected.data, "{parallelism:?}"); + } + } + } } diff --git a/crates/j2k/src/decode/output/native.rs b/crates/j2k/src/decode/output/native.rs index d8c47045..fd0395fe 100644 --- a/crates/j2k/src/decode/output/native.rs +++ b/crates/j2k/src/decode/output/native.rs @@ -86,10 +86,9 @@ pub(in crate::decode) fn region_rows_preferred( /// finished, so that it holds no region-sized `f32` planes, when its rows /// carry the native samples, the region decode streams them (see /// `Image::region_decode_streams_rows`, which the full-image floor checks -/// too) and [`region_rows_preferred`]. The rows route allocates the planes -/// before it decodes, so an image whose region decode would fall back to -/// component planes keeps the planes route, which meets the decode budget -/// before allocating them. +/// too) and [`region_rows_preferred`]. An image whose region decode would +/// fall back to component planes keeps the planes route, which decodes and +/// charges them the same way. pub(crate) fn native_region_components_route(image: &Image<'_>, roi: Rect) -> NativeRoute { match image.streamed_rows_native_precision() { Some(precision) From 14198f3a9304ef9dd4d79ecddcc39cac75934264 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 16:25:22 -0400 Subject: [PATCH 42/94] docs(changelog): note native component planes as decode output --- CHANGELOG.md | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0b02ae9..e53a78e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,7 +81,24 @@ Staged workspace version: `0.13.2`. images without palette, channel definitions, sYCC or CIELab stream their rows instead of holding the region's `f32` planes and native samples, and so do those of other such images whose planes would pass the native decode - budget. Output is unchanged. + budget. `decode_native_region_components` takes this route only for images + of one to four components without image or tile offsets, whose region + decodes stream; others keep the planes route, which meets the decode budget + before it allocates the returned planes. Output is unchanged. +- Native component planes (`decode_native_components`, + `decode_native_region_components` and the native decoder's component-plane + decodes) are the decode's output on both routes: the decode budget holds + the `f32` planes they are packed from, and the returned planes have an + allowance of their own, each held to the per-allocation cap, as a + caller's buffer is. Before, the planes route charged them beside the + `f32` planes, so an image whose `f32` planes fit the budget but not + together with their output (one that cannot stream its rows, such as one + of five components or with image offsets) failed with a resource-limit + error; it now decodes. The streamed rows route allocates its planes when + the first rows arrive, once per decode, so a region decode that falls back + to component planes (a component without decomposition levels, or classic + code-blocks of several segments on the pool) meets the decode budget before + any output exists. Output is unchanged. - 16-bit outputs of images deeper than 16 bits hold each sample's value clamped to the 16-bit range (two's complement when signed), as scaled decodes already wrote them. Full-image and region decodes (and row decodes @@ -115,9 +132,11 @@ Staged workspace version: `0.13.2`. peak heap on the default thread pool (26 MiB on one thread), where the planes route with its budget raised to fit needs about 980 MiB. Untiled RGB images decode within 8% of that route's time, 1024 x 1024-tiled RGB takes - up to 45% longer and grayscale up to 2.8 times as long. 16-bit outputs, and - images whose tiles have empty coarse resolutions, keep the planes route - and its error. + up to 45% longer and grayscale up to 2.8 times as long. Images whose tiles + have empty coarse resolutions (a one-sample tile row or column at an odd + position) stream too, at 8 and 16 bits and into native component planes, + since region decodes reconstruct such tiles: a 20000 x 7000 gray image of + 2857 x 2333 tiles no longer fails at the decode budget. - Full-image 8-bit decodes also stream when streaming was measured to be the faster route for the image, chosen before decoding from the codestream's header and tiles: reduced-resolution decodes (`decode_scaled_into`) of From 9daffe4966c0710a46d7aff10be53d79d1669117 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 17:40:45 -0400 Subject: [PATCH 43/94] perf(j2k-native): read wide empty band rows through the streamed IDWT's SIMD kernel The row-streamed IDWT reads a band row that lies wholly outside its stored rectangle, as rows of code-blocks without coded data do, from a static zero row of 4096 samples. A wider such row found no zero row, so its whole output row took the scalar per-sample interleave and filter. The zero row now holds 16384 samples (64 KiB, zero-initialised static), which covers the band rows of levels up to 32768 wide; wider rows keep the scalar path. An 18000 x 12000 RGB image whose chroma has many empty code-block rows (its second chroma component's finest HH band has 14 of 94 rows of code-blocks) reconstructed that component 1.6 times slower than the others, and every row batch waited for it. On an 18-core ARM64 machine (7 interleaved rounds against the previous commit) its full-image decodes at 18 and default threads are 11 to 12% faster (441 to 391 ms) and 2 to 3% faster at 6 threads; on a 10-core one 5% faster at 10 threads. Other images are unchanged. Output is bit-identical. --- crates/j2k-native/src/j2c/idwt/lines.rs | 7 ++- crates/j2k-native/src/j2c/idwt/tests.rs | 64 +++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/crates/j2k-native/src/j2c/idwt/lines.rs b/crates/j2k-native/src/j2c/idwt/lines.rs index 88a7a9de..6026b324 100644 --- a/crates/j2k-native/src/j2c/idwt/lines.rs +++ b/crates/j2k-native/src/j2c/idwt/lines.rs @@ -460,8 +460,11 @@ fn feed_row(levels: &mut [LevelRows<'_>], level: usize, inputs: &mut Inputs<'_>) } /// Zero samples for band rows that lie wholly outside their stored -/// rectangles, as sub-bands without code-blocks in a row do. -static ZERO_ROW: [f32; 4096] = [0.0; 4096]; +/// rectangles, as sub-bands without code-blocks in a row do: as wide as the +/// band rows of a 32768-wide level. A wider zero row takes the scalar path, +/// which made the empty 9000-wide band rows of an 18000-wide image's chroma +/// 1.6 times slower to reconstruct than its luma. +static ZERO_ROW: [f32; 16384] = [0.0; 16384]; /// The `len` samples of row `band_y` of `source` from band column `band_x0`, /// which read zero outside its rectangle as [`CoefficientSource::get`] does: diff --git a/crates/j2k-native/src/j2c/idwt/tests.rs b/crates/j2k-native/src/j2c/idwt/tests.rs index 934433d1..c09e623d 100644 --- a/crates/j2k-native/src/j2c/idwt/tests.rs +++ b/crates/j2k-native/src/j2c/idwt/tests.rs @@ -842,6 +842,70 @@ fn streamed_rows_match_the_whole_window_levels_bit_for_bit() { } } +/// Band rows wider than 4096 samples that hold no code-block, as in the +/// chroma of an 18000-wide image, read as zero rows through the banded SIMD +/// kernel like narrower ones; every row still matches the whole-window region +/// IDWT bit for bit, with either high band empty. +#[test] +fn wide_empty_band_rows_match_the_whole_window_level_bit_for_bit() { + use super::lines::{reconstruct_rows, ring_samples}; + + for transform in [ + WaveletTransform::Reversible53, + WaveletTransform::Irreversible97, + ] { + let integral = transform == WaveletTransform::Reversible53; + for (x0, empty) in [(0, 3), (1, 2), (1, 3)] { + let window = IntRect::from_xywh(x0, 2, 16_400, 6); + let mut seed = 31_u64; + let bands: Vec<(Vec, IntRect)> = super::banded::packed_band_rects(window) + .into_iter() + .enumerate() + .map(|(index, rect)| { + let stored = if index == empty { + IntRect::from_xywh(0, 0, 0, 0) + } else { + rect + }; + seed += 1; + let len = (stored.width() * stored.height()) as usize; + (pseudo_random_coefficients(len, seed, integral), stored) + }) + .collect(); + let sources: [CoefficientSource<'_>; 4] = core::array::from_fn(|index| { + let (samples, rect) = &bands[index]; + CoefficientSource::new(samples, *rect, rect.width()) + }); + let expected = whole_window_level(sources, window, transform); + let mut rows = StoredBands { bands }; + let mut ring = vec![f32::NAN; ring_samples([window], transform).expect("ring")]; + let mut streamed = vec![f32::NAN; expected.len()]; + let mut output_samples = 0; + reconstruct_rows( + &mut rows, + &[[1, 2, 3]], + &[window], + 0, + transform, + &mut ring, + window, + &mut |y, samples| { + let start = ((y - window.y0) * window.width()) as usize; + streamed[start..][..samples.len()].copy_from_slice(samples); + Ok(()) + }, + &mut output_samples, + ) + .expect("streamed level"); + assert_eq!( + bits(&streamed), + bits(&expected), + "{transform:?} origin x {x0}, empty band {empty}" + ); + } + } +} + /// A tile streaming from pool bands with deep band rings holds one batch of /// rows instead of two, so its batch takes half the rows unless that leaves /// fewer than 128; tiles too short for full batches still split their rows From 39db1c3e2842200ec32edecb0ae84fb52df62bc3 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 17:40:45 -0400 Subject: [PATCH 44/94] perf(j2k-native): reconstruct large classic tiles on a reader per component from 10 threads Streamed classic tiles of several components reconstructed every component on one reader below 12 pool threads; that reader also decoded bands while it waited and packed every row alone. On a 10-thread pool it bounded large tiles: an instrumented decode of an 18000 x 12000 RGB image kept it busy for the whole 1000 ms while 2.5 of the 10 workers idled. From 10 threads, a classic tile of at least 4096 x 4096 samples per component now reconstructs each component on a reader of its own, as from 12 threads, but in one batch of rows, so peak heap is unchanged; deep band rings stay at 12 threads. Smaller tiles spend more of their time in Tier-1, from which the readers take two pool threads, and keep one reader: with readers, a 4096 x 2048 lossless RGB image and the 1/2 to 1/8 reduced decodes of a 5661 x 8287 RGB one were 1 to 11% slower on 10 threads. So do pools of fewer than 10 threads: on 8, readers were 4 to 5% slower. On a 10-core ARM64 machine at 10 threads (7 interleaved rounds against the previous commit): 18000 x 12000 RGB 980 -> 868 ms, 5661 x 8287 RGB 269 -> 248 ms, 16384 x 16384 RGB 1770 -> 1645 ms, 8192 x 8192 RGB in one tile 469 -> 447 ms; tiles below the size, 1024 x 1024 regions and reduced decodes run as before; peak heap equal. 1 to 9 and 12 or more threads run as before. Output is bit-identical. --- .../src/j2c/decode/parallel_bands.rs | 67 ++++++++++++------- 1 file changed, 44 insertions(+), 23 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index 0103f91b..9fcd5e95 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -383,30 +383,30 @@ pub(super) fn decode_tile_rows<'a>( .map(|route| route.workers()) .max() .unwrap_or(0); - let readers = if parallel_synthesis(components, coding, bank, route_workers) { + let samples = (copy.width() as usize).saturating_mul(copy.height() as usize); + let readers = if parallel_synthesis(components, coding, bank, route_workers, samples) { components } else { 1 }; // Rows reconstruct in batches, charged by the region's width (see - // `roi_idwt_peak_elements`): two of them with a reader per component, so - // one finishes while the next reconstructs, unless the band rings are - // deep instead: then one batch of half the rows. + // `roi_idwt_peak_elements`): two of them with a reader per component of + // an HTJ2K tile, so one finishes while the next reconstructs. A classic + // tile holds one batch, of half the rows when its band rings are deep. let batch = streamed_batch_rows( region.width() as usize, copy.height() as usize, storage.deep_band_rings, ); + let batches = if readers > 1 && coding == CodeBlockCoding::HighThroughput { + 2 + } else { + 1 + }; let scratch_len = (copy.width() as usize) .checked_mul(components) .and_then(|samples| samples.checked_mul(batch)) - .and_then(|samples| { - samples.checked_mul(if readers > 1 && !storage.deep_band_rings { - 2 - } else { - 1 - }) - }) + .and_then(|samples| samples.checked_mul(batches)) .ok_or(ValidationError::ImageTooLarge)?; let (mut buffer, ring_lens) = banded_tier1::take_line_buffer(tile_ctx, storage, &line_plans, scratch_len)?; @@ -536,11 +536,12 @@ pub(super) fn decode_tile_rows<'a>( /// Whether a tile streaming its rows from pool bands holds deeper band rings /// (see `DecompositionStorage::deep_band_rings`): a classic tile of several -/// components with the pool threads to reconstruct each on its own reader -/// (see [`parallel_synthesis`]). Its readers wait on classic Tier-1, which -/// more bands in flight keep busier; the memory comes from holding one batch -/// of rows instead of two. HTJ2K bands keep up with the shallower rings, -/// and its readers gain more from the second batch. +/// components on a pool of [`MIN_CLASSIC_SYNTHESIS_THREADS`] threads or +/// more, which reconstructs each on its own reader (see +/// [`parallel_synthesis`]). Its readers wait on classic Tier-1, which more +/// bands in flight keep busier; the memory comes from holding a batch of +/// half the rows. HTJ2K bands keep up with the shallower rings, and its +/// readers gain more from the second batch. pub(super) fn deep_rings(tile: &Tile<'_>) -> bool { tile.component_infos.len() > 1 && rayon::current_num_threads() >= MIN_CLASSIC_SYNTHESIS_THREADS @@ -553,26 +554,46 @@ pub(super) fn deep_rings(tile: &Tile<'_>) -> bool { }) } -/// Pool threads a classic tile needs before its components reconstruct on -/// the pool. A reader waiting for a band holds its thread, and classic -/// Tier-1 keeps readers waiting, so a smaller pool does better with every -/// thread decoding bands. +/// Pool threads from which a classic tile reconstructs each component on its +/// own reader whatever its size, and its band rings are deep. const MIN_CLASSIC_SYNTHESIS_THREADS: usize = 12; +/// Pool threads from which a classic tile of [`MIN_CLASSIC_READER_SAMPLES`] +/// or more reconstructs each component on its own reader below +/// [`MIN_CLASSIC_SYNTHESIS_THREADS`]. A reader waiting for a band holds its +/// thread, and classic Tier-1 keeps readers waiting, so a smaller pool does +/// better with one reader that decodes bands while it waits (4-7% on 8 +/// threads). From 10 threads that one reader, which reconstructs every +/// component and packs every row alone, bounds a large tile: a reader per +/// component decodes 18000 x 12000 RGB 11-19% and 5661 x 8287 RGB 8-13% +/// faster on 10 and 11 threads. +const MIN_CLASSIC_READER_THREADS: usize = 10; + +/// Samples of each component from which a classic tile takes a reader per +/// component on [`MIN_CLASSIC_READER_THREADS`] threads. Smaller tiles spend +/// more of their time in Tier-1, from which the readers take two pool +/// threads: on 10 threads a 4096 x 2048 lossless RGB image and the 2830 x +/// 4143 to 708 x 1036 reduced decodes of a 5661 x 8287 one were 1-11% slower +/// with them. +const MIN_CLASSIC_READER_SAMPLES: usize = 1 << 24; + /// Whether a streamed tile's `components` reconstruct on the pool, one per -/// task, given its block coding, a workspace bank of `bank` and routes -/// planned for `route_workers` pool threads. +/// task, given its block coding, a workspace bank of `bank`, routes planned +/// for `route_workers` pool threads and `samples` reconstructed per component. fn parallel_synthesis( components: usize, coding: CodeBlockCoding, bank: usize, route_workers: usize, + samples: usize, ) -> bool { + let threads = rayon::current_num_threads(); components > 1 && route_workers >= 2 && bank > components && (coding == CodeBlockCoding::HighThroughput - || rayon::current_num_threads() >= MIN_CLASSIC_SYNTHESIS_THREADS) + || threads >= MIN_CLASSIC_SYNTHESIS_THREADS + || (threads >= MIN_CLASSIC_READER_THREADS && samples >= MIN_CLASSIC_READER_SAMPLES)) } /// The route of a band's `strip`, counted over every component's route from From 86601a87b92c4ac486f8e25b7370c2e0859c8564 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 15:36:49 -0400 Subject: [PATCH 45/94] perf(j2k-native): hold 32 rows per batch when one reader reconstructs every component A tile streaming its rows from pool bands through one reader (one component, or classic code-blocks on fewer than 12 pool threads) reconstructs a batch of rows of every component, then transforms and hands it over before it reconstructs the next one. Nothing overlaps that batch, yet it held up to 2 Mi samples per component (370 rows of a 5661-wide tile, 512 of a 4096-wide one). Such a tile now holds at most 32 rows of each component. The charge of the streamed rows follows: a tile that can only have one reader charges one such batch instead of two full ones; tiles with a reader per component keep their batches and charge. Measured with full-image decodes to RGB8 against the parent on an 18-core ARM64 machine (7 interleaved fresh-process rounds and 7 warm decodes per cell, output byte-identical), and with 1 to 128 rows per batch on a 10-core ARM64 machine (5 to 7 rounds), where every size from 1 row up decoded as fast as 370: - 5661 x 8287 RGB, one 9/7 tile, 6 threads: peak heap 66.9 -> 45.0 MiB (-21.9 MiB, -33%), footprint (max RSS less input and output) 69.7 -> 47.8 MiB, warm 1.00x, cold 0.99x. At 1/2, 1/4 and 1/8 scale -15.5, -7.7 and -1.8 MiB at 0.98-1.00x. At 18 and default threads (a reader per component) and on one thread nothing changes. - 4096 x 2048 classic RGB, 18000 x 12000 and 16384 x 16384 RGB at 6 threads: -22.5, -17.3 and -18.0 MiB at 0.99-1.02x; 5792 and 8192 square gray at 6, 18 and default threads: -7.3 and -7.0 MiB. --- crates/j2k-native/src/j2c/decode.rs | 2 + .../src/j2c/decode/parallel_bands.rs | 19 +++++++ crates/j2k-native/src/j2c/decode/state.rs | 5 ++ crates/j2k-native/src/j2c/idwt/roi.rs | 34 ++++++++++--- crates/j2k-native/src/j2c/idwt/tests.rs | 50 +++++++++++++++---- 5 files changed, 93 insertions(+), 17 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode.rs b/crates/j2k-native/src/j2c/decode.rs index 13e8d4b5..dd32a701 100644 --- a/crates/j2k-native/src/j2c/decode.rs +++ b/crates/j2k-native/src/j2c/decode.rs @@ -918,6 +918,8 @@ fn prepare_region_storage( { storage.deep_band_rings = storage.parallel_bands && storage.lockstep_lines && parallel_bands::deep_rings(tile); + storage.one_synthesis_reader = + storage.parallel_bands && storage.lockstep_lines && parallel_bands::one_reader(tile); } build::allocate_deferred_coefficients( storage, diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index 9fcd5e95..295441f3 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -393,10 +393,13 @@ pub(super) fn decode_tile_rows<'a>( // `roi_idwt_peak_elements`): two of them with a reader per component of // an HTJ2K tile, so one finishes while the next reconstructs. A classic // tile holds one batch, of half the rows when its band rings are deep. + // One reader holds one short batch, which it finishes before it + // reconstructs the next. let batch = streamed_batch_rows( region.width() as usize, copy.height() as usize, storage.deep_band_rings, + readers == 1, ); let batches = if readers > 1 && coding == CodeBlockCoding::HighThroughput { 2 @@ -554,6 +557,22 @@ pub(super) fn deep_rings(tile: &Tile<'_>) -> bool { }) } +/// Whether a tile streaming its rows from pool bands reconstructs them +/// through one reader whatever workers its routes plan (see +/// [`parallel_synthesis`]): it has one component, or classic code-blocks on +/// fewer pool threads than their components need to reconstruct on the pool. +pub(super) fn one_reader(tile: &Tile<'_>) -> bool { + tile.component_infos.len() < 2 + || (rayon::current_num_threads() < MIN_CLASSIC_SYNTHESIS_THREADS + && tile.component_infos.iter().all(|component| { + !component + .coding_style + .parameters + .code_block_style + .uses_high_throughput_block_coding() + })) +} + /// Pool threads from which a classic tile reconstructs each component on its /// own reader whatever its size, and its band rings are deep. const MIN_CLASSIC_SYNTHESIS_THREADS: usize = 12; diff --git a/crates/j2k-native/src/j2c/decode/state.rs b/crates/j2k-native/src/j2c/decode/state.rs index 84822eaf..9b7e6269 100644 --- a/crates/j2k-native/src/j2c/decode/state.rs +++ b/crates/j2k-native/src/j2c/decode/state.rs @@ -125,6 +125,10 @@ pub(crate) struct DecompositionStorage<'a> { /// every level, for a classic tile that streams its rows from a reader /// per component (see [`parallel_bands::deep_rings`]). pub(crate) deep_band_rings: bool, + /// Whether a tile streaming its rows from `parallel_bands` reconstructs + /// them through one reader whatever its pool's workers (see + /// [`parallel_bands::one_reader`]), which holds one short batch of rows. + pub(crate) one_synthesis_reader: bool, /// Whether every component's IDWT rings, and a scratch row each, are /// live at once, because the tile streams its rows to a sink. pub(crate) lockstep_lines: bool, @@ -167,6 +171,7 @@ impl DecompositionStorage<'_> { self.banded_tier1 = false; self.parallel_bands = false; self.deep_band_rings = false; + self.one_synthesis_reader = false; self.lockstep_lines = false; self.roi_idwt_charge_elements = 0; self.tier1_band = None; diff --git a/crates/j2k-native/src/j2c/idwt/roi.rs b/crates/j2k-native/src/j2c/idwt/roi.rs index 75752e29..3d185a08 100644 --- a/crates/j2k-native/src/j2c/idwt/roi.rs +++ b/crates/j2k-native/src/j2c/idwt/roi.rs @@ -30,14 +30,27 @@ const STREAMED_MIN_BATCHES: usize = 4; /// halved: halving the 116-row batches of an 18000-wide tile cost 2 to 5%. const STREAMED_DEEP_MIN_ROWS: usize = 128; +/// Rows of each component a tile reconstructing through one reader holds +/// at most in its batch. The reader transforms and hands over each batch +/// before it reconstructs the next, so longer batches overlap nothing. +const STREAMED_ONE_READER_ROWS: usize = 32; + /// Rows of each component a tile streaming from pool bands reconstructs at /// once, for `height` rows of `width` samples. A tile whose band rings are /// deep holds one batch, which is transformed before the next one /// reconstructs, so its batches take half the rows while that leaves at -/// least `STREAMED_DEEP_MIN_ROWS`: as fast, with half the rows held. -pub(crate) fn streamed_batch_rows(width: usize, height: usize, deep_band_rings: bool) -> usize { +/// least `STREAMED_DEEP_MIN_ROWS`: as fast, with half the rows held. A tile +/// with `one_reader` holds one batch of at most `STREAMED_ONE_READER_ROWS`. +pub(crate) fn streamed_batch_rows( + width: usize, + height: usize, + deep_band_rings: bool, + one_reader: bool, +) -> usize { let rows = STREAMED_BATCH_SAMPLES / width.max(1); - let rows = if deep_band_rings && rows / 2 >= STREAMED_DEEP_MIN_ROWS { + let rows = if one_reader { + rows.min(STREAMED_ONE_READER_ROWS) + } else if deep_band_rings && rows / 2 >= STREAMED_DEEP_MIN_ROWS { rows / 2 } else { rows @@ -58,8 +71,8 @@ pub(crate) fn streamed_batch_rows(width: usize, height: usize, deep_band_rings: /// level it holds nothing, since it copies the LL window straight into the /// plane. When the tile streams its rows to a sink, every component's rings /// are live at once, with a scratch row each as wide as its final window, or -/// two batches of such rows from pool bands (see [`streamed_batch_rows`]), so -/// the peak is their sum. +/// one or two batches of such rows from pool bands (see +/// [`streamed_batch_rows`]), so the peak is their sum. pub(crate) fn roi_idwt_peak_elements( storage: &DecompositionStorage<'_>, skipped_resolution_levels: u8, @@ -103,13 +116,20 @@ pub(crate) fn roi_idwt_peak_elements( if storage.parallel_bands { // Two batches, when each component reconstructs // on its own reader, unless the band rings hold - // more bands instead. - let batches = if storage.deep_band_rings { 1 } else { 2 }; + // more bands instead; one when the tile has one + // reader. + let one_reader = storage.one_synthesis_reader; + let batches = if storage.deep_band_rings || one_reader { + 1 + } else { + 2 + }; width .saturating_mul(streamed_batch_rows( width, height, storage.deep_band_rings, + one_reader, )) .saturating_mul(batches) } else { diff --git a/crates/j2k-native/src/j2c/idwt/tests.rs b/crates/j2k-native/src/j2c/idwt/tests.rs index c09e623d..86e7d8fe 100644 --- a/crates/j2k-native/src/j2c/idwt/tests.rs +++ b/crates/j2k-native/src/j2c/idwt/tests.rs @@ -917,19 +917,49 @@ fn deep_band_rings_halve_the_streamed_row_batch() { // 5661 x 8287 (one tile): 370 rows of each component in each of two // pipelined batches, 185 rows in the one batch of deep rings. - assert_eq!(streamed_batch_rows(5661, 8287, false), 370); - assert_eq!(streamed_batch_rows(5661, 8287, true), 185); + assert_eq!(streamed_batch_rows(5661, 8287, false, false), 370); + assert_eq!(streamed_batch_rows(5661, 8287, true, false), 185); // 4096 wide: 512 rows, or 256 with deep rings. - assert_eq!(streamed_batch_rows(4096, 4096, false), 512); - assert_eq!(streamed_batch_rows(4096, 4096, true), 256); + assert_eq!(streamed_batch_rows(4096, 4096, false, false), 512); + assert_eq!(streamed_batch_rows(4096, 4096, true, false), 256); for deep_band_rings in [false, true] { // 18000 wide: 116 rows, too few to halve. - assert_eq!(streamed_batch_rows(18000, 12000, deep_band_rings), 116); + assert_eq!( + streamed_batch_rows(18000, 12000, deep_band_rings, false), + 116 + ); // 8200 wide: 255 rows, too few to halve. - assert_eq!(streamed_batch_rows(8200, 12000, deep_band_rings), 255); - assert_eq!(streamed_batch_rows(256, 200, deep_band_rings), 50); - assert_eq!(streamed_batch_rows(256, 3, deep_band_rings), 1); - assert_eq!(streamed_batch_rows(3 << 20, 4096, deep_band_rings), 1); - assert_eq!(streamed_batch_rows(1024, 1 << 16, deep_band_rings), 512); + assert_eq!( + streamed_batch_rows(8200, 12000, deep_band_rings, false), + 255 + ); + assert_eq!(streamed_batch_rows(256, 200, deep_band_rings, false), 50); + assert_eq!(streamed_batch_rows(256, 3, deep_band_rings, false), 1); + assert_eq!( + streamed_batch_rows(3 << 20, 4096, deep_band_rings, false), + 1 + ); + assert_eq!( + streamed_batch_rows(1024, 1 << 16, deep_band_rings, false), + 512 + ); } } + +/// A tile reconstructing through one reader finishes each batch before the +/// next, so its batch holds at most 32 rows of each component, within the +/// limits every streamed batch keeps. +#[test] +fn one_reader_holds_a_short_streamed_row_batch() { + use super::roi::streamed_batch_rows; + + // 5661 x 8287 at six threads: 32 rows instead of 370. + assert_eq!(streamed_batch_rows(5661, 8287, false, true), 32); + assert_eq!(streamed_batch_rows(18000, 12000, false, true), 32); + // 100000 wide: 20 rows fit the batch's samples. + assert_eq!(streamed_batch_rows(100_000, 12000, false, true), 20); + // 100 rows: at least four batches. + assert_eq!(streamed_batch_rows(256, 100, false, true), 25); + assert_eq!(streamed_batch_rows(256, 3, false, true), 1); + assert_eq!(streamed_batch_rows(3 << 20, 4096, false, true), 1); +} From 7006a048732c135058283a611aa50f1e5d58220f Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 16:51:14 -0400 Subject: [PATCH 46/94] perf(j2k-native): hold deep band rings only for whole-row full-resolution reads Classic tiles with a reader per component (12 pool threads or more) held one band more in every ring, for the Tier-1 lookahead of a full-image decode. Overviews and regions narrower than the tile hold as many bands on fewer or narrower rows and gained nothing from the extra band. Such a tile now holds deep rings only when no resolution level is skipped and every sub-band window it reads spans its sub-band; the others keep the shallower rings. Every classic tile with a reader per component keeps one batch of half the rows, with deep rings or not; HTJ2K tiles keep their two pipelined batches. Measured with decodes to RGB8 (interleaved rounds, output byte-identical) on a 10-core ARM64 machine at 12 and 18 pool threads against the parent: a 5661 x 8287 RGB image at 1/2, 1/4 and 1/8 scale peaks 5.5-5.7, 2.3-2.9 and 1.0-1.3 MiB lower, a 1024 x 1024 region of an 18000 x 12000 image 2.2-2.4 MiB lower, at 0.98-1.02x time; full-resolution full-image decodes are unchanged. --- crates/j2k-native/src/j2c/decode.rs | 6 ++- .../src/j2c/decode/parallel_bands.rs | 51 ++++++++++++------- crates/j2k-native/src/j2c/decode/state.rs | 5 +- crates/j2k-native/src/j2c/idwt/roi.rs | 17 ++++--- 4 files changed, 49 insertions(+), 30 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode.rs b/crates/j2k-native/src/j2c/decode.rs index dd32a701..33898b6c 100644 --- a/crates/j2k-native/src/j2c/decode.rs +++ b/crates/j2k-native/src/j2c/decode.rs @@ -916,8 +916,10 @@ fn prepare_region_storage( (storage.banded_tier1 || storage.parallel_bands) && tile_ctx.stream_rows; #[cfg(feature = "parallel")] { - storage.deep_band_rings = - storage.parallel_bands && storage.lockstep_lines && parallel_bands::deep_rings(tile); + storage.deep_band_rings = storage.parallel_bands + && storage.lockstep_lines + && parallel_bands::deep_rings(tile) + && parallel_bands::reads_whole_rows(header, storage); storage.one_synthesis_reader = storage.parallel_bands && storage.lockstep_lines && parallel_bands::one_reader(tile); } diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index 295441f3..6d782c4f 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -391,25 +391,21 @@ pub(super) fn decode_tile_rows<'a>( }; // Rows reconstruct in batches, charged by the region's width (see // `roi_idwt_peak_elements`): two of them with a reader per component of - // an HTJ2K tile, so one finishes while the next reconstructs. A classic - // tile holds one batch, of half the rows when its band rings are deep. - // One reader holds one short batch, which it finishes before it - // reconstructs the next. + // an HTJ2K tile, so one finishes while the next reconstructs; a classic + // tile's readers wait on Tier-1 instead, and hold one batch of half the + // rows, with deep band rings when they read whole rows. One reader holds + // one short batch, which it finishes before it reconstructs the next. + let pipelined = readers > 1 && coding == CodeBlockCoding::HighThroughput; let batch = streamed_batch_rows( region.width() as usize, copy.height() as usize, - storage.deep_band_rings, + readers > 1 && !pipelined, readers == 1, ); - let batches = if readers > 1 && coding == CodeBlockCoding::HighThroughput { - 2 - } else { - 1 - }; let scratch_len = (copy.width() as usize) .checked_mul(components) .and_then(|samples| samples.checked_mul(batch)) - .and_then(|samples| samples.checked_mul(batches)) + .and_then(|samples| samples.checked_mul(if pipelined { 2 } else { 1 })) .ok_or(ValidationError::ImageTooLarge)?; let (mut buffer, ring_lens) = banded_tier1::take_line_buffer(tile_ctx, storage, &line_plans, scratch_len)?; @@ -537,14 +533,15 @@ pub(super) fn decode_tile_rows<'a>( } } -/// Whether a tile streaming its rows from pool bands holds deeper band rings -/// (see `DecompositionStorage::deep_band_rings`): a classic tile of several -/// components on a pool of [`MIN_CLASSIC_SYNTHESIS_THREADS`] threads or -/// more, which reconstructs each on its own reader (see +/// Whether a tile streaming its rows from pool bands may hold deeper band +/// rings (see `DecompositionStorage::deep_band_rings`): a classic tile of +/// several components on a pool of [`MIN_CLASSIC_SYNTHESIS_THREADS`] threads +/// or more, which reconstructs each on its own reader (see /// [`parallel_synthesis`]). Its readers wait on classic Tier-1, which more -/// bands in flight keep busier; the memory comes from holding a batch of -/// half the rows. HTJ2K bands keep up with the shallower rings, and its -/// readers gain more from the second batch. +/// bands in flight keep busier when it reads whole rows (see +/// [`reads_whole_rows`]); it holds one batch of rows instead of two. HTJ2K +/// bands keep up with the shallower rings, and its readers gain more from +/// the second batch. pub(super) fn deep_rings(tile: &Tile<'_>) -> bool { tile.component_infos.len() > 1 && rayon::current_num_threads() >= MIN_CLASSIC_SYNTHESIS_THREADS @@ -557,6 +554,24 @@ pub(super) fn deep_rings(tile: &Tile<'_>) -> bool { }) } +/// Whether a tile reads whole rows at full resolution: no resolution level +/// skipped, and every sub-band window the IDWT reads spans its sub-band. Only +/// such tiles hold deep band rings: a reduced or narrower read decodes as +/// fast without the extra band in every ring. +pub(super) fn reads_whole_rows(header: &Header<'_>, storage: &DecompositionStorage<'_>) -> bool { + header.skipped_resolution_levels == 0 + && storage.roi_plan.as_ref().is_some_and(|plan| { + storage + .sub_bands + .iter() + .enumerate() + .all(|(index, sub_band)| { + plan.sub_band_window(index) + .is_none_or(|window| window.width() == sub_band.rect.width()) + }) + }) +} + /// Whether a tile streaming its rows from pool bands reconstructs them /// through one reader whatever workers its routes plan (see /// [`parallel_synthesis`]): it has one component, or classic code-blocks on diff --git a/crates/j2k-native/src/j2c/decode/state.rs b/crates/j2k-native/src/j2c/decode/state.rs index 9b7e6269..78a8f7a3 100644 --- a/crates/j2k-native/src/j2c/decode/state.rs +++ b/crates/j2k-native/src/j2c/decode/state.rs @@ -122,8 +122,9 @@ pub(crate) struct DecompositionStorage<'a> { /// `coefficients` then hold a ring of bands. pub(crate) parallel_bands: bool, /// Whether the band rings of `parallel_bands` hold one band more at - /// every level, for a classic tile that streams its rows from a reader - /// per component (see [`parallel_bands::deep_rings`]). + /// every level, for a classic tile that streams whole rows at full + /// resolution from a reader per component (see + /// [`parallel_bands::deep_rings`] and [`parallel_bands::reads_whole_rows`]). pub(crate) deep_band_rings: bool, /// Whether a tile streaming its rows from `parallel_bands` reconstructs /// them through one reader whatever its pool's workers (see diff --git a/crates/j2k-native/src/j2c/idwt/roi.rs b/crates/j2k-native/src/j2c/idwt/roi.rs index 3d185a08..2a70535f 100644 --- a/crates/j2k-native/src/j2c/idwt/roi.rs +++ b/crates/j2k-native/src/j2c/idwt/roi.rs @@ -26,7 +26,7 @@ const STREAMED_BATCH_SAMPLES: usize = 2 * 1024 * 1024; /// samples. const STREAMED_MIN_BATCHES: usize = 4; -/// Rows a batch of a tile with deep band rings keeps at least when it is +/// Rows a batch of a tile holding one batch keeps at least when it is /// halved: halving the 116-row batches of an 18000-wide tile cost 2 to 5%. const STREAMED_DEEP_MIN_ROWS: usize = 128; @@ -36,21 +36,22 @@ const STREAMED_DEEP_MIN_ROWS: usize = 128; const STREAMED_ONE_READER_ROWS: usize = 32; /// Rows of each component a tile streaming from pool bands reconstructs at -/// once, for `height` rows of `width` samples. A tile whose band rings are -/// deep holds one batch, which is transformed before the next one -/// reconstructs, so its batches take half the rows while that leaves at -/// least `STREAMED_DEEP_MIN_ROWS`: as fast, with half the rows held. A tile -/// with `one_reader` holds one batch of at most `STREAMED_ONE_READER_ROWS`. +/// once, for `height` rows of `width` samples. A tile with a reader per +/// component that holds `one_batch` (a classic tile; HTJ2K tiles pipeline +/// two) transforms it before the next one reconstructs, so its batches take +/// half the rows while that leaves at least `STREAMED_DEEP_MIN_ROWS`: as +/// fast, with half the rows held. A tile with `one_reader` holds one batch +/// of at most `STREAMED_ONE_READER_ROWS`. pub(crate) fn streamed_batch_rows( width: usize, height: usize, - deep_band_rings: bool, + one_batch: bool, one_reader: bool, ) -> usize { let rows = STREAMED_BATCH_SAMPLES / width.max(1); let rows = if one_reader { rows.min(STREAMED_ONE_READER_ROWS) - } else if deep_band_rings && rows / 2 >= STREAMED_DEEP_MIN_ROWS { + } else if one_batch && rows / 2 >= STREAMED_DEEP_MIN_ROWS { rows / 2 } else { rows From 8f5f57897c3c153d900b34c40582417bfcf1081e Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 20:00:13 -0400 Subject: [PATCH 47/94] fix(j2k-native): charge the row batches of large classic tiles on 10 and 11 threads A tile streaming its rows from pool bands is charged one short batch of rows when it can only reconstruct them through one reader. That test treated every classic tile on fewer than 12 pool threads as one reader, while classic tiles of 4096 x 4096 samples per component or more take a reader per component from 10 threads, and those readers hold a batch of up to half the rows each. On 10 and 11 threads such a tile held more line-buffer samples than the ROI plan charged. The reader choice and the one-reader charge now ask one function with the same thread counts and size: the charge sizes by the largest final window of a component, which holds the rows the reader choice counts, so a tile charged for one reader never runs several. Those tiles are charged the two batches they were charged before the short batches; other tiles and every other pool size are charged as before. Output is unchanged. A unit test streams a classic RGB image on 9, 10, 11 and 12 pool threads, at full size and at half size on either side of a lowered test-only size gate: every case decodes on pool bands, matches the serial planes, takes the expected readers and holds no more than its charge. Without this change the 10-thread full-size case held 1971 samples beyond its charge. --- crates/j2k-native/src/j2c/decode.rs | 13 +++- .../src/j2c/decode/parallel_bands.rs | 73 +++++++++++++++---- crates/j2k-native/src/tests/row_sink.rs | 70 ++++++++++++++++++ 3 files changed, 140 insertions(+), 16 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode.rs b/crates/j2k-native/src/j2c/decode.rs index 33898b6c..f5c8fa5a 100644 --- a/crates/j2k-native/src/j2c/decode.rs +++ b/crates/j2k-native/src/j2c/decode.rs @@ -723,6 +723,14 @@ pub(crate) struct DecodeDebugCounters { pub(crate) ht_task_workspace_growths: usize, #[cfg(all(test, feature = "parallel"))] pub(crate) classic_task_workspace_growths: usize, + /// The most readers a tile streaming its rows from pool bands + /// reconstructed them on. + #[cfg(all(test, feature = "parallel"))] + pub(crate) pool_band_readers: usize, + /// The most line-buffer samples a tile streaming its rows from pool + /// bands held beyond the IDWT samples its ROI plan charged. + #[cfg(all(test, feature = "parallel"))] + pub(crate) uncharged_line_samples: usize, #[cfg(all(test, feature = "parallel"))] pub(crate) parallel_coefficients: ParallelCoefficientStats, } @@ -920,8 +928,9 @@ fn prepare_region_storage( && storage.lockstep_lines && parallel_bands::deep_rings(tile) && parallel_bands::reads_whole_rows(header, storage); - storage.one_synthesis_reader = - storage.parallel_bands && storage.lockstep_lines && parallel_bands::one_reader(tile); + storage.one_synthesis_reader = storage.parallel_bands + && storage.lockstep_lines + && parallel_bands::one_reader(tile, storage); } build::allocate_deferred_coefficients( storage, diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index 6d782c4f..08b64241 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -409,6 +409,17 @@ pub(super) fn decode_tile_rows<'a>( .ok_or(ValidationError::ImageTooLarge)?; let (mut buffer, ring_lens) = banded_tier1::take_line_buffer(tile_ctx, storage, &line_plans, scratch_len)?; + #[cfg(test)] + { + let held = ring_lens + .iter() + .fold(scratch_len, |held, len| held.saturating_add(*len)); + let counters = &mut tile_ctx.debug_counters; + counters.pool_band_readers = counters.pool_band_readers.max(readers); + counters.uncharged_line_samples = counters + .uncharged_line_samples + .max(held.saturating_sub(storage.roi_idwt_charge_elements)); + } let mut output = RowOutput::new(tile, sink, tile_ctx, copy, region); let workers = route_workers.min(bank.saturating_sub(readers)); // One reader for every component of a classic tile leaves the pool's @@ -575,17 +586,31 @@ pub(super) fn reads_whole_rows(header: &Header<'_>, storage: &DecompositionStora /// Whether a tile streaming its rows from pool bands reconstructs them /// through one reader whatever workers its routes plan (see /// [`parallel_synthesis`]): it has one component, or classic code-blocks on -/// fewer pool threads than their components need to reconstruct on the pool. -pub(super) fn one_reader(tile: &Tile<'_>) -> bool { - tile.component_infos.len() < 2 - || (rayon::current_num_threads() < MIN_CLASSIC_SYNTHESIS_THREADS - && tile.component_infos.iter().all(|component| { - !component - .coding_style - .parameters - .code_block_style - .uses_high_throughput_block_coding() - })) +/// too few pool threads for its size to reconstruct each component on its +/// own (see [`reader_per_component`]). The size is the largest final window +/// of a component, which holds the rows [`parallel_synthesis`] counts. +pub(super) fn one_reader(tile: &Tile<'_>, storage: &DecompositionStorage<'_>) -> bool { + let components = tile.component_infos.len(); + let classic = tile.component_infos.iter().all(|component| { + !component + .coding_style + .parameters + .code_block_style + .uses_high_throughput_block_coding() + }); + let coding = if classic { + CodeBlockCoding::Classic + } else { + CodeBlockCoding::HighThroughput + }; + let samples = storage.roi_plan.as_ref().map_or(usize::MAX, |plan| { + (0..components) + .filter_map(|component_idx| plan.final_window(component_idx)) + .map(|window| (window.width() as usize).saturating_mul(window.height() as usize)) + .max() + .unwrap_or(0) + }); + !reader_per_component(components, coding, rayon::current_num_threads(), samples) } /// Pool threads from which a classic tile reconstructs each component on its @@ -609,7 +634,12 @@ const MIN_CLASSIC_READER_THREADS: usize = 10; /// threads: on 10 threads a 4096 x 2048 lossless RGB image and the 2830 x /// 4143 to 708 x 1036 reduced decodes of a 5661 x 8287 one were 1-11% slower /// with them. +#[cfg(not(test))] const MIN_CLASSIC_READER_SAMPLES: usize = 1 << 24; +/// The unit tests' fixtures are far below the size, so their gate lies +/// between a 203 x 157 component and its 102 x 79 half. +#[cfg(test)] +const MIN_CLASSIC_READER_SAMPLES: usize = 1 << 14; /// Whether a streamed tile's `components` reconstruct on the pool, one per /// task, given its block coding, a workspace bank of `bank`, routes planned @@ -621,10 +651,25 @@ fn parallel_synthesis( route_workers: usize, samples: usize, ) -> bool { - let threads = rayon::current_num_threads(); - components > 1 - && route_workers >= 2 + route_workers >= 2 && bank > components + && reader_per_component(components, coding, rayon::current_num_threads(), samples) +} + +/// Whether a streamed tile of `components` of `coding` blocks may reconstruct +/// each component on its own reader on a pool of `threads`, with `samples` +/// reconstructed per component: HTJ2K tiles on any pool, classic ones from +/// [`MIN_CLASSIC_SYNTHESIS_THREADS`], or from [`MIN_CLASSIC_READER_THREADS`] +/// at [`MIN_CLASSIC_READER_SAMPLES`] or more. [`parallel_synthesis`] decides +/// by it and [`one_reader`] charges by it, so a tile charged for one reader +/// never runs several. +fn reader_per_component( + components: usize, + coding: CodeBlockCoding, + threads: usize, + samples: usize, +) -> bool { + components > 1 && (coding == CodeBlockCoding::HighThroughput || threads >= MIN_CLASSIC_SYNTHESIS_THREADS || (threads >= MIN_CLASSIC_READER_THREADS && samples >= MIN_CLASSIC_READER_SAMPLES)) diff --git a/crates/j2k-native/src/tests/row_sink.rs b/crates/j2k-native/src/tests/row_sink.rs index f9d03f64..087a95bc 100644 --- a/crates/j2k-native/src/tests/row_sink.rs +++ b/crates/j2k-native/src/tests/row_sink.rs @@ -825,6 +825,76 @@ fn streamed_rows_charge_every_components_rings() { } } +/// A classic tile of several components reconstructs each on a reader of +/// its own from 12 pool threads, and on 10 and 11 only from a size; the +/// readers then hold a batch of rows each where one reader would hold a +/// short one. The ROI plan must charge that batch on every such pool: the +/// line buffer each streamed tile takes stays within the IDWT charge, and +/// the rows match the serial planes. The unit tests' size gate lies between +/// this full image and its half, so both sides of it run on 10 and 11 +/// threads. +#[cfg(feature = "parallel")] +#[test] +fn classic_streamed_rows_charge_the_batches_of_their_readers() { + let pixels = noisy_pixels(3); + let bytes = encode( + &pixels, + WIDTH, + HEIGHT, + 3, + 8, + false, + &EncodeOptions { + num_decomposition_levels: 4, + // 16-row blocks give every component more rows of blocks than + // the pool threads, so its bands decode on the pool. + code_block_height_exp: 2, + ..EncodeOptions::default() + }, + ) + .expect("encode reader charge fixture"); + for threads in [9, 10, 11, 12] { + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("build pool"); + for reduction in [0_u8, 1] { + let image = Image::new_with_reduction(&bytes, &DecodeSettings::default(), reduction) + .expect("parse reader charge fixture"); + let roi = (0, 0, image.width(), image.height()); + let label = format!("threads={threads} reduction={reduction}"); + let planes = decode_planes(&image, roi); + let (rows, counters) = pool.install(|| { + let mut context = DecoderContext::default(); + context.set_cpu_decode_parallelism(CpuDecodeParallelism::Auto); + let mut sink = CollectingSink::new(roi.2, roi.3); + image + .decode_region_rows_for_integer_output_with_context( + roi, + &mut context, + &mut sink, + ) + .expect("decode region rows"); + assert!(context.tile_decode_context.stream_rows, "{label}"); + assert!(sink.arrivals.iter().all(|&count| count == 1), "{label}"); + (sink.planes, context.tile_decode_context.debug_counters) + }); + assert_eq!(rows, planes.planes, "{label}"); + assert_eq!(counters.pool_band_components, 3, "{label}: pool bands"); + let readers = if threads >= 12 || (threads >= 10 && reduction == 0) { + 3 + } else { + 1 + }; + assert_eq!(counters.pool_band_readers, readers, "{label}: readers"); + assert_eq!( + counters.uncharged_line_samples, 0, + "{label}: line buffer beyond the charge" + ); + } + } +} + /// A streamed decode that holds the image's packet directory and runs out of /// budget gives the directory up and reruns only while no row has reached the /// sink, since a rerun after that would send those rows again. This decode From edcc998ba1a96c809a010c2f44ab1437065e7534 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 16:19:46 -0400 Subject: [PATCH 48/94] perf(j2k-native): keep streamed classic bands on the pool when it has more threads than code-block rows A region decode whose rows stream to a sink has no per-sub-band path, so when `route_fallback` sent a classic component with fewer rows of code-blocks than pool threads to `PerSubBand`, the streamed tile decoded all of its bands serially, at one-thread speed. A lockstep decode now keeps such a component on the band route, whose bands split into parts for the idle threads. Measured on an 18-core ARM64 machine (forced streamed route vs the planes, same build, warm): 512 x 512 RGB at 2 bpp on 18 threads 4.44x -> 0.90x the planes' time; 768 x 512 classic at 1/2 resolution on 12 / 18 threads 4.2x / 4.9x -> 1.02x / 0.98x; 512 x 512 classic at 1/2 on 18 threads 3.7x -> 0.89x; 256 x 256 classic tiles 2.1x -> 1.14x. Bytes unchanged. A streamed tile plans every component's route in one budget through `plan_band_route_beside`, which takes the same `lockstep` flag; per-component routes and their unit tests pass `false` and plan as before. --- .../src/j2c/decode/parallel_bands.rs | 19 ++--- .../src/j2c/decode/subband/tile_parallel.rs | 48 +++++++----- crates/j2k-native/src/tests/row_sink.rs | 77 +++++++++++++++++++ 3 files changed, 116 insertions(+), 28 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index 08b64241..1593e1ab 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -118,7 +118,7 @@ pub(super) fn decode_component<'a>( profile_timings: &mut DecodeProfileTimings, ) -> Result { let stage_start = profile::profile_now(profile_enabled); - let routing = plan_band_route(tile, component_idx, tile_ctx, storage, header); + let routing = plan_band_route(tile, component_idx, tile_ctx, storage, header, false); let route = match routing { // Planning can fail in a later band before earlier blocks decode. Err(error) => { @@ -337,7 +337,7 @@ pub(super) fn decode_tile_rows<'a>( // left, as one budget. let mut live = None; for component_idx in 0..components { - match plan_band_route_beside(tile, component_idx, tile_ctx, storage, header, live) { + match plan_band_route_beside(tile, component_idx, tile_ctx, storage, header, live, true) { Ok(BandRouting::Route(route)) => { live = Some(route.budget()); routes.push(route); @@ -1920,7 +1920,7 @@ mod tests { assert_eq!(tile_ctx.debug_counters.pool_band_components, 1); let (route_bytes, executor) = { let BandRouting::Route(route) = - plan_band_route(&tiles[0], 0, tile_ctx, storage, &image.header) + plan_band_route(&tiles[0], 0, tile_ctx, storage, &image.header, false) .expect("plan control route") else { panic!("control fixture must take the pool route"); @@ -1948,12 +1948,13 @@ mod tests { .live_bytes(); storage.structural_workspace_bytes += DEFAULT_MAX_DECODE_BYTES - baseline - headroom; - let route = match plan_band_route(&tiles[0], 0, tile_ctx, storage, &image.header) { - Ok(BandRouting::Route(route)) => route, - Ok(BandRouting::Serial | BandRouting::WholeWindows) => return, - Err(error) if super::super::reuse::is_capacity_error(&error) => return, - Err(error) => panic!("unexpected planning error: {error:?}"), - }; + let route = + match plan_band_route(&tiles[0], 0, tile_ctx, storage, &image.header, false) { + Ok(BandRouting::Route(route)) => route, + Ok(BandRouting::Serial | BandRouting::WholeWindows) => return, + Err(error) if super::super::reuse::is_capacity_error(&error) => return, + Err(error) => panic!("unexpected planning error: {error:?}"), + }; let result = decode_route( &route, 0, diff --git a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs index 40374a35..5a9439b4 100644 --- a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs +++ b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs @@ -1595,24 +1595,35 @@ pub(in crate::j2c::decode) enum BandRouting<'a> { /// bytes and its rows of code-blocks. With a plan, the coding's retained bank /// holds a workspace for each of `workers` pool threads and one more for the /// calling thread, charged to the structural workspace. +/// +/// A `lockstep` decode, whose rows stream to a sink, has no per-sub-band +/// path: a classic component with fewer rows of code-blocks than threads +/// stays on the route, whose bands split into parts for the idle threads, +/// rather than decode its bands serially. A 512 x 512 RGB image at 6.6 bits +/// per pixel decoded 4.5 times as long on 18 threads as on 12 that way. pub(in crate::j2c::decode) fn plan_band_route<'a>( tile: &Tile<'a>, component_idx: usize, tile_ctx: &mut TileDecodeContext, storage: &mut DecompositionStorage<'a>, header: &Header<'_>, + lockstep: bool, ) -> Result> { - plan_band_route_beside(tile, component_idx, tile_ctx, storage, header, None) + plan_band_route_beside( + tile, + component_idx, + tile_ctx, + storage, + header, + None, + lockstep, + ) } /// [`plan_band_route`], charging the route to `live` when other routes stay /// live beside it: the budget of the route planned last for the same /// executor, which holds every earlier route's charges. Without it, the /// route is charged to the decode's storage budget. -#[expect( - clippy::too_many_lines, - reason = "the route's coding, tile route, plan and cutoffs decide one band routing in order" -)] pub(in crate::j2c::decode) fn plan_band_route_beside<'a>( tile: &Tile<'a>, component_idx: usize, @@ -1620,6 +1631,7 @@ pub(in crate::j2c::decode) fn plan_band_route_beside<'a>( storage: &mut DecompositionStorage<'a>, header: &Header<'_>, live: Option, + lockstep: bool, ) -> Result> { let components = component_idx..component_idx + 1; let Some(coding) = tile_route_coding(tile, components.clone()) else { @@ -1632,13 +1644,17 @@ pub(in crate::j2c::decode) fn plan_band_route_beside<'a>( }; let (required, segment_bytes, rows) = required_code_blocks(tile, components.clone(), storage, header, coding)?; - if let Some(route) = route_fallback( - coding, - rows, - segment_bytes, - threads, - MIN_PARALLEL_CODED_BYTES, - ) { + let fallback = |strips, coded_bytes| { + route_fallback( + coding, + strips, + coded_bytes, + threads, + MIN_PARALLEL_CODED_BYTES, + ) + .filter(|route| !(lockstep && matches!(route, TileRoute::PerSubBand))) + }; + if let Some(route) = fallback(rows, segment_bytes) { return Ok(declined(route)); } let mut budget = match live { @@ -1663,13 +1679,7 @@ pub(in crate::j2c::decode) fn plan_band_route_beside<'a>( } Err(error) => return Err(error), }; - if let Some(route) = route_fallback( - coding, - plan.strips.len(), - plan.coded_bytes, - threads, - MIN_PARALLEL_CODED_BYTES, - ) { + if let Some(route) = fallback(plan.strips.len(), plan.coded_bytes) { return Ok(declined(route)); } let work = plan diff --git a/crates/j2k-native/src/tests/row_sink.rs b/crates/j2k-native/src/tests/row_sink.rs index 087a95bc..1b3eb845 100644 --- a/crates/j2k-native/src/tests/row_sink.rs +++ b/crates/j2k-native/src/tests/row_sink.rs @@ -600,6 +600,34 @@ fn rgb8_destination_matches_the_sink_pack_byte_for_byte() { } } +/// A streamed decode has no per-sub-band path, so a classic component with +/// fewer rows of code-blocks than the pool has threads must still decode its +/// bands on the pool, in parts, rather than serially, with the planes' rows +/// bit for bit. +#[cfg(feature = "parallel")] +#[test] +fn pool_band_rows_use_a_pool_larger_than_the_code_block_rows() { + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(48) + .build() + .expect("build pool"); + for case in row_sink_cases().into_iter().filter(|case| { + case.components == 3 && !case.options.use_ht_block_coding && case.options.num_layers <= 1 + }) { + let pixels = noisy_pixels(case.components); + let bytes = encode(&pixels, WIDTH, HEIGHT, 3, 8, false, &case.options) + .expect("encode large pool fixture"); + let image = Image::new_with_reduction(&bytes, &DecodeSettings::default(), 0) + .expect("parse large pool fixture"); + let roi = (0, 0, image.width(), image.height()); + let planes = decode_planes(&image, roi); + let rows = pool.install(|| decode_rows(&image, roi, CpuDecodeParallelism::Auto)); + assert_eq!(rows.planes, planes.planes, "{}", case.label); + assert!(rows.streamed, "{}: streamed", case.label); + assert!(rows.pool_components > 0, "{}: decoded serially", case.label); + } +} + /// Outcome of a region decode, through the planes or the row sink: a hash /// of the samples, or the error. fn outcome( @@ -971,3 +999,52 @@ fn streamed_rows_evict_the_packet_directory_before_the_first_row() { metadata ); } + +/// Stitched windows of a reduced resolution, as a tiled viewer or an IIIF +/// server requests them, hold few rows of code-blocks per component: fewer +/// than a pool of 18 threads has workers. Such a window's streamed rows must +/// still decode its classic bands on the pool, in parts, and not fall back to +/// one serial thread for the whole tile, with the planes' rows bit for bit. +#[cfg(feature = "parallel")] +#[test] +fn reduced_windows_stream_on_a_pool_with_more_threads_than_their_rows() { + for threads in [18, 48] { + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("build pool"); + for case in row_sink_cases().into_iter().filter(|case| { + case.components == 3 + && !case.options.use_ht_block_coding + && case.options.num_layers <= 1 + && case.options.tile_size.is_none() + }) { + let pixels = noisy_pixels(case.components); + let bytes = encode(&pixels, WIDTH, HEIGHT, 3, 8, false, &case.options) + .expect("encode reduced window fixture"); + for reduction in [1_u8, 2] { + let image = + Image::new_with_reduction(&bytes, &DecodeSettings::default(), reduction) + .expect("parse reduced window fixture"); + let (width, height) = (image.width(), image.height()); + let (half_width, half_height) = (width.div_ceil(2), height.div_ceil(2)); + for roi in [ + (0, 0, half_width, half_height), + (half_width, 0, width - half_width, half_height), + (0, height - 3, half_width, 3), + ] { + let label = format!( + "{} threads={threads} reduction={reduction} roi={roi:?}", + case.label + ); + let planes = decode_planes(&image, roi); + let rows = + pool.install(|| decode_rows(&image, roi, CpuDecodeParallelism::Auto)); + assert_eq!(rows.planes, planes.planes, "{label}"); + assert!(rows.streamed, "{label}: streamed"); + assert_eq!(rows.pool_components, 3, "{label}: decoded serially"); + } + } + } + } +} From 9b617292f98e0760078e1d791fbba20e4c9e339c Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 16:19:55 -0400 Subject: [PATCH 49/94] perf(j2k-native): pack streamed RGB8 rows through the fused ICT pack A streamed decode of three 8-bit components copied each reconstructed row out of the line IDWT's ring, transformed it in place and handed it to the sink's generic pack. Where the sink has an RGB8 destination, rows now go straight from the IDWT's rows into it through the planes route's fused ICT pack (9/7 with the MCT; same expressions, same bytes), with no copy and no in-place transform; on the pool each batch row is packed the same way, and other transforms pack into the destination after the in-place transform instead of going through `write_rows`. Measured on an 18-core ARM64 machine (forced streamed route, warm, vs the planes of the same build): 4096 x 2048 HTJ2K at 2 bpp on 1 thread 1.15x -> 1.08x, on 6 / 18 threads 1.12x / 1.40x -> 1.11x / 1.36x; 4096 x 2048 classic at 2 bpp on 1 / 6 / 18 threads 1.02x / 0.99x / 0.96x -> 1.01x / 0.96x / 0.95x. Bytes unchanged. --- .../j2k-native/src/j2c/decode/banded_tier1.rs | 157 +++++++++++++----- crates/j2k-native/src/j2c/mct.rs | 90 ++++++---- crates/j2k-native/src/tests/row_sink.rs | 63 ++++++- 3 files changed, 232 insertions(+), 78 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode/banded_tier1.rs b/crates/j2k-native/src/j2c/decode/banded_tier1.rs index f5158e21..edf025ec 100644 --- a/crates/j2k-native/src/j2c/decode/banded_tier1.rs +++ b/crates/j2k-native/src/j2c/decode/banded_tier1.rs @@ -27,6 +27,7 @@ use crate::error::{DecodeError, DecodingError, ValidationError}; use crate::j2c::build; use crate::j2c::codestream::WaveletTransform; use crate::j2c::idwt::{ComponentLines, LinePlan, LineReconstruction, SubBandRows}; +use crate::j2c::mct::DeferredMct; use crate::j2c::rect::IntRect; use crate::profile; use crate::{try_resize_decode_elements, ComponentPlane, ComponentRowSink}; @@ -312,6 +313,9 @@ pub(super) struct RowOutput<'o, 't> { /// The rows, in the coordinates of `region`, that the tile reconstructs. copy: IntRect, region: IntRect, + /// The inverse MCT of three 8-bit components, which the RGB8 pack can + /// take over (see [`Self::fused_rgb8_pack`]). + mct: Option, /// Samples handed to the sink. pub(super) streamed: usize, } @@ -326,26 +330,67 @@ impl<'o, 't> RowOutput<'o, 't> { copy: IntRect, region: IntRect, ) -> Self { + let depths: [(u8, bool); 4] = core::array::from_fn(|index| { + tile_ctx + .channel_data + .get(index) + .map_or((0, false), |component| { + (component.bit_depth, component.signed) + }) + }); + let rgb8 = + tile.component_infos.len() == 3 && depths[..3].iter().all(|(depth, _)| *depth == 8); Self { tile, sink, round_irreversible_output: tile_ctx.round_irreversible_output, - depths: core::array::from_fn(|index| { - tile_ctx - .channel_data - .get(index) - .map_or((0, false), |component| { - (component.bit_depth, component.signed) - }) - }), + depths, copy, region, + mct: (rgb8 && tile.mct).then(|| { + DeferredMct::for_rows(&tile.component_infos, tile_ctx.round_irreversible_output) + }), streamed: 0, } } + /// The inverse MCT when the RGB8 pack fuses it (see + /// [`DeferredMct::fuses_rgb8_pack`]): rows then go to the pack as the + /// IDWT left them, never transformed in place. + fn fused_rgb8_pack(&self) -> Option { + self.mct.filter(|mct| mct.fuses_rgb8_pack()) + } + + /// The sink's RGB8 rows for `rows` rows of the tile from row `y`, when + /// it has a destination for three 8-bit components. + fn rgb8_rows(&mut self, y: u32, rows: usize) -> Result>> { + let (copy, region) = (self.copy, self.region); + let rgb8 = self.depths.len() >= 3 + && self.tile.component_infos.len() == 3 + && self.depths[..3].iter().all(|(depth, _)| *depth == 8); + if !rgb8 { + return Ok(None); + } + let Some(out) = self.sink.rgb8_region(region.width(), region.height()) else { + return Ok(None); + }; + let stride = region.width() as usize * 3; + let start = (y - region.y0) as usize * stride; + let out = out + .get_mut(start..start + rows * stride) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + Ok(Some(Rgb8Rows { + out, + stride, + x: (copy.x0 - region.x0) as usize * 3, + })) + } + /// Advance every component's reconstruction a row at a time, copying - /// each row into `scratch`, one row per component, and hand them over. + /// each row into `scratch`, one row per component, and hand them over: + /// packed into the sink's RGB8 rows when it has them, through + /// [`DeferredMct::pack_ict_rgb8_rows`] straight from the IDWT's rows + /// where that pack transforms them, else through `write_rows`. pub(super) fn write( &mut self, lines: &mut [LineReconstruction<'_>], @@ -357,7 +402,34 @@ impl<'o, 't> RowOutput<'o, 't> { if lines.is_empty() || width == 0 { return Ok(()); } + // With an RGB8 destination, rows go from the IDWT's rings straight + // to the pack, which transforms them on the way. + let fused = self + .fused_rgb8_pack() + .filter(|_| matches!(self.rgb8_rows(copy.y0, 0), Ok(Some(_)))); for y in copy.y0..copy.y1 { + if let Some(mct) = fused { + let mut samples: [&[f32]; 3] = [&[]; 3]; + for (line, row) in lines.iter_mut().zip(samples.iter_mut()) { + let (row_y, line_row) = line + .next_row(rows)? + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + if row_y != y || line_row.len() != width { + return Err(DecodingError::CodeBlockDecodeFailure.into()); + } + *row = line_row; + } + let rgb8 = self + .rgb8_rows(y, 1)? + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let pixels = rgb8 + .out + .get_mut(rgb8.x..rgb8.x + width * 3) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + mct.pack_ict_rgb8_rows(samples, pixels); + self.streamed = self.streamed.saturating_add(width * components); + continue; + } let mut outputs: [&mut [f32]; 4] = Default::default(); for ((line, output), chunk) in lines .iter_mut() @@ -375,6 +447,16 @@ impl<'o, 't> RowOutput<'o, 't> { } let outputs = &mut outputs[..components]; store::transform_streamed_rows(self.tile, self.round_irreversible_output, outputs)?; + if let Some(rgb8) = self.rgb8_rows(y, 1)? { + let pixels = rgb8 + .out + .get_mut(rgb8.x..rgb8.x + width * 3) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let [red, green, blue] = [&*outputs[0], &*outputs[1], &*outputs[2]]; + crate::color::pack_rgb8([red, green, blue], pixels); + self.streamed = self.streamed.saturating_add(width * components); + continue; + } let component_rows: [ComponentPlane<'_>; 4] = core::array::from_fn(|index| ComponentPlane { samples: outputs.get(index).map_or(&[][..], |row| &**row), @@ -502,8 +584,10 @@ impl RowOutput<'_, '_> { ) -> Result<()> { let width = self.copy.width() as usize; let mut parts = batch_parts(buffer, batch_len, rows * width, components)?; + let fused = self.fused_rgb8_pack(); let destination = self.rgb8_rows(y, rows)?; let packed = destination.is_some(); + let fused = fused.filter(|_| packed); match alongside { Some(alongside) => { let mut finished = Ok(()); @@ -511,7 +595,7 @@ impl RowOutput<'_, '_> { overlap( || { finished = finish_rows( - transform, + (transform, fused), &mut parts[..components], width, destination, @@ -524,7 +608,7 @@ impl RowOutput<'_, '_> { finished?; } None => finish_rows( - transform, + (transform, fused), &mut parts[..components], width, destination, @@ -538,31 +622,6 @@ impl RowOutput<'_, '_> { Ok(()) } - /// The sink's RGB8 rows for `rows` rows of the tile from row `y`, when - /// it has a destination for three 8-bit components. - fn rgb8_rows(&mut self, y: u32, rows: usize) -> Result>> { - let (copy, region) = (self.copy, self.region); - let rgb8 = self.depths.len() >= 3 - && self.tile.component_infos.len() == 3 - && self.depths[..3].iter().all(|(depth, _)| *depth == 8); - if !rgb8 { - return Ok(None); - } - let Some(out) = self.sink.rgb8_region(region.width(), region.height()) else { - return Ok(None); - }; - let stride = region.width() as usize * 3; - let start = (y - region.y0) as usize * stride; - let out = out - .get_mut(start..start + rows * stride) - .ok_or(DecodingError::CodeBlockDecodeFailure)?; - Ok(Some(Rgb8Rows { - out, - stride, - x: (copy.x0 - region.x0) as usize * 3, - })) - } - /// Hand `rows` rows of each component in `parts`, from row `y`, to the /// sink. fn write_component_rows(&mut self, parts: &[&mut [f32]], y: u32, rows: usize) { @@ -732,10 +791,6 @@ fn reconstruct_rows_into( } /// Output rows that take three 8-bit components packed as RGB8. -#[cfg_attr( - not(feature = "parallel"), - expect(dead_code, reason = "only the parallel band reader writes batches") -)] struct Rgb8Rows<'o> { /// Whole output rows, `stride` bytes each, one per reconstructed row. out: &'o mut [u8], @@ -760,14 +815,30 @@ struct BatchRows<'p, 'o> { )] impl BatchRows<'_, '_> { /// Transform the rows one at a time, as [`RowOutput::write`] does, then - /// pack them into the RGB8 rows if there are any. + /// pack them into the RGB8 rows if there are any; with a `fused` MCT, + /// the pack transforms each row as it reads it instead. fn finish( mut self, - (tile, round_irreversible_output): (&Tile<'_>, bool), + ((tile, round_irreversible_output), fused): ((&Tile<'_>, bool), Option), components: usize, width: usize, ) -> Result<()> { let rows = self.parts[0].len().checked_div(width).unwrap_or(0); + if let (Some(mct), Some(rgb8)) = (fused, self.rgb8.as_mut()) { + let [red, green, blue, _] = &self.parts; + let samples = red + .chunks_exact(width) + .zip(green.chunks_exact(width)) + .zip(blue.chunks_exact(width)); + for (line, ((red, green), blue)) in rgb8.out.chunks_exact_mut(rgb8.stride).zip(samples) + { + let pixels = line + .get_mut(rgb8.x..rgb8.x + width * 3) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + mct.pack_ict_rgb8_rows([red, green, blue], pixels); + } + return Ok(()); + } for row in 0..rows { let mut outputs: [&mut [f32]; 4] = Default::default(); for (output, part) in outputs.iter_mut().zip(self.parts.iter_mut()) { @@ -807,7 +878,7 @@ impl BatchRows<'_, '_> { expect(dead_code, reason = "only the parallel band reader writes batches") )] fn finish_rows( - transform: (&Tile<'_>, bool), + transform: ((&Tile<'_>, bool), Option), parts: &mut [&mut [f32]], width: usize, mut rgb8: Option>, diff --git a/crates/j2k-native/src/j2c/mct.rs b/crates/j2k-native/src/j2c/mct.rs index 960e93f7..6c3f1d37 100644 --- a/crates/j2k-native/src/j2c/mct.rs +++ b/crates/j2k-native/src/j2c/mct.rs @@ -121,17 +121,7 @@ pub(crate) fn apply_inverse_rows( round_irreversible_output: bool, rows: (&mut [f32], &mut [f32], &mut [f32]), ) { - let transform = component_infos[0].wavelet_transform(); - let addends = [ - unsigned_level_shift(&component_infos[0]), - unsigned_level_shift(&component_infos[1]), - unsigned_level_shift(&component_infos[2]), - ]; - let centered_output = - round_irreversible_output && transform == WaveletTransform::Irreversible97; - let transform_addends = if centered_output { [0.0; 3] } else { addends }; - let rounding = centered_output.then_some(addends); - transform_planes(transform, rows, transform_addends, rounding, false); + DeferredMct::for_rows(component_infos, round_irreversible_output).transform_chunk(rows); } /// Samples per plane in one chunk: three planes' worth stays in cache @@ -166,6 +156,64 @@ const PACK_CHUNK: usize = 1 << 10; const PACK_TASK_CHUNKS: usize = 64; impl DeferredMct { + /// The transform [`apply_inverse_rows`] applies to rows of + /// `component_infos`, for a streamed decode to pack its rows with. + pub(crate) fn for_rows( + component_infos: &[ComponentInfo], + round_irreversible_output: bool, + ) -> Self { + let transform = component_infos[0].wavelet_transform(); + let addends = [ + unsigned_level_shift(&component_infos[0]), + unsigned_level_shift(&component_infos[1]), + unsigned_level_shift(&component_infos[2]), + ]; + let centered_output = + round_irreversible_output && transform == WaveletTransform::Irreversible97; + Self { + transform, + addends: if centered_output { [0.0; 3] } else { addends }, + rounding: centered_output.then_some(addends), + last_level: None, + } + } + + /// Whether [`Self::pack_ict_rgb8_rows`] packs rows: this build fuses the + /// transform into the RGB8 pack for 9/7 samples (see [`Self::pack_rgb8`]). + pub(crate) fn fuses_rgb8_pack(self) -> bool { + self.transform == WaveletTransform::Irreversible97 + && cfg!(all( + feature = "simd", + any( + target_arch = "x86_64", + all(feature = "std", target_arch = "aarch64") + ) + )) + } + + /// Transform `rows` and pack them as RGB8 into `output` in one pass, + /// reading the rows only, as the fused pack of [`Self::pack_rgb8`] + /// does; with the same expressions as [`Self::transform_chunk`] and the + /// standard RGB8 pack, so the bytes are theirs. Only where + /// [`Self::fuses_rgb8_pack`]; otherwise nothing is written. + pub(crate) fn pack_ict_rgb8_rows(self, rows: [&[f32]; 3], output: &mut [u8]) { + if !self.fuses_rgb8_pack() { + return; + } + #[cfg(all(feature = "simd", target_arch = "x86_64"))] + { + self.pack_ict_rgb8_x86(rows, output); + } + #[cfg(not(all(feature = "simd", target_arch = "x86_64")))] + { + if self.rounding.is_some() { + self.pack_ict_rgb8::(rows, output); + } else { + self.pack_ict_rgb8::(rows, output); + } + } + } + /// Transform the planes in place, as [`apply_inverse`] would have. Planes /// that hold a last level's sub-bands are left to [`Self::pack_rgb8`]. pub(crate) fn apply( @@ -215,22 +263,8 @@ impl DeferredMct { output, parallel, |[r, g, b], output| { - #[cfg(all(feature = "simd", target_arch = "x86_64"))] - if self.transform == WaveletTransform::Irreversible97 { - self.pack_ict_rgb8_x86([r, g, b], output); - return; - } - if cfg!(all( - feature = "std", - feature = "simd", - target_arch = "aarch64" - )) && self.transform == WaveletTransform::Irreversible97 - { - if self.rounding.is_some() { - self.pack_ict_rgb8::([r, g, b], output); - } else { - self.pack_ict_rgb8::([r, g, b], output); - } + if self.fuses_rgb8_pack() { + self.pack_ict_rgb8_rows([r, g, b], output); } else { self.transform_chunk((&mut *r, &mut *g, &mut *b)); pack([r, g, b], output); @@ -387,7 +421,7 @@ impl DeferredMct { ); } - fn transform_chunk(self, planes: (&mut [f32], &mut [f32], &mut [f32])) { + pub(crate) fn transform_chunk(self, planes: (&mut [f32], &mut [f32], &mut [f32])) { transform_planes(self.transform, planes, self.addends, self.rounding, false); } } diff --git a/crates/j2k-native/src/tests/row_sink.rs b/crates/j2k-native/src/tests/row_sink.rs index 1b3eb845..6f21d17d 100644 --- a/crates/j2k-native/src/tests/row_sink.rs +++ b/crates/j2k-native/src/tests/row_sink.rs @@ -485,13 +485,6 @@ fn pool_band_rows_fit_every_components_clipped_blocks() { /// A sink with an RGB8 destination, which [`ComponentRowSink::write_rows`] /// fills the way the destination's contract says: each 8-bit sample rounded /// and saturated. -#[cfg_attr( - not(feature = "parallel"), - expect( - dead_code, - reason = "only the parallel band reader offers an RGB8 destination" - ) -)] struct Rgb8Sink { width: usize, rgb: Vec, @@ -600,6 +593,62 @@ fn rgb8_destination_matches_the_sink_pack_byte_for_byte() { } } +/// The serial bands pack three 8-bit components into a sink's RGB8 +/// destination too: a 9/7 image with the MCT straight from the IDWT's rows +/// through the fused pack, others after transforming a copy of the rows. The +/// bytes must be those the sink packs from the rows. +#[test] +fn serial_rgb8_destination_matches_the_sink_pack_byte_for_byte() { + for case in row_sink_cases() + .into_iter() + .filter(|case| case.components == 3) + { + let pixels = noisy_pixels(case.components); + let bytes = encode(&pixels, WIDTH, HEIGHT, 3, 8, false, &case.options) + .expect("encode serial RGB8 destination fixture"); + let mut packed_into_destination = 0; + for reduction in [0_u8, 1] { + let image = Image::new_with_reduction(&bytes, &DecodeSettings::default(), reduction) + .expect("parse serial RGB8 destination fixture"); + let (width, height) = (image.width(), image.height()); + for roi in [ + (0, 0, width, height), + (width / 3, height / 3, width / 7, height / 9), + (0, height - 9, width, 9), + ] { + let decode = |offers_destination| { + let mut context = DecoderContext::default(); + context.set_cpu_decode_parallelism(CpuDecodeParallelism::Serial); + let mut sink = Rgb8Sink { + width: roi.2 as usize, + rgb: alloc::vec![0; (roi.2 * roi.3 * 3) as usize], + offers_destination, + rectangles: 0, + }; + image + .decode_region_rows_for_integer_output_with_context( + roi, + &mut context, + &mut sink, + ) + .expect("decode serial RGB8 destination rows"); + sink + }; + let label = format!("{} reduction={reduction} roi={roi:?}", case.label); + let expected = decode(false); + let packed = decode(true); + assert_eq!(packed.rgb, expected.rgb, "{label}"); + packed_into_destination += usize::from(packed.rectangles == 0); + } + } + assert!( + packed_into_destination > 0, + "{}: never packed into the destination", + case.label + ); + } +} + /// A streamed decode has no per-sub-band path, so a classic component with /// fewer rows of code-blocks than the pool has threads must still decode its /// bands on the pool, in parts, rather than serially, with the planes' rows From 84df53a71a9d98242eaadadbc7b8112dc19089d7 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 16:19:55 -0400 Subject: [PATCH 50/94] perf(j2k): stream large low-rate classic images on small and large pools Full-resolution classic images below 4 coded bits per pixel kept the planes route at every thread count. With the streamed rows packed through the fused RGB8 pack, a 4096 x 2048 RGB image at 2 bpp decodes at least as fast streamed on small and large pools, in about half the memory: on an 18-core ARM64 machine 1% faster on 6 threads (10% cold) and 4-5% faster on 18 (18-20% cold), footprint 137 -> 76 MB and 149 -> 73 MB; on a 10-core one 7% faster on 4 threads. Such images from 4096 x 2048 samples now stream on 2 to 6 pool threads and from 12. They keep the planes on one thread, where the planes' fused last level is 1% faster, and on 7 to 11 pool threads: there one pool reader reconstructs all three components and Tier-1 of so few coded bits outruns it (16-19% slower streamed on 10 threads). 512 x 512 images at 2 bpp keep the planes too (22% slower streamed on 6 threads). Also corrects the reduced-resolution threshold's note: with classic bands kept on the pool, a 384 x 256 output of a 768 x 512 image decodes 2-3% slower streamed on 12 threads and 2% faster on 18, no longer 4.9 times as slow. --- crates/j2k/src/decode/route.rs | 48 +++++++++++++++++++++------- crates/j2k/src/decode/route/tests.rs | 18 +++++++++-- 2 files changed, 52 insertions(+), 14 deletions(-) diff --git a/crates/j2k/src/decode/route.rs b/crates/j2k/src/decode/route.rs index e1b40a4c..0bcb3a40 100644 --- a/crates/j2k/src/decode/route.rs +++ b/crates/j2k/src/decode/route.rs @@ -135,10 +135,10 @@ const MIN_STREAMED_TILE_SAMPLES: u64 = 1 << 20; /// Output samples from which a reduced-resolution decode streams on any /// pool (a 1024 x 512 output and larger: 6-53% faster on 18 threads). On 12 -/// pool threads or more a smaller classic output decodes on the pool's -/// bands as slowly as on one thread (a 384 x 256 output of a 768 x 512 -/// image 4.9 times as long as on the planes), and smaller HTJ2K outputs are -/// up to 59% slower streamed on any pool. +/// pool threads or more a smaller classic output gains nothing from +/// streaming (a 384 x 256 output of a 768 x 512 image 2-3% slower on 12 +/// threads, 2% faster on 18), and smaller HTJ2K outputs are up to 59% slower +/// streamed on any pool. const MIN_REDUCED_STREAMED_SAMPLES: u64 = 1 << 19; /// Output samples from which a smaller reduced classic decode streams on 2 @@ -160,12 +160,28 @@ const MIN_STREAMED_GRAY_SAMPLES: u64 = 2896 * 2896; const MIN_STREAMED_GRAY_SAMPLES_ON_A_LARGE_POOL: u64 = 5792 * 5792; /// Coded bits per sample from which a full-resolution untiled classic image -/// streams: below it Tier-1 is light, the decode is bound by the inverse -/// transforms and the pack, and the planes' fused last level does those -/// faster (a 2 bits-per-pixel image is 0-3% slower streamed, 5-11 bits per -/// pixel images are 10% faster). +/// streams as [`full_resolution_streams`] says: below it Tier-1 is light, the +/// decode is bound by the inverse transforms and the pack, and the planes' +/// fused last level does those faster on one thread and on small images. const MIN_STREAMED_CODED_BITS_PER_SAMPLE: u64 = 4; +/// Samples from which a full-resolution classic image below +/// [`MIN_STREAMED_CODED_BITS_PER_SAMPLE`] streams on a pool of up to +/// [`MAX_LOW_RATE_SHARED_READER_THREADS`] threads or of +/// [`CLASSIC_SYNTHESIS_THREADS`] or more: with its rows packed through the +/// fused RGB8 pack, a 4096 x 2048 image at 2 bits per pixel decodes 4-7% +/// faster streamed on 4 to 6 threads and 5% faster on 18, in less than half +/// the memory, and 1% slower on one; 512 x 512 ones at 2 bits per pixel are +/// 22% slower on 6 threads. +const MIN_STREAMED_LOW_RATE_SAMPLES: u64 = 1 << 23; + +/// Pool threads up to which a large image below +/// [`MIN_STREAMED_CODED_BITS_PER_SAMPLE`] streams while one pool reader +/// reconstructs all of its components (below [`CLASSIC_SYNTHESIS_THREADS`]): +/// on more, Tier-1 of so few coded bits outruns that reader, and the +/// 4096 x 2048 image decodes 16-19% slower streamed on 10 threads. +const MAX_LOW_RATE_SHARED_READER_THREADS: usize = 6; + /// Whether an image the planes route can decode streams anyway, because /// streaming was measured at least as fast for its kind; for three /// components: @@ -176,8 +192,9 @@ const MIN_STREAMED_CODED_BITS_PER_SAMPLE: u64 = 4; /// more (one thread decodes a half resolution HTJ2K image 2-9% faster on /// the planes); a smaller classic one of /// [`MIN_SMALL_REDUCED_STREAMED_SAMPLES`] or more on 2 to 11 pool threads; -/// - a full-resolution classic image of at least 4 coded bits per pixel (see -/// [`full_resolution_streams`]); a tiled one whose tiles do not decode at +/// - a full-resolution classic image of at least 4 coded bits per pixel, or a +/// large one at fewer on most pools (see [`full_resolution_streams`]); a tiled +/// one whose tiles do not decode at /// once streams them one after another, so each of its tiles must meet /// that as an image of its own, of [`MIN_STREAMED_TILE_SAMPLES`] or more /// (4096 x 4096 RGB in 16 tiles of 1024 x 1024: 116.9 vs 124.9 ms on the @@ -251,13 +268,20 @@ fn reduced_streams(samples: u64, htj2k: bool, threads: usize) -> bool { /// from [`MIN_STREAMED_SAMPLES`] and [`MIN_STREAMED_CODED_BITS_PER_SAMPLE`], /// on 2 to 11 pool threads, on more when it is below /// [`MAX_STREAMED_SAMPLES_ON_A_LARGE_POOL`], and on one from -/// [`MIN_STREAMED_TILE_SAMPLES`]. +/// [`MIN_STREAMED_TILE_SAMPLES`]; at fewer coded bits, from +/// [`MIN_STREAMED_LOW_RATE_SAMPLES`] on 2 to +/// [`MAX_LOW_RATE_SHARED_READER_THREADS`] pool threads and from +/// [`CLASSIC_SYNTHESIS_THREADS`]. fn full_resolution_streams(samples: u64, coded_bytes: usize, threads: usize) -> bool { let coded_bits = u64::try_from(coded_bytes) .unwrap_or(u64::MAX) .saturating_mul(8); + if coded_bits < samples.saturating_mul(MIN_STREAMED_CODED_BITS_PER_SAMPLE) { + return samples >= MIN_STREAMED_LOW_RATE_SAMPLES + && (matches!(threads, 2..=MAX_LOW_RATE_SHARED_READER_THREADS) + || threads >= CLASSIC_SYNTHESIS_THREADS); + } samples >= MIN_STREAMED_SAMPLES - && coded_bits >= samples.saturating_mul(MIN_STREAMED_CODED_BITS_PER_SAMPLE) && match threads { 0 | 1 => samples >= MIN_STREAMED_TILE_SAMPLES, 2..CLASSIC_SYNTHESIS_THREADS => true, diff --git a/crates/j2k/src/decode/route/tests.rs b/crates/j2k/src/decode/route/tests.rs index 5f25abf8..6586b5e3 100644 --- a/crates/j2k/src/decode/route/tests.rs +++ b/crates/j2k/src/decode/route/tests.rs @@ -507,14 +507,28 @@ fn tiles_at_once_report_the_first_failing_tile() { /// Full-resolution classic images stream from 4 coded bits per pixel on 2 /// to 11 pool threads from 512 x 512 samples, on larger pools too below 2^19 -/// samples, and on one thread from a megapixel. +/// samples, and on one thread from a megapixel; at fewer bits per pixel from +/// 4096 x 2048 samples on pools of 2 to 6 threads and of 12 or more. #[test] fn full_resolution_images_stream_when_tier1_bound() { use super::full_resolution_streams; let mosaic = 4096 * 2048; // 11 MB lossless and 2.2 MB at 2 bits per pixel. assert!(full_resolution_streams(mosaic, 11_104_128, 6)); - assert!(!full_resolution_streams(mosaic, 2_246_803, 6)); + // At 2 bits per pixel on small and large pools, but not on one thread or + // on the 7 to 11 threads where one pool reader holds the decode back. + for threads in [2, 6, 12, 18] { + assert!( + full_resolution_streams(mosaic, 2_246_803, threads), + "{threads}" + ); + } + for threads in [7, 10, 11] { + assert!( + !full_resolution_streams(mosaic, 2_246_803, threads), + "{threads}" + ); + } for threads in [12, 18] { assert!( !full_resolution_streams(mosaic, 11_104_128, threads), From e62675f221c9ba7a981a1a2da13fbcfddb809959 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 17:40:36 -0400 Subject: [PATCH 51/94] perf(j2k-native): look streamed sub-bands up by index The row-streamed IDWT asks the banded Tier-1 for each sub-band's rows several times a row, and every call searched the tile's list of sub-band bands for the sub-band, up to 16 entries per component. A table from sub-band id to its entry, built with the list, replaces the search. Measured on a 12-core ARM64 machine (streamed route, one thread, warm): 256 x 256 and 512 x 512 tiles decode 4-9% faster, 4096 x 2048 images 0-1%. Bytes unchanged. --- .../j2k-native/src/j2c/decode/banded_tier1.rs | 37 +++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode/banded_tier1.rs b/crates/j2k-native/src/j2c/decode/banded_tier1.rs index edf025ec..bd2c8120 100644 --- a/crates/j2k-native/src/j2c/decode/banded_tier1.rs +++ b/crates/j2k-native/src/j2c/decode/banded_tier1.rs @@ -163,6 +163,7 @@ fn stream_rows<'a>( let components = tile.component_infos.len(); let mut bands = Vec::new(); let line_plans = plan_tile_lines(tile, header, tile_ctx, storage, &mut bands)?; + let band_of = band_lookup(&bands)?; let empty = IntRect::from_xywh(0, 0, 0, 0); let (copy, region) = line_plans .first() @@ -201,6 +202,7 @@ fn stream_rows<'a>( tile_ctx: &mut *tile_ctx, storage: &mut *storage, bands, + band_of, precinct_rows, profile_enabled, tier1_us: 0, @@ -1002,6 +1004,7 @@ fn decode_bands<'a>( ) -> Result<()> { let mut bands = Vec::new(); let active_levels = push_component_bands(storage, header, component_idx, &mut bands)?; + let band_of = band_lookup(&bands)?; let (precinct_rows, index_bytes) = index_bands(storage, tile_ctx, &bands)?; tile_ctx.idwt_output.placement = crate::j2c::idwt::IdwtPlacement::Coefficients; @@ -1028,6 +1031,7 @@ fn decode_bands<'a>( tile_ctx: &mut *tile_ctx, storage: &mut *storage, bands, + band_of, precinct_rows, profile_enabled, tier1_us: 0, @@ -1157,6 +1161,30 @@ struct SubBandBands { last_band: Option, } +/// The [`band_lookup`] entry of a sub-band id without bands. +const NO_BAND: usize = usize::MAX; + +/// For every sub-band id up to the largest in `bands`, the index of its +/// first entry there, or [`NO_BAND`]: the IDWT looks its sub-bands up +/// several times a row. +fn band_lookup(bands: &[SubBandBands]) -> Result> { + let len = bands + .iter() + .map(|bands| bands.sub_band + 1) + .max() + .unwrap_or(0); + let mut band_of = Vec::new(); + band_of + .try_reserve_exact(len) + .map_err(|_| DecodingError::HostAllocationFailed)?; + band_of.resize(len, NO_BAND); + // From the last, so that the first entry of a sub-band wins. + for (index, bands) in bands.iter().enumerate().rev() { + band_of[bands.sub_band] = index; + } + Ok(band_of) +} + struct PrecinctRow { first_y: u32, last_y: u32, @@ -1211,6 +1239,8 @@ struct BandedRows<'r, 'a> { tile_ctx: &'r mut TileDecodeContext, storage: &'r mut DecompositionStorage<'a>, bands: Vec, + /// [`band_lookup`] of `bands`. + band_of: Vec, precinct_rows: Option>>, profile_enabled: bool, tier1_us: u128, @@ -1218,9 +1248,10 @@ struct BandedRows<'r, 'a> { impl BandedRows<'_, '_> { fn position(&self, sub_band: usize) -> Result { - self.bands - .iter() - .position(|bands| bands.sub_band == sub_band) + self.band_of + .get(sub_band) + .copied() + .filter(|&index| index != NO_BAND) .ok_or(DecodingError::CodeBlockDecodeFailure.into()) } From 0fdb93307e608974e26ab0f18d45f3be399fea00 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 21:58:06 -0400 Subject: [PATCH 52/94] fix(j2k-native): charge the route list and release executor records before fallbacks A tile whose rows stream plans every component's route in one decode budget. That budget now starts with the list of routes, which was reserved uncharged. When the decode falls back, to serial bands or to the whole-window error rerun, the executor's rings, bands, strip offsets, active levels, line plans and ring lengths are released together with the routes, since neither fallback's budget includes them. A failure to record the routes' workspace now takes the same path as a failed band decode. The combined plan and executor budget test also runs a three-component image through the executor that streams rows, and checks that the cap rejects the decode before any row reaches the sink. The changelog gains the entry for the one budget, the band bookkeeping and the cheap-band cutoff. --- .../src/j2c/decode/parallel_bands.rs | 313 +++++++++++------- 1 file changed, 196 insertions(+), 117 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index 1593e1ab..2f6782d9 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -328,18 +328,24 @@ pub(super) fn decode_tile_rows<'a>( ) -> Result { let stage_start = profile::profile_now(profile_enabled); let components = tile.component_infos.len(); - let mut routes = Vec::new(); - routes - .try_reserve_exact(components) - .map_err(|_| DecodingError::HostAllocationFailed)?; // Every component's route stays live beside the others' while the // executor runs them all, so each is planned in the budget the last one - // left, as one budget. - let mut live = None; + // left, as one budget, which starts with the list of routes. + let mut live = DecodeAllocationBudget::for_storage(storage)?; + let mut routes = Vec::new(); + live.reserve_new(&mut routes, components)?; for component_idx in 0..components { - match plan_band_route_beside(tile, component_idx, tile_ctx, storage, header, live, true) { + match plan_band_route_beside( + tile, + component_idx, + tile_ctx, + storage, + header, + Some(live), + true, + ) { Ok(BandRouting::Route(route)) => { - live = Some(route.budget()); + live = route.budget(); routes.push(route); } Ok(BandRouting::Serial | BandRouting::WholeWindows) => return Ok(false), @@ -358,10 +364,6 @@ pub(super) fn decode_tile_rows<'a>( let line_plans = plan_tile_lines(tile, header, tile_ctx, storage, &active_levels)?; // The executor's records live beside every route's plan and scratch // while the bands decode. - let mut live = match live { - Some(live) => live, - None => DecodeAllocationBudget::for_storage(storage)?, - }; live.include_bytes(executor_bytes(&rings, &bands)?)?; let empty = IntRect::from_xywh(0, 0, 0, 0); let (copy, region) = line_plans @@ -510,14 +512,25 @@ pub(super) fn decode_tile_rows<'a>( let elapsed = profile::elapsed_us(stage_start); profile_timings.codeblock_us += tier1_us; profile_timings.idwt_us += elapsed.saturating_sub(tier1_us); - if matches!(result, Ok(true)) { - for route in &routes { - route.record(tile_ctx, storage)?; + let result = result.and_then(|completed| { + if completed { + for route in &routes { + route.record(tile_ctx, storage)?; + } } - } + Ok(completed) + }); // Neither the serial bands' nor the whole-window rerun's budget includes - // the plans' live allocations. - drop(routes); + // the plans or the executor's records, so they go first. + drop(( + routes, + rings, + bands, + strip_starts, + active_levels, + line_plans, + ring_lens, + )); match result { Ok(true) => { tile_ctx.debug_counters.pool_band_components += components; @@ -1844,9 +1857,9 @@ mod tests { use crate::reader::BitReader; use crate::{encode, DecodeSettings, EncodeOptions, Image, DEFAULT_MAX_DECODE_BYTES}; - fn budget_fixture(width: u32, height: u32) -> Vec { + fn budget_fixture(width: u32, height: u32, components: u16) -> Vec { let mut state = 0x2545_f491_u32; - let pixels: Vec = (0..width * height) + let pixels: Vec = (0..width * height * u32::from(components)) .map(|_| { state ^= state << 13; state ^= state >> 17; @@ -1858,7 +1871,7 @@ mod tests { &pixels, width, height, - 1, + components, 8, false, &EncodeOptions { @@ -1872,109 +1885,175 @@ mod tests { .expect("encode budget fixture") } - /// The route's plan and clipped scratch stay live beside the executor. - /// Leave enough simulated headroom for either alone, but not both. No - /// large retained allocation is needed to exercise the cap boundary. + struct CountingSink(usize); + + impl ComponentRowSink for CountingSink { + fn write_rows(&mut self, _x: u32, _y: u32, planes: &[crate::ComponentPlane<'_>]) { + self.0 += planes + .iter() + .map(|plane| plane.samples().len()) + .sum::(); + } + } + + /// Every live component plan and its clipped scratch must fit beside + /// the executor, including the shared executor that streams RGB rows. + /// Simulated retained memory leaves room for either owner, but not both. #[test] + #[expect( + clippy::too_many_lines, + reason = "one budget matrix checks the plans, the executor and the shared RGB executor together" + )] fn pool_bands_respect_the_combined_plan_and_executor_budget() { let (width, height) = (203, 157); - let bytes = budget_fixture(width, height); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("parse budget fixture"); - rayon::ThreadPoolBuilder::new() - .num_threads(2) - .build() - .expect("build budget test pool") - .install(|| { - let mut context = DecoderContext::default(); - context.set_output_region(Some((0, 0, width, height))); - context.tile_decode_context.line_region_idwt = true; - let tiles = tile::parse(&mut BitReader::new(image.codestream), &image.header, 0) - .expect("parse tile"); - let baseline = context - .reset( - &image.header, + for components in [1_u16, 3] { + let bytes = budget_fixture(width, height, components); + let image = + Image::new(&bytes, &DecodeSettings::default()).expect("parse budget fixture"); + rayon::ThreadPoolBuilder::new() + .num_threads(2) + .build() + .expect("build budget test pool") + .install(|| { + let mut context = DecoderContext::default(); + context.set_output_region(Some((0, 0, width, height))); + context.tile_decode_context.line_region_idwt = true; + context.tile_decode_context.stream_rows = components > 1; + let tiles = + tile::parse(&mut BitReader::new(image.codestream), &image.header, 0) + .expect("parse tile"); + let baseline = context + .reset( + &image.header, + &tiles[0], + tiles.structural_workspace_bytes(), + 0, + false, + None, + ) + .expect("prepare output planes"); + let (tile_ctx, storage) = + (&mut context.tile_decode_context, &mut context.storage); + let mut sink = CountingSink(0); + // Warm the retained bank and leave parsed block storage + // available at the tile boundary, before applying the cap. + let output = if components == 1 { + super::super::TileOutput::Planes + } else { + super::super::TileOutput::Sink(&mut sink) + }; + super::super::decode_tile( &tiles[0], - tiles.structural_workspace_bytes(), - 0, + &image.header, + tile_ctx, + storage, + &mut None, + CpuDecodeParallelism::Auto, false, - None, + &mut DecodeProfileTimings::default(), + baseline, + output, ) - .expect("prepare output planes"); - let (tile_ctx, storage) = (&mut context.tile_decode_context, &mut context.storage); - // Stop at the tile boundary so its parsed block storage stays - // available. This also prepares the retained worker bank, so - // the admission check below needs no replacement bank. - super::super::decode_tile( - &tiles[0], - &image.header, - tile_ctx, - storage, - &mut None, - CpuDecodeParallelism::Auto, - false, - &mut DecodeProfileTimings::default(), - baseline, - super::super::TileOutput::Planes, - ) - .expect("decode with ample headroom"); - assert_eq!(tile_ctx.debug_counters.pool_band_components, 1); - let (route_bytes, executor) = { - let BandRouting::Route(route) = - plan_band_route(&tiles[0], 0, tile_ctx, storage, &image.header, false) - .expect("plan control route") - else { - panic!("control fixture must take the pool route"); + .expect("decode with ample headroom"); + assert_eq!( + tile_ctx.debug_counters.pool_band_components, + usize::from(components) + ); + let (route_bytes, executor) = { + let mut routes = Vec::new(); + let mut route_bytes = 0; + let mut live = None; + for component in 0..usize::from(components) { + let BandRouting::Route(route) = plan_band_route_beside( + &tiles[0], + component, + tile_ctx, + storage, + &image.header, + live, + components > 1, + ) + .expect("plan control route") else { + panic!("control fixture must take the pool route"); + }; + live = Some(route.budget()); + // Reuse the existing peak accounting for the + // route's owned tables and clipped scratch, + // excluding the retained bank. + let retained = tile_ctx.tier1_capacity_bytes().expect("bank bytes"); + tile_ctx.debug_counters.workspace_peak = WorkspacePeak::default(); + route + .record(tile_ctx, storage) + .expect("record route owners"); + route_bytes += + tile_ctx.debug_counters.workspace_peak.bytes().tier1 - retained; + routes.push(route); + } + let (rings, bands, _, _) = plan_tile_rings(&routes, storage, &image.header) + .expect("plan control rings"); + ( + route_bytes, + executor_bytes(&rings, &bands).expect("executor bytes"), + ) }; - // Reuse the existing peak accounting for the route's owned - // tables and clipped scratch, excluding the retained bank. - let retained_tier1 = tile_ctx.tier1_capacity_bytes().expect("bank bytes"); - tile_ctx.debug_counters.workspace_peak = WorkspacePeak::default(); - route - .record(tile_ctx, storage) - .expect("record route owners"); - let route_bytes = - tile_ctx.debug_counters.workspace_peak.bytes().tier1 - retained_tier1; - let (rings, bands, _) = - plan_rings(&route, storage, &image.header, 0).expect("plan control rings"); - ( - route_bytes, - executor_bytes(&rings, &bands).expect("executor bytes"), - ) - }; - let headroom = route_bytes + executor / 2; - assert!(headroom >= executor, "each owner must fit separately"); - let baseline = DecodeAllocationBudget::for_storage(storage) - .expect("control budget") - .live_bytes(); - storage.structural_workspace_bytes += - DEFAULT_MAX_DECODE_BYTES - baseline - headroom; - let route = - match plan_band_route(&tiles[0], 0, tile_ctx, storage, &image.header, false) { - Ok(BandRouting::Route(route)) => route, - Ok(BandRouting::Serial | BandRouting::WholeWindows) => return, - Err(error) if super::super::reuse::is_capacity_error(&error) => return, - Err(error) => panic!("unexpected planning error: {error:?}"), + let headroom = route_bytes + executor / 2; + assert!(headroom >= executor, "each owner must fit separately"); + let baseline = DecodeAllocationBudget::for_storage(storage) + .expect("control budget") + .live_bytes(); + storage.structural_workspace_bytes += + DEFAULT_MAX_DECODE_BYTES - baseline - headroom; + sink.0 = 0; + let result = if components == 1 { + let route = match plan_band_route( + &tiles[0], + 0, + tile_ctx, + storage, + &image.header, + false, + ) { + Ok(BandRouting::Route(route)) => route, + Ok(BandRouting::Serial | BandRouting::WholeWindows) => return, + Err(error) if super::super::reuse::is_capacity_error(&error) => return, + Err(error) => panic!("unexpected planning error: {error:?}"), + }; + decode_route( + &route, + 0, + &image.header, + tile_ctx, + storage, + tiles[0].component_infos[0].wavelet_transform(), + false, + &mut 0, + ) + } else { + decode_tile_rows( + &tiles[0], + &image.header, + tile_ctx, + storage, + &mut sink, + false, + &mut DecodeProfileTimings::default(), + ) }; - let result = decode_route( - &route, - 0, - &image.header, - tile_ctx, - storage, - tiles[0].component_infos[0].wavelet_transform(), - false, - &mut 0, - ); - match result { - Err(error) if super::super::reuse::is_capacity_error(&error) => {} - Ok(false) => {} - other => panic!( - "pool route must decline or reject combined live owners: \ - headroom={headroom}, route={route_bytes}, executor={executor}, \ - excess={}, result={other:?}", - route_bytes + executor - headroom, - ), - } - }); + match result { + Err(error) if super::super::reuse::is_capacity_error(&error) => {} + Ok(false) => {} + other => panic!( + "pool route must decline or reject combined live owners: \ + components={components}, headroom={headroom}, route={route_bytes}, \ + executor={executor}, excess={}, result={other:?}", + route_bytes + executor - headroom, + ), + } + assert_eq!( + sink.0, 0, + "budget rejection must precede streaming any rows" + ); + }); + } } } From 2e76ce3def0c3a62869b8ecc417ab00c4206686c Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 21:58:06 -0400 Subject: [PATCH 53/94] docs(changelog): note the streamed-route speedups and the one decode budget --- CHANGELOG.md | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e53a78e5..19d086e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,13 @@ Staged workspace version: `0.13.2`. status, detail and batch source index. CUDA keeps its typed source and job attribution. Unsupported statuses, raw runtime failures and combined cleanup or release failures keep their existing classes. +- Parallel band decoding charges each component's plan and the band + executor's records to one decode budget, across all components when a + tile's rows stream together, and releases the plans and the records before + whole-window error recovery or the serial fallback. Band bookkeeping no + longer rescans completed bands, and components whose bands average too + little work for the pool decode over whole windows instead, or on serial + bands when their rows stream. - Classic code-blocks with BYPASS, per-pass termination, context resets or segmentation symbols use the packed-column passes, including raw segments. Vertically causal blocks retain the per-coefficient passes. Clean and corrupt @@ -147,7 +154,10 @@ Staged workspace version: `0.13.2`. smaller classic outputs from 256 x 256 on 2 to 11 pool threads; full-resolution classic images coded at 4 bits per pixel or more from 512 x 512 on 2 to 11 pool threads (5-17% faster on 6 threads) and, below - 2^19 samples, on larger pools too; and full-resolution images with a tile + 2^19 samples, on larger pools too; full-resolution classic images coded at + fewer bits per pixel from 4096 x 2048 on 2 to 6 pool threads and from 12 + (a 4096 x 2048 RGB image at 2 bpp 1% faster on 6 threads and 4-5% on + 18, in about half the memory); and full-resolution images with a tile for every thread. Those now decode their tiles at once on the calling thread and the pool's threads, each tile on one thread, instead of one tile after another: a 4096 x 8192 RGB image of 1024 x 1024 tiles decodes 8% faster than @@ -165,6 +175,21 @@ Staged workspace version: `0.13.2`. third of the memory). Reduced and tiled gray images, full-resolution HTJ2K images, reduced tiled images, small images and images with few tiles on a large pool keep the planes route. Output is unchanged. +- Row-streamed decodes of three 8-bit components into RGB8 output write the + reconstructed rows straight into the output: 9/7 images with the + multiple-component transform pack each row through the planes route's + fused inverse-ICT pack instead of copying it out of the inverse wavelet + transform, transforming it in place and packing it through the row sink, + and other images pack into the output after the in-place transform. + Parallel row-streamed decodes keep a classic component's bands on the + worker pool when the component has fewer rows of code-blocks than the pool + has threads, where they decoded on one thread (a whole-image region of a + 512 x 512 RGB image at 2 bpp decodes 4.9 times as fast on 18 threads), + and serial ones look each sub-band up directly instead of searching the + tile's sub-bands several times a row (whole-image regions of 256 x 256 + and 512 x 512 tiles 4-9% faster on one thread). An 18000 x 12000 RGB + image decodes 1-2% faster on one thread and 3-5% faster on 18. Output is + unchanged. - x86-64 RGB8 decodes of 9/7 images round and pack the inverse ICT output with the runtime SIMD level (AVX2 where present) instead of a scalar library call per sample: the centered rounding of the inverse MCT runs eight @@ -242,6 +267,35 @@ Staged workspace version: `0.13.2`. at least 128 rows, since they hold only one batch. A full-image region decode of a 5661 x 8287 RGB image to RGB8 peaks 12 MiB (12%) lower at 18 threads, at the same speed. +- The row-streamed IDWT reads band rows of up to 16384 samples that hold no + code-block as zeros through its SIMD kernel; such rows wider than 4096 + samples took the scalar path. An 18000 x 12000 RGB image, whose chroma + sub-bands have many such rows, decodes 11 to 12% faster at 18 threads + (441 to 391 ms) and 5% faster at 10. Output is bit-identical. +- Parallel CPU region decodes of classic tiles of several components and + at least 4096 x 4096 samples per component reconstruct each component on + its own pool reader from 10 pool threads, instead of 12, holding one + batch of rows as before. With one reader that reconstructs every + component and packs every row, such tiles were bound by that reader on + 10 and 11 threads: on a 10-thread pool an 18000 x 12000 RGB image now + decodes 11% faster (980 to 868 ms), 5661 x 8287 RGB 8%, 16384 x 16384 + RGB 7% and 8192 x 8192 RGB 5%, at the same peak heap. Smaller tiles, which + spend more of their time in Tier-1, keep one reader. Output is + bit-identical. +- Parallel CPU region decodes that reconstruct every component through one + reader (gray images, and classic tiles on fewer than 12 pool threads that + keep one reader) hold at most 32 rows of each component at a time instead + of up to 512, since that reader transforms and hands over each batch + before it reconstructs the next. A full-image decode of a 5661 x 8287 RGB + image to RGB8 on 6 threads peaks 22 MiB (33%) lower, at the same speed, + and 8 to 16 MiB lower at 1/4 and 1/2 scale; 4096 x 2048, 16384 x 16384 and + 18000 x 12000 RGB images drop 17 to 23 MiB, gray images 7 MiB at any pool + size. +- Classic tiles with a reader per component hold the extra band in every + ring only when they read whole rows at full resolution; overviews and + narrower regions keep one batch of rows without it. A 5661 x 8287 RGB + image's overviews at 1/2, 1/4 and 1/8 scale peak about 5.5, 2.5 and 1 MiB + lower at 12 threads or more, and 1024 x 1024 regions about 2.3 MiB lower. ## [0.13.1] - 2026-10-29 From eeb9700ba240b808d365eb89daf034b9b0820ae9 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 18:59:07 -0400 Subject: [PATCH 54/94] perf(j2k-native): take fresh decode buffers zeroed from the allocator Component planes, tile coefficient storage, IDWT level buffers and the streamed line rings were reserved and then zero-filled on the calling thread. In a process's first decode that fill is the first write to each page, so every page fault of the decode's largest buffers ran on that thread, one after another, while the pool waited. On a pool of up to 10 threads, a buffer that needs a new allocation now comes zeroed from the allocator (fallible `alloc_zeroed` through `bytemuck::allocation::try_zeroed_vec`, reported as a host allocation failure like `try_reserve`). Memory fresh from the system is already zero, so the threads that fill the buffer take its page faults. On larger pools buffers are zeroed on the calling thread as before: there the spread faults contend in the kernel, and on 18 threads they took about four times the system time while HTJ2K images, which decode each page quickly, lost 3 to 7%. Reused buffers keep their allocation and are zeroed as before. A buffer that must grow is released before the new one is taken instead of being copied into it, since nothing left in it is read, and the coefficient reservation refuses an over-cap capacity the way `reserve` does. New dependency: `bytemuck` (no default features, `extern_crate_alloc`), already in the lockfile. Measured with one decode per fresh process (the first decode, including pool start) and warm decodes with a fresh decoder each, 7 interleaved rounds, output byte-identical, against the parent: - 18-core ARM64 machine, 6 threads, first decode: 4096 x 2048 RGB HTJ2K mosaics 12.4 -> 9.8 ms (0.77x), 14.7 -> 12.0 ms (0.83x) and 13.1 -> 10.5 ms (0.82x); a 4096 x 4096 HTJ2K RGB image 28.3 -> 21.6 ms (0.76x); a 4096 x 2048 classic RGB mosaic 40.5 -> 34.8 ms (0.87x); a 1/4-scale decode of an HTJ2K image 18.7 -> 16.5 ms (0.89x). At 18 threads and the default pool the decode takes the old path. - 10-core ARM64 machine (4 performance, 6 efficiency cores), 4 and 10 threads: the same images 0.84-0.91x, the reduced decode 0.90x; at one thread 0.98-1.00x. - Warm decodes 0.98-1.02x; streamed and tiled images 0.99-1.01x on the 10-core machine. Peak heap unchanged. --- Cargo.lock | 1 + Cargo.toml | 1 + crates/j2k-native/Cargo.toml | 1 + crates/j2k-native/src/j2c/build.rs | 8 +-- crates/j2k-native/src/j2c/build/allocation.rs | 8 +-- .../src/j2c/build/allocation/reuse.rs | 50 +++++++++++++++- .../src/j2c/build/allocation/tests.rs | 60 +++++++++++++++++++ .../j2k-native/src/j2c/decode/banded_tier1.rs | 4 +- crates/j2k-native/src/j2c/idwt.rs | 9 ++- crates/j2k-native/src/j2c/idwt/roi.rs | 7 ++- crates/j2k-native/src/lib.rs | 46 ++++++++++++++ crates/j2k-native/src/math.rs | 4 +- crates/j2k-native/src/tests.rs | 46 ++++++++++++++ 13 files changed, 226 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f03e74a5..7d8d12ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2772,6 +2772,7 @@ dependencies = [ name = "j2k-native" version = "0.13.2" dependencies = [ + "bytemuck", "criterion", "fearless_simd", "j2k-codec-math", diff --git a/Cargo.toml b/Cargo.toml index 61574394..aaae971c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -103,6 +103,7 @@ objc2-metal-performance-shaders-graph = { version = "=0.3.2", default-features = ] } block2 = { version = "=0.6.2", default-features = false, features = ["std"] } rayon = "1" +bytemuck = { version = "1.25", default-features = false } cc = "1.2" libc = "0.2" windows-sys = "0.61" diff --git a/crates/j2k-native/Cargo.toml b/crates/j2k-native/Cargo.toml index 89bc3796..55e57c52 100644 --- a/crates/j2k-native/Cargo.toml +++ b/crates/j2k-native/Cargo.toml @@ -26,6 +26,7 @@ name = "j2k_native" path = "src/lib.rs" [dependencies] +bytemuck = { workspace = true, features = ["extern_crate_alloc"] } fearless_simd = { workspace = true, optional = true } libm = { workspace = true } log = { workspace = true, optional = true } diff --git a/crates/j2k-native/src/j2c/build.rs b/crates/j2k-native/src/j2c/build.rs index 57500721..f07a88fe 100644 --- a/crates/j2k-native/src/j2c/build.rs +++ b/crates/j2k-native/src/j2c/build.rs @@ -6,7 +6,7 @@ use super::roi::RoiPlan; use super::tag_tree::{TagNode, TagTree}; use super::tile::{ResolutionTile, Tile}; use crate::error::{DecodingError, Result, ValidationError}; -use crate::try_resize_decode_elements; +use crate::{try_resize_decode_elements, try_zero_fill_decode_elements}; use core::{iter, ops::Range}; mod allocation; @@ -222,11 +222,7 @@ fn resize_planned_coefficients( storage: &mut DecompositionStorage<'_>, plan: &DecompositionAllocationPlan, ) -> Result<()> { - try_resize_decode_elements( - &mut storage.coefficients, - plan.coefficient_storage_len(), - 0.0, - )?; + try_zero_fill_decode_elements(&mut storage.coefficients, plan.coefficient_storage_len())?; if storage.exact_integer_decode { try_resize_decode_elements( &mut storage.coefficients_i64, diff --git a/crates/j2k-native/src/j2c/build/allocation.rs b/crates/j2k-native/src/j2c/build/allocation.rs index a2956bd4..06bcb057 100644 --- a/crates/j2k-native/src/j2c/build/allocation.rs +++ b/crates/j2k-native/src/j2c/build/allocation.rs @@ -380,8 +380,8 @@ pub(crate) fn allocate_deferred_coefficients( .unwrap_or(0) }; let mut budget = reuse::ReallocationBudget::from_live_bytes(live_bytes)?; - budget.reserve(&mut storage.coefficients, coefficient_count)?; - crate::try_resize_decode_elements(&mut storage.coefficients, coefficient_count, 0.0)?; + budget.reserve_zeroed(&mut storage.coefficients, coefficient_count)?; + crate::try_zero_fill_decode_elements(&mut storage.coefficients, coefficient_count)?; storage.structural_workspace_bytes = budget.live_bytes(); Ok(()) } @@ -519,8 +519,8 @@ pub(crate) fn lay_out_whole_region_coefficients( let coefficient_count = lay_out_region_coefficients(storage, true, 0)?; let mut budget = reuse::ReallocationBudget::from_live_bytes(storage.structural_workspace_bytes)?; - budget.reserve(&mut storage.coefficients, coefficient_count)?; - crate::try_resize_decode_elements(&mut storage.coefficients, coefficient_count, 0.0)?; + budget.reserve_zeroed(&mut storage.coefficients, coefficient_count)?; + crate::try_zero_fill_decode_elements(&mut storage.coefficients, coefficient_count)?; storage.structural_workspace_bytes = budget.live_bytes(); Ok(()) } diff --git a/crates/j2k-native/src/j2c/build/allocation/reuse.rs b/crates/j2k-native/src/j2c/build/allocation/reuse.rs index 0577021f..d094157f 100644 --- a/crates/j2k-native/src/j2c/build/allocation/reuse.rs +++ b/crates/j2k-native/src/j2c/build/allocation/reuse.rs @@ -65,6 +65,54 @@ impl ReallocationBudget { self.live_bytes } + /// [`Self::reserve`] for storage that starts as zeros. A buffer that + /// needs a new allocation is replaced by a zeroed one of `target_len` + /// elements (see [`crate::try_zero_fill_decode_elements`]); its old + /// contents are dead, so it is released first and never held beside it. + pub(super) fn reserve_zeroed( + &mut self, + values: &mut Vec, + target_len: usize, + ) -> Result<()> { + values.clear(); + if target_len <= values.capacity() { + return Ok(()); + } + let old_bytes = values + .capacity() + .checked_mul(size_of::()) + .ok_or(ValidationError::ImageTooLarge)?; + *values = Vec::new(); + self.live_bytes = self + .live_bytes + .checked_sub(old_bytes) + .ok_or(ValidationError::ImageTooLarge)?; + let requested_bytes = target_len + .checked_mul(size_of::()) + .ok_or(ValidationError::ImageTooLarge)?; + if self + .live_bytes + .checked_add(requested_bytes) + .is_none_or(|bytes| bytes > DEFAULT_MAX_DECODE_BYTES) + { + return Err(ValidationError::ImageTooLarge.into()); + } + crate::try_zero_fill_decode_elements(values, target_len)?; + let new_live_bytes = values + .capacity() + .checked_mul(size_of::()) + .and_then(|bytes| self.live_bytes.checked_add(bytes)) + .ok_or(ValidationError::ImageTooLarge)?; + if new_live_bytes > DEFAULT_MAX_DECODE_BYTES { + // As in `reserve`: never keep a larger allocation than requested + // after reporting the cap breach. + *values = Vec::new(); + return Err(ValidationError::ImageTooLarge.into()); + } + self.live_bytes = new_live_bytes; + Ok(()) + } + pub(super) fn reserve(&mut self, values: &mut Vec, target_len: usize) -> Result<()> { values.clear(); if target_len <= values.capacity() { @@ -134,7 +182,7 @@ pub(super) fn reserve_decomposition_storage( budget.reserve(&mut storage.code_blocks, plan.code_blocks)?; budget.reserve(&mut storage.layers, plan.layers)?; budget.reserve(&mut storage.tag_tree_nodes, plan.tag_tree_nodes)?; - budget.reserve(&mut storage.coefficients, plan.coefficient_storage_len())?; + budget.reserve_zeroed(&mut storage.coefficients, plan.coefficient_storage_len())?; if storage.exact_integer_decode { budget.reserve( &mut storage.coefficients_i64, diff --git a/crates/j2k-native/src/j2c/build/allocation/tests.rs b/crates/j2k-native/src/j2c/build/allocation/tests.rs index 1d294c86..d2b20a7c 100644 --- a/crates/j2k-native/src/j2c/build/allocation/tests.rs +++ b/crates/j2k-native/src/j2c/build/allocation/tests.rs @@ -441,6 +441,66 @@ fn reusable_capacity_growth_rejects_a_logical_target_over_the_aggregate_cap() { ); } +#[test] +fn zeroed_growth_releases_the_old_buffer_before_taking_the_new_one() { + let mut storage = DecompositionStorage::default(); + storage.coefficients.push(9.0); + let old_bytes = storage.coefficients.capacity() * size_of::(); + let target = storage.coefficients.capacity() + 1; + // Room for the new buffer only once the old one is gone. + let baseline = DEFAULT_MAX_DECODE_BYTES - target * size_of::(); + let mut reallocation = + ReallocationBudget::for_storage(&storage, baseline).expect("live baseline"); + assert_eq!(reallocation.live_bytes(), baseline + old_bytes); + + reallocation + .reserve_zeroed(&mut storage.coefficients, target) + .expect("the old buffer is released first"); + assert_eq!(storage.coefficients, vec![0.0; target]); + assert_eq!( + reallocation.live_bytes(), + baseline + storage.coefficients.capacity() * size_of::() + ); +} + +#[test] +fn zeroed_reuse_keeps_the_allocation_and_none_of_its_samples() { + let mut storage = DecompositionStorage::default(); + storage.coefficients.extend([5.0_f32; 8]); + let capacity = storage.coefficients.capacity(); + let baseline = DEFAULT_MAX_DECODE_BYTES - capacity * size_of::(); + let mut reallocation = + ReallocationBudget::for_storage(&storage, baseline).expect("live baseline"); + + reallocation + .reserve_zeroed(&mut storage.coefficients, 6) + .expect("the retained allocation has room"); + crate::try_zero_fill_decode_elements(&mut storage.coefficients, 6).expect("zero fill"); + assert_eq!(storage.coefficients.capacity(), capacity); + assert_eq!(storage.coefficients, vec![0.0; 6]); + assert_eq!(reallocation.live_bytes(), DEFAULT_MAX_DECODE_BYTES); +} + +#[test] +fn zeroed_growth_rejects_a_target_over_the_aggregate_cap() { + let mut storage = DecompositionStorage::default(); + storage + .coefficients + .try_reserve_exact(1) + .expect("small test capacity"); + let coefficient_bytes = storage.coefficients.capacity() * size_of::(); + let baseline = DEFAULT_MAX_DECODE_BYTES - coefficient_bytes; + let mut reallocation = + ReallocationBudget::for_storage(&storage, baseline).expect("live baseline"); + let target = storage.coefficients.capacity() + 1; + + assert_eq!( + reallocation.reserve_zeroed(&mut storage.coefficients, target), + Err(DecodeError::Validation(ValidationError::ImageTooLarge)) + ); + assert_eq!(storage.coefficients.capacity(), 0); +} + #[test] fn region_decode_scratch_preserves_the_allocation_boundary() { use crate::{ diff --git a/crates/j2k-native/src/j2c/decode/banded_tier1.rs b/crates/j2k-native/src/j2c/decode/banded_tier1.rs index bd2c8120..f579a15c 100644 --- a/crates/j2k-native/src/j2c/decode/banded_tier1.rs +++ b/crates/j2k-native/src/j2c/decode/banded_tier1.rs @@ -30,7 +30,7 @@ use crate::j2c::idwt::{ComponentLines, LinePlan, LineReconstruction, SubBandRows use crate::j2c::mct::DeferredMct; use crate::j2c::rect::IntRect; use crate::profile; -use crate::{try_resize_decode_elements, ComponentPlane, ComponentRowSink}; +use crate::{try_zero_fill_decode_elements, ComponentPlane, ComponentRowSink}; /// Whether a tile's Tier-1 decodes in bands, given that it feeds the /// streamed IDWT: in stored windows clipped to it, on the calling thread @@ -293,7 +293,7 @@ pub(super) fn take_line_buffer( ); let mut buffer = core::mem::take(&mut tile_ctx.idwt_line_rings); let resized = if buffer.len() < total { - try_resize_decode_elements(&mut buffer, total, 0.0) + try_zero_fill_decode_elements(&mut buffer, total) } else { Ok(()) }; diff --git a/crates/j2k-native/src/j2c/idwt.rs b/crates/j2k-native/src/j2c/idwt.rs index f0d4a5f2..c820c116 100644 --- a/crates/j2k-native/src/j2c/idwt.rs +++ b/crates/j2k-native/src/j2c/idwt.rs @@ -2,7 +2,7 @@ use alloc::vec::Vec; -use crate::{try_resize_decode_elements, Result}; +use crate::{try_zero_fill_decode_elements, Result}; mod banded; mod direct; @@ -65,7 +65,12 @@ fn try_grow_level_buffer(buffer: &mut Vec, len: usize) -> Result<()> { } #[cfg(test)] ZERO_INITIALIZED_SAMPLES.set(ZERO_INITIALIZED_SAMPLES.get() + len - buffer.len()); - try_resize_decode_elements(buffer, len, 0.0) + if buffer.capacity() < len { + // Nothing left behind is read, so a buffer that must grow is replaced + // by a zeroed one rather than copied into a larger allocation. + buffer.clear(); + } + try_zero_fill_decode_elements(buffer, len) } #[cfg(test)] diff --git a/crates/j2k-native/src/j2c/idwt/roi.rs b/crates/j2k-native/src/j2c/idwt/roi.rs index 2a70535f..2b2ff075 100644 --- a/crates/j2k-native/src/j2c/idwt/roi.rs +++ b/crates/j2k-native/src/j2c/idwt/roi.rs @@ -15,7 +15,10 @@ use crate::error::{DecodingError, ValidationError}; use crate::j2c::roi::output_region_rect; use crate::j2c::Header; use crate::math::{SimdBuffer, SIMD_WIDTH}; -use crate::{checked_decode_usize_product2, try_resize_decode_elements, Result}; +use crate::{ + checked_decode_usize_product2, try_resize_decode_elements, try_zero_fill_decode_elements, + Result, +}; /// Samples of each component's rows a tile streaming from pool bands /// reconstructs at most at once, before they are transformed and handed over. @@ -490,7 +493,7 @@ impl ComponentLines { let mut ring = core::mem::take(&mut tile_ctx.idwt_line_rings); let ring_len = plan.ring_len()?; if ring.len() < ring_len { - try_resize_decode_elements(&mut ring, ring_len, 0.0)?; + try_zero_fill_decode_elements(&mut ring, ring_len)?; } tile_ctx.idwt_line_rings = ring; record_workspace_now(tile_ctx, storage)?; diff --git a/crates/j2k-native/src/lib.rs b/crates/j2k-native/src/lib.rs index aca22e07..5d00c464 100644 --- a/crates/j2k-native/src/lib.rs +++ b/crates/j2k-native/src/lib.rs @@ -379,6 +379,52 @@ pub(crate) fn try_resize_decode_elements( Ok(()) } +/// [`try_resize_decode_elements`] to zeros. On a pool of at most +/// [`MAX_POOL_THREADS_FOR_ZEROED_BUFFERS`] threads, an empty buffer whose +/// capacity is short of `target_len` is replaced by a zeroed allocation +/// instead of being zeroed here: memory fresh from the system is already +/// zero, so the threads that fill the buffer take its page faults, instead +/// of this thread taking every one of them first while the pool waits. +pub(crate) fn try_zero_fill_decode_elements( + values: &mut Vec, + target_len: usize, +) -> Result<()> { + zero_fill_decode_elements( + values, + target_len, + cpu_decode_pool_threads() <= MAX_POOL_THREADS_FOR_ZEROED_BUFFERS, + ) +} + +/// Most pool threads for which fresh decode buffers come zeroed from the +/// allocator. First-touch page faults on one allocation from many threads +/// at once contend in the kernel: on 4 to 10 threads, spreading them over +/// the pool cut a process's first decode of a 4096 x 2048 RGB image by 10 to +/// 15%, but on 18 threads the faults took about four times the system time +/// and HTJ2K images, which decode each page quickly, lost 3 to 7%. +const MAX_POOL_THREADS_FOR_ZEROED_BUFFERS: usize = 10; + +/// [`try_zero_fill_decode_elements`] with the choice made. An empty buffer +/// short of capacity holds nothing that is read, so it is released rather +/// than copied into a larger allocation; the new one comes zeroed from the +/// allocator when `zeroed_allocation`, else it is zeroed here. +fn zero_fill_decode_elements( + values: &mut Vec, + target_len: usize, + zeroed_allocation: bool, +) -> Result<()> { + checked_decode_byte_len2(target_len, core::mem::size_of::())?; + if values.is_empty() && values.capacity() < target_len { + *values = Vec::new(); + if zeroed_allocation { + *values = bytemuck::allocation::try_zeroed_vec(target_len) + .map_err(|()| DecodingError::HostAllocationFailed)?; + return Ok(()); + } + } + try_resize_decode_elements(values, target_len, T::default()) +} + #[inline] pub(crate) fn checked_decode_sample_count(width: u32, height: u32) -> Result { #[cfg(target_pointer_width = "64")] diff --git a/crates/j2k-native/src/math.rs b/crates/j2k-native/src/math.rs index e5de1cef..da54e976 100644 --- a/crates/j2k-native/src/math.rs +++ b/crates/j2k-native/src/math.rs @@ -922,9 +922,9 @@ impl SimdBuffer { let capacity = Self::padded_len(capacity_len.max(original_len)).ok_or(SimdBufferAllocationError)?; let mut data = Vec::new(); - data.try_reserve_exact(capacity) + crate::try_zero_fill_decode_elements(&mut data, capacity) .map_err(|_| SimdBufferAllocationError)?; - data.resize(padded_len, 0.0); + data.truncate(padded_len); Ok(Self { data, original_len }) } diff --git a/crates/j2k-native/src/tests.rs b/crates/j2k-native/src/tests.rs index cf26bc60..120b9109 100644 --- a/crates/j2k-native/src/tests.rs +++ b/crates/j2k-native/src/tests.rs @@ -2979,3 +2979,49 @@ fn decode_settings_constructors_make_strictness_explicit() { assert!(!DecodeSettings::strict().lenient_tolerance_enabled()); assert!(DecodeSettings::strict().strict); } + +#[test] +fn zero_filled_decode_elements_start_as_zeros_whatever_the_buffer_held() { + for zeroed_allocation in [true, false] { + // An empty buffer without room gets a fresh zeroed buffer. + let mut fresh = Vec::::new(); + zero_fill_decode_elements(&mut fresh, 4099, zeroed_allocation).expect("fresh buffer"); + assert_eq!(fresh.len(), 4099); + assert!(fresh.iter().all(|&sample| sample == 0.0)); + + // An emptied buffer with room keeps its allocation, and none of the + // samples it held before shows through. + let mut reused = vec![7.5_f32; 64]; + let capacity = reused.capacity(); + reused.clear(); + zero_fill_decode_elements(&mut reused, 48, zeroed_allocation).expect("reused buffer"); + assert_eq!(reused.capacity(), capacity); + assert_eq!(reused, vec![0.0; 48]); + + // A buffer that already holds samples keeps them, as + // `try_resize_decode_elements` does, and grows with zeros. + let mut grown = vec![3_i64; 3]; + zero_fill_decode_elements(&mut grown, 5, zeroed_allocation).expect("grown buffer"); + assert_eq!(grown, [3, 3, 3, 0, 0]); + } +} + +#[test] +fn zero_filled_decode_elements_reject_lengths_past_the_decode_cap() { + for zeroed_allocation in [true, false] { + let mut values = Vec::::new(); + assert_eq!( + zero_fill_decode_elements( + &mut values, + DEFAULT_MAX_DECODE_BYTES / 4 + 1, + zeroed_allocation + ), + Err(DecodeError::Validation(ValidationError::ImageTooLarge)) + ); + assert_eq!( + zero_fill_decode_elements(&mut values, usize::MAX, zeroed_allocation), + Err(DecodeError::Validation(ValidationError::ImageTooLarge)) + ); + assert_eq!(values.capacity(), 0); + } +} From f963f0e1a605850f6905f9737b4813542f9db5b5 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 18:59:07 -0400 Subject: [PATCH 55/94] docs(changelog): note decode buffers zeroed by the allocator --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19d086e5..f8989bbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,16 @@ Staged workspace version: `0.13.2`. term the charge grows to match; elsewhere it is unchanged and the IDWT holds less within it (a 1024 x 768 gray image on 4 threads: 3.9 MB in place of 4.5 MB, against a 4.1 MB charge). Output is unchanged. +- On a pool of up to 10 threads, decodes take fresh component planes, + coefficient storage and IDWT buffers zeroed from the allocator instead of + zeroing them on the calling thread, so the page faults of a process's first + decode are spread over the threads that fill those buffers instead of + running one after another while the pool waits. In a fresh process, + 4096-wide RGB images that decode to planes, HTJ2K and classic, and a + 1/4-scale HTJ2K decode take 5 to 24% less time at 4 to 10 threads. Warm + decodes, output and peak heap are unchanged, and so are decodes on larger + pools, where the faults would contend in the kernel. New dependency: + `bytemuck`. - Decodes that would exceed a resource limit now fail with `J2kError::ResourceLimit { requested, cap, source }` instead of the generic `J2kError::NativeDecode`, on every decode route: `J2kDecoder`, the tile From fd77ee338a109f0441de457c4ffc35fff36b896a Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 22:49:21 -0400 Subject: [PATCH 56/94] fix(j2k-native): stream rows from serial bands when the planes would pass the budget A region decode under Auto whose rows go to a sink streams them from the pool's bands, or on one thread from serial bands. When a tile's blocks are ones the pool's bands cannot decode (classic code-blocks with selective bypass or per-pass termination), the decode fell back to component planes of the region, and on a pool of two threads or more a full-image decode whose planes pass the decode budget failed with a resource limit, although the same decode streams on one thread. Such a decode now chooses serial bands before anything is allocated: when its rows would stream serially, and the region's planes, as `reset` charges them, beside the structural bytes without the context's retained capacity, pass the budget. The planes decode would be refused there, so no decode that finishes today changes route or parallelism. On a 12-core ARM64 machine, full-image decodes of a 5661 x 8287 RGB image with selective bypass and of a 17311 x 13396 one that failed on 2, 4 and 12 threads now finish in 1.34-1.40 s and 3.52 s, their one-thread times, with the one-thread bytes; images that decoded keep their time and heap. The tests decode ten small OpenJPEG codestreams with selective bypass, per-pass termination, both, context resets, segmentation symbols and all six code-block styles, RGB and gray, both wavelets, RPCL with three to seven layers, one with 64 x 64 blocks, precincts and packet lengths: on pools of 2, 4 and 12 threads the full image, regions and streamed rows equal one thread's, and on one thread the rows equal the planes. A test-only planes budget, read by `reset` and by the new choice, shows a full-image rows decode on a pool whose pool bands decline the tile failing on the planes without the change, and streaming from serial bands with the same samples with it. --- .../multi_segment/all_modes_gray_97.j2k | Bin 0 -> 5461 bytes .../fixtures/multi_segment/bypass_gray_53.j2k | Bin 0 -> 8198 bytes .../fixtures/multi_segment/bypass_rgb_97.j2k | Bin 0 -> 6137 bytes .../multi_segment/bypass_rgb_97_64x64.j2k | Bin 0 -> 5253 bytes .../multi_segment/bypass_termall_gray_97.j2k | Bin 0 -> 5464 bytes .../multi_segment/bypass_termall_rgb_53.j2k | Bin 0 -> 8206 bytes .../fixtures/multi_segment/reset_rgb_97.j2k | Bin 0 -> 6140 bytes .../multi_segment/segmark_gray_53.j2k | Bin 0 -> 5447 bytes .../multi_segment/termall_gray_97.j2k | Bin 0 -> 5464 bytes .../fixtures/multi_segment/termall_rgb_53.j2k | Bin 0 -> 6127 bytes crates/j2k-native/src/j2c/decode.rs | 68 +++- .../j2k-native/src/j2c/decode/planes_floor.rs | 11 +- crates/j2k-native/src/j2c/decode/reuse.rs | 20 +- crates/j2k-native/src/j2c/decode/state.rs | 9 + crates/j2k-native/src/tests.rs | 2 + crates/j2k-native/src/tests/multi_segment.rs | 309 ++++++++++++++++++ 16 files changed, 397 insertions(+), 22 deletions(-) create mode 100644 crates/j2k-native/fixtures/multi_segment/all_modes_gray_97.j2k create mode 100644 crates/j2k-native/fixtures/multi_segment/bypass_gray_53.j2k create mode 100644 crates/j2k-native/fixtures/multi_segment/bypass_rgb_97.j2k create mode 100644 crates/j2k-native/fixtures/multi_segment/bypass_rgb_97_64x64.j2k create mode 100644 crates/j2k-native/fixtures/multi_segment/bypass_termall_gray_97.j2k create mode 100644 crates/j2k-native/fixtures/multi_segment/bypass_termall_rgb_53.j2k create mode 100644 crates/j2k-native/fixtures/multi_segment/reset_rgb_97.j2k create mode 100644 crates/j2k-native/fixtures/multi_segment/segmark_gray_53.j2k create mode 100644 crates/j2k-native/fixtures/multi_segment/termall_gray_97.j2k create mode 100644 crates/j2k-native/fixtures/multi_segment/termall_rgb_53.j2k create mode 100644 crates/j2k-native/src/tests/multi_segment.rs diff --git a/crates/j2k-native/fixtures/multi_segment/all_modes_gray_97.j2k b/crates/j2k-native/fixtures/multi_segment/all_modes_gray_97.j2k new file mode 100644 index 0000000000000000000000000000000000000000..f0a4bfee1ca343fab04f6aa92dfcf96b233958b3 GIT binary patch literal 5461 zcmV-b6{_n0PybN>DF6Tf004jh004jh0000000000004jh004jh00000000000S5sA z|55-900IC500II6KLGz+01-l8Qcwj@1yDy<(^u11V*g|SB>(|Ka%Ew3Wn>^?c_2@4 zWo}APMMofZWpZb@M$6f6T^!;W=$e_zamqjybX1k^e%wPr8Cw&Gdin+!j@!`lV zc|B{xXo6I!l!KErkEZav7mRAWYF-Jw@`rL2S;n9P9wv(D`i1Ze+A_}V z4(NLI7%Na>1d5%AzD{~5VDUc-52Cc%{9zrL;T$E3c>|Ty5xpwj(+2F2k?Ey0p8DlP zGOz!DxD>q+vaLAdIx=4%i7YeIttNS7X?zf^2tUBh)61`)i4kMMIB*So4A&_`)+fmB z5Z?0dRyxq++iH9MecaxZSN0){%Bib|@6JCV9g))Dxyd4&FD6x&z_3{;{%P7gG=&T| z<1e`B4zqiRFzSw3kEV$p@S%ejxW^Abajt2k1QxA@yDBv1JP*8Qn~#dz#dj3+Rpc`~ zSz1{^l8$2Klq&Y@S*rD{YsG_X7&p~w{=VU_+T8M#VRJ|LF++Zl zH4Oh0ViU??M(u7997Zsohl2kHm;q?J9oon@@Xwu;3Y z`N9_Hr=^xc_+C^9RLq6>69?sw6sR4V<`s$|y5}L*RqTM8S~RktTnjNNHUE;(FTjs9ymt7lUJnoF^O>E7LQU^&%iHrYMr3l=6v3U{c+`z zO95oK?d_&SEzYEESkO`M37P#eA;TX?_2nX;{%j$jY_}hg@ouFl`=k~imT6cRrA;I- zImc^{;=SM0OfNLXide0QL_b6GjXiI6uJNv?>Utk7>DYx#Y`5Ut8V#9J;P7Ja23f6%boG1 zbt={T3tdf39oeUYqZwEQd#VaE8iA> z4~hR0!c$Q2^hlsj7B^1aYQVrqQMS8;VQ~))z3mx-#(cSQ02;0|>5SDx2hdA1Qi}g% z;-J2A>R17pgI*GCfjzidm}muw{v-&IAgy@Ar`MySkd$S3ARSGj(Hf_}?Q7zVIq?}- zDF%*X`2*76MC_@J5LdTLIOxVpiscePVoG6b?L=Cr)7}Rp#0(;DE#`1Kl%l#anYks8ST|Udj>(lYyegsTH>aGMW0Z>#w)R6F5kBCtl(`HQh6MZ9?kUc1BjKpyS-;PE;jpl}F zYLZ0KMZjchmruGh6e>I6KCD~xGwni4X@Nw<&Vw>c5dwPsjnx`*bszrnJ)i+(HZ)lvwG~neWx~|=69c&b2jA{eSceS zEC}})01C_}a_@yyabyMk0k`I;WxJEqZRsQez}g6#eZW1FZWMDF03B227o}qL2OUf} zNULy?j7vVrD%7@J8V?d&6Wg@M1cq$7Q_Ng5UJ{DxRxcwe>Z8}@rO7`)2u}|T0KjM3 z;I3mkh8}9r#7;XZtF&8T4iOuAEi{7^azGwoh-nm@y~7xP`-mX%?ifSk+);9zDbue69DmV-D5OJ;M-h?imBOxMB_i+%g9q;ecf5ngl9W?2Zy19&B77n_Q4VxIsV9bYTc%Sj~tX*!SgnAPY_-ns5=8IAAF zvEgUdH)KTQbLhB`rEm?Ai(50WZaRd;?M+o6-*qcR;b zA{$sO9&pdtbTBn=vLV4k&z?if-kU3(F5BC~01=7v%XeI@s(|!%K1n2-fI==MZ z?f&x6LxLlQF=9kiObQbeHK$z(=WDm?&%ei2QCgvHuM%b8lzp{w9Uue*7&0^zI!1^w zB0Ka>-N-8V!qc>dWn`bN5FD%r<;DE~AK@K=BGoI6-&>lY%$o7Q_EI1SxEBtC3822v^uv4;^!62NXOcNGj zDv5z>3hrSK*3It>8vz?f+SJ8Nl=f~cz=r=)RFTglAYLK>HTX_=GqI_uS@3ih7_c+z z`{z_|4BiQkJs_tVgJH*WkG1NDEjAeSEQ*d5?$DGK1LPD-`41+~A+)VdZ}M&m028g< zJQq3;%Y1*N3m@n#{TP4R+h8R*hj3$Qi>&Ve6vsNxVpufMrVzTFvW6GC2G2uni<-Eb zV)CC9yG=ZRZyX;iXyx~s)8O$Sbj!7007o`N#Mf`kjAUG z82iiD(S;f8-4D|fZ-Oq9mjDeB)!+T#sk>6!lzCm*igEx#tKNjQ`+!_ShX^=>00FkU z&XpUX+n5nNt`bE(hXm9!HiI0&EOEJo(OHhBLUaH!*(?a!!{;0j<+$;HQ~pfWSH_|a z`I}`VehJYLpOh;iTw1NPwKf|DE+uD(8|;b4J_$G*?C`@y=vwizt2kaz3OC{4!3F3= zCyo)Eq7+-Ze~~_6>Yha)@EFVSR&pm0(*aFJa9@VL!VeuRmw%hdOY( z`)uN09Y4ZpQvr-f$^p1rhxPJMpy4QKCsaW4Jr*wr0A4j0yc1>dMDe@a@#a3>0H(Qp zRbMs=UFUY#U^ybx^P+7k!WA7txf3(^gbDCNFQI9u={c;uyqZH@`mJE~Q>0Y&oJxj) z01ehM<$qp&PiCuPE@9QY>#=rO-lrq|p{u?ycL6#b=7Qs`F?g71Tye54<1X3W@N~dpBz!+OMFcdy<`@xv|lZV(4@Vn4N=0h*|k$$irDwvnL4) zMIe^ER(%k3ekX8XPx)+i)5(5mt;S2xI|rFbycnHr*T(PF&Fn{@uOh3tAe*FCN1i>- zRKD=BEo|r1Q_&aqHRP~j@H&igILItJ=JdY?z1;f3Gx zACcFw%n1#1gj$<4E2hklVGhRBSJNZ0x~9wCd(7cv zOt#@$xBqEJ$5aaz4^~w7IzWthmmwl*Ww54uC=W%yP}P5HdI|g%jgEA}c<-0A`br{c=<=$Ru(!+^PI{ARJsI>HH*!rF``7OGX~Px9 z&jn&VG&L-~oQEe+-_!erBytq^?`0OITl%h6SjoC5~YsqKHaKi)5~pN9&in6Pyx<;J!t_jx*@L^8vGIdzyZIvX6MnwT|Mg`K{YB7 zgNtz*owVF(7`Tu^xg2{Ln4=4-wS6N~4HUPSJupLMpdO@!h^WbrX{9@zAih@w+<@mE zLg)}eH;V3>E-c6-=W>e=9{ApK4c>>Y2AaEVbI-T!Me4X*>KR4?EGXwUM?$z$OV;=50gd0%k$i53pJqysQcHm zm)Wn}rR9qnWm46RIDw7V`0tP?=dV9iDfl)4d<>E{DByso{fae^o`N@*1_s71=w6ac zW&6TT_8pfxu8_8&vB7h#2J%8=;=1~=z+5@0b+@ zmS4Z-V>vq!!mb|W*0#yUR0d*qBG2!MO!gzk=9@r9_Vv|IH#9GKFB*1w?&5s|x{B~^ zfZcURn7DLJHF}{wm$Jmg6B0A7-P!YOuwRAK3aAnUBz3LIx7vdiGR1NzmOjnMc`4v1 z>%2YWf&Fl^70KHTPNbISlgVC3#nHrN__Rd0rN9eBudv7k^b8MP18Qg)mZ)3WDgmUu z1k!U6x_n2pq+hKQ=jg<~0*d2>@#D0*VX6l%KM+V)w59!^RcqS5j+2uemHCuE?hAnn z5yNF=1(%H2D)u1f)Vd8}3$z4Wy&k~;z}%QLe2HSnkHIAmDT3>iZ)U1u1w2VV2kW-X zfrZ!S>z>8GX_-8J7?bY@i8g~SILDy+WSyw}d0hlICtF-j6=b^kef1-SblZk*{`%lQ zPNTaxS*(raG(A8;?Gt9MNm+%WOx)W>Vw$d0gwy?$u*j;91%^gHtZCsn{8IsGhZsDf z7gf5iENnT9r3@r}BUG!M=sWx*fou#ejkjF5=c-QU&#PJx4^xbw&o?dMu}t+BD&_BfIQ6B`DyMW@kVOg(s)O z^Oz1Ed6d#v(AKQg2q2!RA~nTVzIpQ~%poF6*Kq}g(Sk0$jTezgo&wVdnLrWeMR;ZG zt}gGOGx7y*(h_=`=lt}Q2LUl%yOVY1g9w)VeD?lGufudQzSIpx)MU9uo#ZMwl*DNP z7kkM+v6frP02ki%r{dd5{%hf@Ah_93ww-u(i-xd-LJX<{@ULZXiCG&?0yPbIg4~gP zr7QjGPn6s$l1PZ&nQk>&Uc7+HAM(XNi&Pm|JZth5uUg-cEL>yRYB@P>uP8g1n$NPh z=emq4f#fW-l|j8lu)zgk23@bg^u3k=?v?}_j`ARiBO~rjcV44^%kh_r*wdfN2KN;* z=uIL0RvWm69M&xMI6j<*&#cB4xUZ?0FyxPM^ zfKMgO@DY63uI;&{J%a_#aP}$%V#n&T41}e9mg7cj9oSOvC&Behn-WA}a247^y*=9-xZ1@P*7(z3s%oL5g}3cNKphU1_& z(oMr>1x;ZA2AqpB>HapM828satyFLT2ZSjxulw&dc3#=2rn9Rw2PaU~x@T|~QBWU! zf!009QoO<~e9X{qLKVP@0GLpbw~`K?2%JfRNjLT)aOQ7|Iu;_iN?rMxAT7{947x0H zRAVyrv3twogUW0O^DF6Tf004jh004jh0000000000004jh004jh00000000000S5sA z|55-900IC600II60RjJ9017}rNJvmfNKpS|03`qcLvm$dbY)~9VtF7>aAj^vP(?=| zc4cyNX>V>IGA=bPH2;tQ3IG5A03VhB0soWdt9XxA^`H_qa4N*j@zSF8*XVa7*AsI@ zfz=|UCbeCD2N?HDBnui2C5Fk81}KLf#5H`nFgg)*#`d4=G{2$xn}o{DG2=*byY6B^ zo%=0-vc{K;sz6P>P%^lU^C~S<0?xxuHr|z0m>pS6b?R)i?uDV0Em)H3aNn&MEWJSitH};u@ zxwW22@*z#yS+~GF##yzw8uQrWiT#Xfz>hpar{IZ00GDKxS3Kj8(r7vj>%1|b>Z*y~OZ2%e;f>W2>^2Vl{jFDC>HRkdCldMZ{a0H}gaRa6 z`YrbqOgrk&-C%Y6pk-EMI2XSyKMkRguLG4>e|oTd65rH&A5`SJ!&QBC@|qt6o+~~U zz13^rr7=)GX1vP5%=0cf7oe>DXK*oq9d7*RQQA9x$lpwm@fUwTjjstSzvUeusBL7Q zDOwt?ubyop6y0Cy<>l$t0*geEg%g&L4Y_l2n=TE|)jK@kLU9G(W3Gk7-I;$sy@4F6=z*-})H7k}48 zNYHR7JU$M;?IVlXcqoHy6KSkq{jpi%;dzsN0&*lCbEj;#XT7D1He)}tQ{a|6X!OxP zGSBWx7Xa;u|3_;u)E$LQ1e8n#ty4!2L?FMzH2WUQ)NMspQD;Bw!x_eJ*z0;6<>{e= z^!V+l?5U%^)gO6QGggVkncH-s8g#i8WepcFRx}x-$R_{NGL8stY3bTq}SPM!8xNb2ftp9MbIbxVq9g+G~|A=2fvh^>C~6 z^Iy%+MyqApI63&?)i!0_AdJES9!uT+^{s1oF7~l14SG+35|EmV-)bIst631DCx;iB zt;nOgzg*^xPJSl}@9&gcw}9-)f?w~)Ex z0C&3b?^B@T;4dfUnl7mUZtOVcZ=PTOP#cAa+w5eVdK4BLl>9!@8=weU4NF!4wp+~? zqi7orW)#5tl6Y_|qurFcOwhFi&zIT&C+G?RnkX4jfC=cgr7%FwwwYPUN<9bD@#oX* zJieA>Tf-1B@)e|X2+AKGD1KXlREjP!q5nad?YLwM3y6-3y>8YIb~MaH?vZJPIRSu zW;Ft#-cM*aE)@?Xe=VYkLD}JB65>zE_M(gEG2Q|J;c^3-A<<27($W(>fO(q4gYOyh z5*CN$l5<~|?}@d(ok?cz>m%psZSvsvaDNER69y`87VPS1+^nq%kLnQM_vOCf^bVMq zeB<%OvJpZYm#H$2m;QLBq;7IGWP+s)5T7*Q7WO7$5!Q4WnS?!Nd>B}Oo9q4*e&scB zl|2dL!wDz1qQ=Gw7T2#udQg?}z;pRZZ~R!E{j{AOwCCf%{*a1wE?1{hDiR3P45LHs z>W@ZOTL}tspti|Sp%@3=MVCjth!M)$-kL5z2A7R+V50bK&>9{@SDjJv8Zj{N27aZ7 zy)IE%cWgRcsH@w5K>_Qly9J!JjGSO=TDSqn*g_~sd#yr5@{v<$$OI(i54r>~E$6kzUD~rs>ZK1qK98@$X z1<*i^gP!&TrDoE zU z!#NIZU7IyXp_0wC#n)=6nj#9g!aSF`OL;8S)Fj6 zI1XMIhl_4B9wH0_%yUA&U|?UXdXez%IVLlXQeQdGwpOyD$r=DZz`%Gd-LclWnAvH} zjK8)`1a9`C6r<|eY9IMmt1}fA1s7?)E?X4x*z6Hg`hlcRZC+9sV0t4B&$4@=kLmG3 zbr|Mmgkcgy~szdbL{1F;AYT4JYN^x4L-Tx z;z^G^4uqK9QW|n~AO7)G(N7{(>&z1;^?QG^k+AJ2mZvY`B9*5UCE<^Cr~H;dNkgquj0r^5@L+$NG7coNP|9LEs5Q z09HV(bs+`ngN{LC*o2q7PE(4)t7cUjgAbyTY)c-Q*K)O5-iiR{vj8gQVN*>%u_IAs zUd6?5|1{o!5?uV-j}^0AA$-U_IYysqMK)K?3Rw5qSGfq#JME~p7)-Lp0vVS~@g~rn ziGXsG?U@{ZTgY4#9s^;_?U~whcEvqxZF3kZB>Eaf-=UT`_)TvmmV+{7B6v05yBG5n zqn9vizUisR5oWvS2;P1{$VE=)PI$>PE$_BtlVNV{9G>NVirLn;f<_*;EuW&oji~yr`F%v z8$)+Tv~n2?cA&JX^+zPR(Ro!rs8tH)>u;(F1*EhxHz|0Nl$@#JZYJVRA>~gL$MT<* z981c+QGjWa++-k||4mwX{^fd%!B0V1Y>uMdLWvk*smIOnZ*`R|1(! z4mAEjO^0AKswYg{&m6IGvl@0F=Tzwj>sEV*emTr`1`Vy=!0n_mpuhOoLV6x=G$z0- zPxwaZ6=ncY=xG=CHRP<6GOWB)x_JLcduFe8yrnov%A^~cwzJ-5y3;@kROCO zbJ+pH$8#9^s5V;|LqaS##)NXJ;P$YawV~;N9Dw{+#w!|vso#)W11W+S0fD|-nt@-b z3E#lFS=L$wz0^40_lhM%tY$g-V8?vwMwB;j z)XE$wwSzIpe<8VD@QERFk->8AYpk`H_nl__Xytb{HMcV7ocz*@;_ADQW)i}fRT%ip z!=k1>w~Fl6kVaV2tEG6$M>CL2X_;H9|)cn^y@a@=d^IwrrVK<}SEs70-S z1JXB$k#eA)(QE;Jo&S5Av{~%?r#MoMc=|8ewJI<#;IpkvxsY^ddsuf2((weYP-9M` zmt{C@k2xFN-83Z)Wx#o8xemMNnzxQnP}FjQ?jMXNHxI8+<3H7%wRJKFE(NuV!^el@=C3hc6Xa2O-WJCy`XMCR z7vpo3vYa>!#W?#(>jpNaW=6yoxz}ox;_)6!XhYR69}}vCzX-r)MLH+JS97E^2vzEO zZlNhqa|tJvWZ74WHhmcDmyH;AH+gucNvrlmro{wRQYJW^9MYw8+f!@QRj7`w29y3M zQ{pAA!m6F#Ho+RWZMC%@bY$s_mhVdKEFdQO=TKI>A&3ap3576mD0OsqgP-fh*MS2w z22KUQvywO6u@Xx;mxNpdp49fbNS!r(3R{eASLIaqxI=$V1lmI`i>tluR$Xf#RIMaH zE8s3kp4gpeP9W>E2*@iyf>ZHc1?XV|TEK|(Q{;Z|W*TjebK{ocEyvhdroxr0L~^Iu z2HN8I-sWF1`(+KDqTd%YA25$P0cqhO`*ekw*|52PoG$Rrfk+HOtwh96?xbSP&nV?D zsWnB=vWj^<4~c5y%sAC4$hTSVeOW^g>#p(L{RXz-9LIAGg>pcl_;Z)2(5!26T!BX( zb3-aY%TW>dzpy#pU6}oxj;9kf{)+nGNb{1L+`uCZw7NwQ{iH`U131c+^~~+ zFH5mM9<_k6J!0Lfh9H1P&Ddr^Wcl&2BhpsI(cf}V5J;&{x`2L^v14Q-BY5?zn(#~} z+6(F3ONi)Nu?2G3pg=j?$)zzQi6`zkl1N8#N>Q?yeR(>Ux4NiG`25+-@zaXhhQ4H7=!a8Q!ymC92Ab zQXw}T%6g6b3SR+>ej1zlYtfH}IbTJ;0Q;jJjeY^4zW}`NguCNDjCwNRW>M2FmwKhq zf5A}WUYYo7(U(SD82Dl0hfcbG)Bc|MKc{^=>F-XudDG7*bo0smJL%w^n=$kh8B=2E zPZ=*4(!qVc0k3uW_&D$LS>hp-)rcULd7SIh^KQ2p{Uju(W)f7h`X*(SlKHy9>YWpS z(gh%bFzI5G4sxiIWQ~$LCk`Cm=a+!J(eUo@zF|emroyFV@GZl1;s*QppH>nay|pJ} z&~eJ|?DV=3HzV2~;0$uB)ah+#Sa-$o=s+$xeY(WP(2t>_%a^efil6B1?Z0!_6Qk}k z<<-;_SF>M&wU$t3*3qzyhMHQYS7}H#|9^--Y_=%*TL;eHejd4 zl`WUpl~e@f5&97II5`zbxUm#nXnn>Pce0Ae&He7%sI*12YPpkmmONx-)_a<;7M)bNn%x?b1gf=<7j8POnxY1XcK9%$j3~|nAG|_;9=3MEy25Wcu3Wctq<=_ zX;i1F5`@X9E|7MM9s)g4`=q#UAgOx^1?vITf~_*Wu;W9 zmnxk*i&*q#9@;qp#?R5=c+x2MhcvbD2eb%DXzan zt4KXedZt*+muq~Dd9vA%NtnTBW}*8g>lCyBD3~4ac39ZUvVFuEP%X8fF%5)(^$JC4ah&P+l*hk8%-vLc1*AiMJclWo(LOMz}mylDpBb zE?9o-b;_fm9s*?^lF{`Z5y>Aj<)>nJJRiPEhcSj9?Lif*SFnt*;6>6)50u0~^oBL2 z>(>z+fngv%+>PE|F_Y&0RD9zTRa55c;I+RSYOwL=lGM>_X!*_J6pnWY22q*NH?}f{ zlYpAs1^}XS)?i9ZXC=Bp(f*y{J!f^}@~g$>AZT&?qdMfYb;3117jMm@azCF(KT0DL zSq60)ubGJ}QUp7tV{|W3Fn(^XEO@zFH->04U!O&fBBN=g~YM z>K)$?Q5Epsq4;0&zfqER?J29AL9!}Tre(C+{*`-LsHI|PAdw5BMdPHob#i;3&um{w z;6z4QGyHWd$IT=PRR>a1THZoScSY*}$e1)W;&&8y0HdJi7l8MKJlrbZzkTMKNAqb9 zw81^DW;)%u&;@LpwW&Pq)KA2SJf*Pfw3rTx49jz_jQT_&c-pdRXqv#Yv{0Hpd0G+V z4=kMn7{Vb9+cvM4@d1r|UiH{4-6h=Vw%Z zR~44vFbehUkrB|&I2I8K>#-D7cnCjht!Xgb0#jtw zs^OTC234Ux(LVfyZk90wg%c$p%@3Px32Le}@IdegGu>1GpG>#Q*RHXhI8dQjjQTM~ zf75QZ8=oiRf+4?}*Dwj%iU8S3?0(H&iR{<-_on8b7YgAgd6K|8l;nKzx z(+y3TL=M&(Jh`RvcCpBR&Q&fNZ|5)ISWM1ckbx(X!&Pn7HTtk~5sTxst9XhMTpGj1 zOF+_JAv|i0yBlRpDn1tOrO@LYg-b(F;- zLRfn`-9M8?Z-b4_CS(rvUIp_u>pK<%x+Ryn{FvMbyhnez#3eq}6+0=5J0tNFsiwYl z;m8#Q&F20olxOH*fewjzb6IaRM*5yOu? zoHeMNqsrG7O*f-$ReY%&4x4WrG!C-tkf&@;Y|Q&tFi8@=9^YUMtqY|CYCe(YR^T4* za5N{MUn%K~$*LT4zh$0Tn>Q5&U}ne50{<1p(L_R#@@OdkS#_-*Yu_ENTc$!Ao7c&q z7^J@tR_wu!D`xCWs`m394?5Sephjd|2QNRRyBV4S8p}4eRr{!n59`F3YAU(;dK+4< z1~xyE!Yw2!X#PFVI<@2@E)C+}$uQE!?+zACXF9P5_d9fb(?UBA79*L>Y#GPh8MGtR?4&rW70G0VA$Y@aMBji@o+zCVF1iPL~hR{48V?rpNJZ z`LG!l0b4?PwsiB?;>WLMIt56TIidA|_V|X6qUfQTYVRlF_JXAUUG>DlYHekSyRMwU z(b}EKL?!0VCel02Y8Sr^Q=xZoHhf_vIAka?LZcxMO*oMLewX~d(2UMLcywG}6fHp)id6sV9)eWIDX`z!1=* zBDIf7qE*bzzQu3xyY8Y(>CKJh0Y&*OXVgs&uEDYQSY0%H8lzH&J3=cM#~VmcfU?r( z7gq`vi*}-rTZ}SPl9ybmGnXw~we7a%;3#62&ej5R{lFT&?S)W+z}AdSoR)OZ0%x)W zk4Y1R20f&^wDfk{pN9ofZoi`E=xAsRl8REH)$Zj9^l%Dip=bPJb1GOjc>v0nNIZo{ z7vCx!@A*xIe=Ao4>uF&CA&+IjrwH#Cmu>Il+=;p($zyrl;P!1Cvi}wG*iGc)pD_?Q ziJE{wQeKBlORr!d0HbpgAbt>Q+X!BJ?Ef@f9EcA@_y1Z{nA9|9{!@p0PCl0Q2EtRf ssO5mY!_r7FRugOa30*H!tW3$zp(sT4&>g&q!7-G+90uXDoP`np+1y#e&Hw-a literal 0 HcmV?d00001 diff --git a/crates/j2k-native/fixtures/multi_segment/bypass_rgb_97.j2k b/crates/j2k-native/fixtures/multi_segment/bypass_rgb_97.j2k new file mode 100644 index 0000000000000000000000000000000000000000..6ac6f243806ed5f3e53cdb99ff0a184f61f1a4c6 GIT binary patch literal 6137 zcmVF8}}l004jh004jh0000000000004jh004jh00000000000|x;C z2LS;G0RjI~01N;E00RL60s;X5|6BkOLSIr)1yBW0M_1EV(^q2uWB?@q0Yh?SVRU6= zAYyqSPjF>!N>D{dAa-SPb7^mGATlmBE;Rp;015yA000+!00IA#;?(^jsI$Nl4<@KB zPAUBsM!@l1?%usV02WQ@bshAnydoqD&d;}{mf<@z=ZULNlo`;~f-i!!$iV=kv6{Hy zq}qUY@TlL-UZhri)K(b9ygv{=-->9KEm=VW(A9WfDF7Aq_Vbp8_oj)61TfGG143Jx z5#=AwL~krz;|@wemWOfsM~GCw5+8J#=;}m_Q5AAli;%Y3?b%Eay8N<6Y)AEU=B>Mj zS+LtEf4G(b>huOL4S9-(^OW?*IiFd3;r+J0hN!C6s1haHD4V*7KERfuRqfM)?FSO? z8sEuKa_1y~U6BVq$b4(CFQwJ#M!QT8fgc%QsQV|B2;WV@Ifd-|oV#5+p0M-OD zQCmjS_<$Qm_&(v77O`9Q zmb&kIHw`X@08{4`#FA_mk_ZFj-^^$7YPh6O%(s$B61vVlgS}hmY1n?*FVzmlKfbti z)3}xW)1pXZ0%htU9{*}}xoAveXM*nA-4MTp<-oiBND!98A_yb^pjB@4p0-nxDXi^o zKEB_9nTAin3DManHFR7)e7Uy9eu*SUB9^8roit4AA$s}(Utz)J44gWe+t69r6Tc=2 zz$WBqQcf6CwpeHzS=n19n%8#p$D9BL5xbry&*LwNdr3xI-qB-`?LbLJg1EvP(N-)J z_)lHVGm6$}2=~>h+R+tWt74a)%$1DV)n?)M;By@;&FqU+&~KgcTs^b5G%Ye~#zVIFwv!F1~u2&1N>O2*>^zqWu6{!6M z(E-CkPTjh%DZv+ z@&VJpSH;4o6T@;Z0)kl|I)a^|{@ygLjEpxc+e{$>seXbc0%K(G8ZCd(p*41(bXq0J zCvGNs^QEmB*o`i`y2Bv$-waA(6^_gBE*o@FARRSbKZT^VGaqOsIxm+LG4{bf5DTPw)@2|tZ@>?L6(erm5+aWOabsP(KB#^y5D-RHHdYq?7JT2jpf>76J8D7XM)cV5S5gz6jM{ z(w?*~vly>*aSQ=q8Q1p{eWr}o3YemA6U+c&YuJv9h`cW$KmtKUb2Pml1$uPsgK~7p6&RO5)wR!9Ie4>loDGu%utmXMvV$m>Z9RS17F6?K;`(A zMAhK8hk~x=qX|V`&v!lpm8y_Jgaw4x0AhNFmV_c0J0)gQ>m;>(5rW)GFb*DjOfyk8 zCa=?ObupXz34B6407`5e==jU+81|-qQscfI?M6|`_>XElHTI9LiS0Lv@xDJt$NaAy zlT$B$vT$+t)BD!lxIG zw6)GzKu`==s+(j%=85|>^8i07Ec8Tb6}6^(^{&N^5Xt+jS$6^>rlOp<)%<@vRV_q9 zUX{IMRrdOF-^FRxIpCWs(uvuEkSuN+nTj;OhGQ{=+=YSp)67~Ca)}c05hr5xKzJm( zV{Z&-NWLCJz=w(ikXN)$?V<2Nv0vUEh&cRjU*Nk=IBe=u+=r zDdO)r2vLz~Ld4f`i^U&XeNRC&K6g_#H+@)~Q0kEpG*&^XY-+!}ZmF7@1Sp^nR?U5G zC|x%6X&sSeu=7f*Xb|Q^9j|(gVwO6)Xm_GP~_AxeDjFPmeax9+PXn4l|?* z815Fo#w`a>q6O1v$2@;s%qxzM8~_@&e*P%_mG%4~A=ij>DCs@&fDfM~jz^=Px5F z>Y4HYGLbhjk6X;cu{#wU^c^*2>F8u*xEx%$ouxZxZ^<^b`-2-?jgF$+7nQ-66qvq7 zp3$DuS3u@_O;pzXrn(OlJ*Vc6+KYSvX|0`MY?VQf@!)|Mesw6Zk0jubXLn; z-78)*`ftm#ViaDNkxw59^EqWmP#CQxk5TnL>!gLV`6+caw6>3Si9%*_Qr#`@AKA&k zAxtC#nwWW5q)uf{7>e)!$z?u{-!J4%Um&zEp)Zwe&9~_WV*!Sc62j%WGcU73%l4aQ zn7R_jmJNDvI(M%nT3OlP;0U>Qpm$lBfhMJYS~~m2RhzGZL61tRv6mP@mITL2zs8MA z@ZYUPFa%{}KX`y<#PO8rT41gw;=4NUU;-J{vD+LI*$&yro9Q{$*v$k znz>jj9Aq_fQEIM1Rn91ao@xI-?E6V<9@7nA7Tel7-vz!aq+1rt+7jAY-qQy(RaLWU zfQ<`qHpL$@Id;CAVA^~YeFD~-FPCjR^;+pXi7!>W0KDhAkjgI_CHfq0jh-R3`6AHnEU@Lnne@Z&+{_ICVY|AhdJ9?lOc*gOfd;`)bpS@O) zCe*{`RBF(qrGyW^H>&Kl85{vz{StcV{?de;__GnI;UK*!xV2FwEHsaKOgW#Xp4J7Y z5=?=!NGE}|MTU*G)<_5snBxqB^&jHafdZz1unZxk$L-z~J}%{K1PeV;A%N>zQ%*L> z?(9c2pCXuXydK^KX!D2eGQ726J}RzUvcwF}4@am$d~RdDOT?lYYA-#afpp|KheO@7 zh;;HyaI^6Ndb3Yz@-fEVloGzMy7tM047lo%nz&0ldcsE;1qVyXeMan$$lX=`h)P4G z?+@H&ZFBuC!9vbxtB{l6NR-U0j9h2E8f7QBBF09WLQDqC2nSDQ0Fts-xqEM0?ZW?5 za^f8m(f%^?=oB>F)@4o>#G3yC$7kMC&RQu(zO@lv)vSU+5Tk$b7O`g zZfQevuHiEeYr%e`*;E({#P~c8V8tad{|Amp2<1ilraMR2=j-(W;qMjfr@1kJidmn{ z2iQZX&`)=810}dyqj|+qZ&#Dg?TH{ec#i57O($5wgz+v^XzF4nfm+2&}mk5Ih7LYjmIuO z6h#ht5HX7)Q?55=F^gRT9{61Z2TF0L1)VJV9$p+u{a8H_x)XvCs*(#lMBibdd zk>TFa9?-VwcR{ZL*GKqY7<&I&EA0sJU=eyTTLP*Av>MHQonX9#6o1=dz4EalnOJcF ze~&FO)76M`fWVbf!vL4N7G;GEJkAD3AMvs@x(~U|Vk=7E2gp?@7e5E5-1ZtL4+P#^ z^z8?PBy0edX)dVi0A~MWO=28~%ot?lfGA)8Di)s_TFBb2S-RpE4mBlL|I+7D13TP}=lWy_XQ$0Oe$$wR60N!Zb`cp%uf#u7^TL8+C zm2TqYN1=u4uW)Boa8L1(Exn89mS=lfpl-#>19&|I2gHN0I94J>tu&nF@y-?8@RLUo zFRUaf`%r~ShnHg@{g(8nnUHxy-Qp8*@j4c0n$kt~)2JURhmiM*)+fBy zv)WqRe$e|zZ&vrT9?~sO+7sGOXuYEzzqj{6%RbV2`qZ|4p!S6JjJGSGwf&*?gpHtm zM6@ieO$4=VtZ-||+Jz%(@NIe9Lft26reejJ7L3C>BHH)!FEvRQ9n=ntqf^F_OhgMA zVHur)SLcv9l3x7ZU}Rx#4>4R?OCTldw`?Fh_`93+AV*7`NE`-NhkV%-BQUcQC4SE& z6;w7$bjux-R(&hbZ!{rTa722PHt(_|G0O46zx>}G{2Ynn0WT{1_o^j}?uZGp@dKOe z+xsQ>6$CZht_NR4QGpN>O1YO7*z=i==V=JAahJgMH<{Ws($jKi*GsF}oTgg}i)kSgwIZ9?t0 zn83u&XHXcZqF-=O&!%z;l@(qRYu*02jzU#0U7nU@%(ces#Zto+ED`@U>vuGt!F2dP zBA!B}zg!O!_YMB8M^_CZI) zw@n2w9$yruv><>Z1Yw*>M9UZ<)FJM|GNUfhxA^%490P!0>!qoRdC(Iv2RbWA;v{^S z|0PQFy1CAY2afq;7p&wk;vBz1a<-l1IxIVH2NE9Jrd)_UzAIWVlOaq`J?<)1Ib|iN zBp$ku*5(uWk8G{ax)Zbx<~g^f|3syU*ubD7=&7yK&1?~hHY3%X zZFsN9;p{&xhxpZ^t~AQ`mL~;?GB`HBZU;B+*?)xW1tjaw9Yf9L1QH;$zT~XKuddgo za`^7ONQIM!DPt-RW!2^#6~}XDdj_(;V1ml(#|Wg&ni+8Np)6dO7M_k%>+Axg@TxG# zse)ElEo~KxXr*(3=p2HQk4OGvNx3OI4}3lk(aR!diATXer05|^Nhv!q1Nj{HK@znf ztLQ}p!1^fug&leEp_jG*&&5RwgOq*Rqc)(USM|c(3?`sN0Y#PLUYlDlEzCO%I$>Axa5%)(8RyG)ypeo zAhzXon*d<=Np!_aKy0Ac5q#OF){%gY=&WG$4b7<}+r=Fv&VsU+=%Hqm>;CM(=kLP4 z1hncbZ@}h3nOhd*JNX?|22VAAaIgk_#em2qTA2?fQ!bD;^6@u-F$bK@UPBDS0@pUd z17m?S%VfOyc)v92L6}KHj)-cVBKF3MmJPxZDls;H0bP!0(2Pt5C%_aS2(zt-^v=I2 z+~Q8SIXR~{gKi@xSbcCXflm_q&;F^i9-SHS>S#U~z zQN5z^Zhy5N&|AMG_KVtIX=|bOnEB5Zdr$dO+A*8juW4Z?9o? zc=k_nXv)slMy~dIT_pL_a87sB=6vp7>Xn`Ao`VxA%efI2@U9(YFBmFIT(cq=lLxEK zNMtf-4Bh*M#1tTI_|kAzIkWnT^+l}@e72rF9`LUset^Woi1dxEk!sVCQszyX z#8CCRR{tpIfxYI-fSrqxjC-{E|7_md3wD(J0)7NT>FUVc@RohcD<;2%sD8rW$K*Xn3#g-VTIPfYEY--oQ-dZ)> z2#0RVQQJW=QpKW`5HFn7Icj!}p^Ky>2b@-4jUhx1(|p4|@i8;>u1VtBscgCCp1UO~hz}fHP3@H&*_YW?Ef&g2LUFw1av9i z*cp0wp{b3iUH;|46uoI0a$2ey za4MI?SUjiPoZE$4fga2+L=_zFp!&+{OuMrpg2thfB>Q{ge!~T4+?T|ECMA6mGSSlo z0VK|?^8zPp5skRk8InHTMQ5Q4unIsn75%8f)*z=c+1n~zR}6x!sl1wrpcQQROHCAk z>Q9TmC6OL$m)Z=qX}A2^?UNgT zq==4NSj!g$RN&LNPgE6mEceru;?(4K_UvVmL_+H(S36o!oLq!?X+_}66=}V(vspIl z_Piwz-wSnzPsE8w>)kJl>Qt6$KBWqW@x=j+W(*G8W<8TOOx~#%&hWNIS7{66Cyd0s L2A`@sLoWZ>C`Y~w literal 0 HcmV?d00001 diff --git a/crates/j2k-native/fixtures/multi_segment/bypass_rgb_97_64x64.j2k b/crates/j2k-native/fixtures/multi_segment/bypass_rgb_97_64x64.j2k new file mode 100644 index 0000000000000000000000000000000000000000..10ff6cafc31c8b08b57440eb449e6158f6db1dbf GIT binary patch literal 5253 zcmZu#WmFVQ+uj9f>5>#t8tHDOLAsY*x?JfNmhNs@x*McBq$EUg2>~hTUb;Sfp7+oD zo%gzC&fGQk+~+=X=Epp$JgNcM0Z#z_Xa1G{;{WN$Xh7h<<53+z07L?y0a2bk0|KP9 zEZxJx9t{Am-@DVe9Y3-A6W|j7^4m)FiPowpPn6Y}dZOv0Ie-oTlytHHyI7dhn0nHv zI9NawRHbEU+$@}&Z9YM0xY&5vcpl>bn12f(JO%(Cbpg>&C;bnQ|7-hiAI*~y;~xhG zkoVsJETGO`fgZ?A3Ivh>fmBZlB@h_E@YfoEYU$7Zw2A=(?SMf4r<`iQ02_aSK&~L; z08%6*pfVCtfOCMG%irK9JSKK4yb5f$RDk{uX{Q@f5#;?|QsT!Vw;}~dm z!V;;_y#CqP4Wn-$40Yq!fKJo66_`7SNabQ^+R;e)XGqSm=k5u{IG_`hK<2yVDwU;n ze_9=F>Y43mZA86BHuL=^WET3LN;CoQ4d*wB9_o#uJs*1@J@x>Jhbhj}T`{GD$}}la z7{*>PJ)5gLVnCsi%!y_*!`v|9@!gm6H$0>{{=MO$524n&sgZK{QX(;~HK*n@h1P1~ z)Ygbg!C#>5k|4re{Jwu0^i{(9(Jmt%Avr(k5oVftX2=z%zOGqLbvkwQOQ@wh?9S8S z^Bo7Y$L5_4^z)W+&-obtEwrWzgfkl|2{g29XC~-AD*t4~@2wHU?R_S}f)WRU>K1-% zEk@S6T^( z@N@4~A?6InZ00cziO8kkxl+&w(c&yN<}zboS&hH$uv+W+EW6)BY?lPQ<_ndtUBU}! zr}8+L%H(y%R7WIbAE2&2pY=LP%O7u@w?mdE9ErHpuO^fBn!FaqYS>_HkBq}X5%gS{ z#!}o6$=13rIx(C8s1>0h?#FBf!oJdS(mXUMTiOb*jpP4-*I(jDoQt7vR}(B<3EA?N z-*7}GkQ02d&y@Y=u*s*A?qGyJRda{_VX?qrHkvB8`Qq=L+OVUN2zyaWu}pJ0(p)U& zd_t$1)%ER|!<$WNHVP&hLgxNe%;&RHr7+q9EdOnYIH?!uy&G09NC-O^YboFb&BNzK z>4uh*&72KeV_}=f>~b-tnD{w=YKM-A$#8r9ZPm7Kk|NjQp1mu=X8p*>)bR{tk+Bgi zo3BLFEp(FW$9v}IGkZuKM)$}L>#}QCM!9D6jc!yBrq>x=*rput=U-B$>HJf)=0$n% z8TA>+!oyKyei9Kt)AcArD5jgvmXAlNIvnjaZH zxL~JKSA9p1gg7Myc}DPhE66wQZPGK*bS2SdQ8C2P8Crd>`#h2IrM2KQ!yzaBtcMIq zhdxTYm?2Lft^}@P_d1#8sacOnjc|W@Q5Td(?pyw=(?As-q~rc=e79OR6L}ehV=OVc zyf2!YAJ+ra_IU*nOYctlV`zgAV{k8o`Pa}22!4G7CnSZ>GgGPIVvCe!)7}3dlJ|5n z6KM&ZmmMj9iNF_C1Y4exWf_P{(d!~lGG_8|aXmk^rkf?{&*#;1o*vU;%(~dDNRrWG zMs-i(3&p^D?i|+Hw31uOW|MoCJyx5VZol(D6HN~NYVm;mb%S815{al zDm&Q??5Eq2cD$pbR{cTZ*NF|d(StxXk>j2#%uO9>Aaj-Fe$&ny!ei7GzZBKiaVto40rg##~ zDOE*wd3ZKF;Ep~PkX-tfIYUHP3&LA})l$336>o-jDTgWeTDkC*E$DZwNydA{{*a64 zlYOhV;xE+z%q5i_=(y{d3`fqlkh&t?``b0UXX??_UYq{kVpCkoRtM(mKqgopZ(mAd z$L+}x2rU`ter&UkIF+!#Cqp-hHt`=Y!lLoTMs1cEXe-QKZO`R$!}d|cd>g0gjhbV! z#)7CKHVE>2^vdQ(XeexM-!zxO7iC6ZdMW@0MB6-=q&qn#*LHKzY8t&S%p{ z;~u?ILir<@`+am!c`$cq386*LX64R&Y-#zd=-pDH%is=7^2KY9ZWgv1K3F(Tnu&Pz z-b2ahv?bw`5of|^)I!YEiABIBL$<(reg(}c9%KMcrpMaL^Pa-b6@P1ic(1wi?F+O` zhd1Z6?Ff9=E!@MQnX34aC9{H;vObcB`zm%^A2iSMMBYCi)&tdp-eCSHDvSNB&AQ;G zK}^rr$F7jt=V+Eb^pR(R`1Ke+HhO7zt1CxdducV8Oa!yB*cM#$Mkq}8>CxRe@#3&1 z9GIKBnHtCRroRbKKrsV**)C-04JYd~%B_tcl5yA-#S+p{&y8>e&=0F?R&&X}TeCQz8gvyWqj1LK(HD zo?vPD>CCbA>YKvQEPvcM67+Q)=}Iaz$#Y*PD=VXfYK-#Iqk^4HNS1&E*&rO|NZ}wG zJF>a;VXADX?cE5+%vsND3Geh!<}&?-*Px%PL-vE%4#7b)mVQGP8uK5X&eB6^A=zPc z^0`|o4Ms^OYB|Ujl_YoAj*PBIwc1&Kd9cr|qGwlmda7lMCN~74q>8cg<B=MR{7Z z)@vT4I$MK?(%Luv(Ha8Te3%JdR3n_7GZ21$5}`Nm*e3W+O={NcrmAvv%hQQ*i~tH# zUG}~MgN-uC_C@cinKL@jOwpDT(5~L&{E>i#OkA`4ASHv}XEUZ3hzcWrkbn6BkkOnu zXgm7>Ts;gjeqH4vb`4XtVs5+=ADNMm zx=QPv2PwnN6csg2Qk^R8>6@m4tG>r$>oLMs4@0PgMQ<-5u^ARcr-$PL%3(EYn+vQ1 z8xegk@xf$$Coe=`onchjRd6ZteWh}}O*5~!8&R+wpgh3tr_4V_?PXDw5k3n^jkK6TBMY{ zc!ObCFI~^|H>}U~>(e*_(>y2Sl6{VD4H>rM(t^2(2Hosf5HY!ZQ|}beM<^3ysbjL~ z&wM)=$0QfM2c}8K>ljoFU#I(9SJ5arV=4{{aj0;O`MPKxuV9$-)plMwHPC(SWLbA^ zk0@;9i*nN}vv@|lNEkz?s8@0N4z0{Bjx;xKMnJkC2t3v{uHloXNUyE8%9PfYJh$g{ zd%Uh!#o@qloft>?<+RA!0BER?H~n8&{wMJSY#km-f~3iYxTr;B!TGMLaGG7@s^q7~yR=WBEJ?SW~Vi_1vM zkR?i>-0J5R6?)zpGW2j_tE~NK*-LAyu$w%$*p%hY8;kf&K%>G$cy6RaIp8f*_1DZ@ zcLHQMt|h3ll5U&L_eQ-1Z{E%=()h=rwsVb~YKNcK9s^0hR7liI7HR0qb}Yao`py%}$r(v8H%T^(es*@r*)LoZ9{V#Pl-MWB^MczVEA2(GQ1E+N z*J82$+mup4rTrbN0O>kwNslp3<`ucSgi&=Pix*q=EEgCz=a{Q|%k8Rz?xu3PMOdkM zTzIAw^tHpD76RBhy3jx=UAOr zjN135k2pOE3(RC@X0a%m!r|!RJiW4^`Zq}=8t9SeH3lZ%o)a91;%4!qwcg2*Nq`ND z^KFp&{h^d-I2`_!2VLpj)#Jq=u{$T7kOhMJgIcP4a415}yPz0&++1Yj-JgIjeg4UK zUEU5qreYlxAxjVHz_aUV@!j#v4or5CpZrJ-6v6T5mL@&_Ty(zR%bC71VG&LS^R)IG z{o1HIeu|lm98MijY%2r1jaU)-8HiXO=i#oPg#euq0q-(W7?g=hcZ2KSHg-H<#7|#u zJBT3U0R16c#$pyn)=|MDBBzpCn>>sR0<$Yu)WeN1q)!yF# z)ej4Yl|i23xT+NN7MkCCFX$vb^#mqO=)%E)!Z9w<20RE{5r-GFBQr4cYwgg}khN3! zF1RfA`;&apU#dPR87aI#xMc8n6I=1py)0_2s?_58t*oTT*Mf(OyQ9Ss_4^`@_Dn8N zPcHqE60;>ILz@{dVP8f{(N?<*23~QB%OLoO&vrilvbFuyb|O|~dV+fFcI<+dmWQxl zyN(9&%v(iLV91Ol&zwyhlF~I$Z9wj=@*yK_7%@5QVXV^dT=jLZWW@D-qQ{BEw84er z(N>HImj>Gb8=Ieb8XGE_+{yZeV`=IWbO;88>!# z`fh{gMGwJv1TvmREe2<*u6*a~{k9l8Ym95}2Dlh$%a4TZ_3~F4ovrsAL|^}=96BrY zoTYX>*)j%zsvNbqNF;J~c>0{3FdA%4+*`rBg;ksyB`6!#g~JemY-oway%OZt2khAX zANE_@MOf?>(Z*wi83j9XZ0=af9Lux?lans&5nmQ-OJMpH4-8f$JX+C;gO++bJT6lk z$o==ib{95mN-OTDqyB6Xm@G6C;>BT6IE%CeH1D6|Pkj5@YE41Aa_A*G6=k3X%8pS~ z+8l8h-A!{xwyxKZ!bfqF=hAzw@jD*5$}vbUdiB5qq#`aI((S%>6 zIw@Ca4t*(a4_ImsCF!-UT4T%^OPXiQI_=;J&TfeI7*lZ@`NOAmS?aM%*z!+!V?OhW zX>PWbQL8YeQRNRH>Ev~d9}%c;jvsyH)nrLOsh!0K zpX*r5LLm&L+c40~7U`H)dAT+?kaq#KuI8+jKT#^h~GYuh_8VdlXWYD6* zXrf3OQUozk9|ow6G*LRF*pylSK7C_zcrXA|$s7~L?$l|>D)Ot}nlFNk@^lkta$mjV zCk68eHDi2fU4giCwni=Q?*`8SpUw3z!z=gSQM&CzVrj zRxcXvSY?gaN8(s!7`XZ?Idg+3#Y*C--G>HinsO8O-~iLXD~r*?FOPX2rVkZDA*TDE*W1 zM?~-+coVNecr@>3s+}GWBZ=9#^-Q109o=^(Ui;({jFTU1`bRo)PRMMh6hru|j#-YS mFHgF~{eK%(S#K~i;>LZC?W#Tpbz+xGc4GYfIC#S2{Qm*H0`E!y literal 0 HcmV?d00001 diff --git a/crates/j2k-native/fixtures/multi_segment/bypass_termall_gray_97.j2k b/crates/j2k-native/fixtures/multi_segment/bypass_termall_gray_97.j2k new file mode 100644 index 0000000000000000000000000000000000000000..b3ad14d47ddfa889b5246e8e8cd3a7558f8ea8e6 GIT binary patch literal 5464 zcmV-e6{qU|PybN>DF6Tf004jh004jh0000000000004jh004jh00000000000S5sA z|55-900IC500II61pxnC01-l8Qcwj@1yDy<(^u11V*g|SB>(|Ka%Ew3Wn>^?c_2@4 zWo}APMMofZWpZZCnO5a&$p$PM%w$Se=X3ZeAUZ<`V2Znd>Db3hL3|7FzFZW1!zSSpB?fg zQOJXqBdi|h(ZTkEA5+P}GzD8mh}t{5g(=^G&|v%Dd4ussFdXn||9`|~SD!^QjD)+N zPy%jf;Ua72zZf-btztd_M8Ocx>L;z0@L|8NXQu|p2WvBll2PgzR0M;g){^{9B}eNT zcn1UTcgD5uLY*~OlX^oLGD$1^aJPRL;&TTN>9|Q6pCsZS4F7+$6#p-{Q<}(_qY%0C z|7pxav-qUbr(RMo`7jFm8^<*d3ea0VPbh2&&v7$<01eha?8{Yu=m8=<@jgjnv#r`I z^e;<8YJcKt*G5^D=)v!-g?9HDD;_f8!^($__Jo zh%o0IM=mawBzwXJ3_jwVJaMjRqZX}&yDBq1`6R-ti}92ekAD$rr9+6B=KCC{qj-}q z$K%PBy2H>aeWOMtt4lwBUql*zbM-%e1@#$RT^!JmHS%)L6idxQUv1SzE;5H}%#$^j zAVx$2Up6Ii&ieE@X1s$YN2_IuG|7z-n{l6i7yPW>x)kp@>-1?ws&?P_D;Ce9_oJyQ z1c+yG4K)wz?~m4_#Md>RQsqokkADnlChH0q`ppn6b7Gbp`dg!UPHce8w41P?2Iun# zJR5<7n1P=#g}6TOK?5HW0Os$16vCNHP{7D>NaUVpLYw~yrl~qz2ukTGA%xr7e+0B4 zQm|p5z>R@GmEG{Qfor1_Wq>09Y9-PxdqcFZKg;^}e}4(#bKT5k*&38f_{J#Df2yB(e}4 zmAy$p7BZ;#rodESS$Ynr0MQ@N810=tKtHEhxln$X_FnW;ayXtQ*o@W|kp|zSo4%HV zaerj8LfS`BQ?n=ObtCBWvOAo?jf-Jnww_1x@a=x-J+=QY+G-eCbv^_-4@-Y$A69JB zuOPlXV+JGqdR7))jhN_>Xs~M&c1Yu7py;a|hdNb)hf{|BAm`C@H@^%M>|Azt=6Vs* zU6TyR9=B0B29E6fkRaw&10-E@_RXo=y(L!@=V59Lz&t+R9h!cdGK#9269%TL!pR9U zkrM#;={DjsgT1{c&3-wRvYs zt%Xo8cSiYROcz7~7k$r}zD4^cAnr4kx#+K3of1n5nf6pH3Eu)yGJ7G9yY0Sl`nv?K z>n4lF@l@CbO}P8G6#x-tZt2JvL620^F3X6UJbw#YwmnM5m_L3+xtW*Sw-bPl$CPGA z@^gS^OF!aAW`rVqhGz5rpl#QJmN&wrh537?4<)D}!0AQ=C`_Ap8eCHwtLc3|dT$1y z-rW_$eN4`_9%Ta*3V`y^(Qn$8$llDf3z~IEUbVYIe*hI2Dn)d^df{k7{{{eAf7T5t z6nFqlSi_kMpndg|!-VCc^3~k6m$F+AKM0>8PJR1-+wra5C|30ApmNsflzDEtA8^JV zaW&$dZoR`GbXM+a0$stP+)}X(8 zk?R(vM*^A)`BEtpXv?W|lb}`5jtI4TPu)Gm~$QAAbz8PVH2y6ZWU6Qy;M(m)&q45IV%e>gky8Mz~s`tbdNDuv~_0x>I^N zOE>J`WiKNt>c1D5nm>O9%{$XYuKdZQUwtKYHr7KF0AlrU1@j5kPTIu}3DFUslq(`! zTCKFTHX8;mC1;2m?1{%d2{;?<@WVytgA{lGfVpG0&$bVLaRwdxgb?@KF$c%Erw+LF zZ@t0@eat}F?jXaTaSR>S`nnB+vtatS+XpJ&dxRj@+%gA`aD*HOxI_*;!Vq+Ogb?}e z5JSL!5rl;Ydkl8TRLjaWFqfq$v_F)1pm)S|dM;ixN0Fj{e>&_pe+YnGwbRw)a|;(1 z+8`b3?x|COEOYxH{4~tJoKLTR1URz=IDQ*vSNGUqbcPO~Q2Iba(Kx|v7tD3>XNqxu z|9^i7Cc}jW5)3-0%T!C11Y~AsJ<^4|^Yrp5JM2o`uVh5zbMI-!K)bP8e}4*P#5kj_ zg+PBfZa1Sem3!HlV(hkDr8b*|lpB9K+xUJMX(3US+byFq9Wo*tSS=oK&)9S@HE^;a z!9&lUL(JZroh=%F0|-fAo6S$&shPB_qCI~b7)u{X`g$v%qFzS>5CW^fTOsM4D=l&X2)6Ga6eDnd2NXJ? zQPJ8q-8_276fF`i-)pQsjLSsqpRwD20=Gl4d_>8^#cXmEnRbA96HogXf8&N7GzkM> z|9=)@Dv5z>3hrSK*3It>8vz?f+SJ8Nl=f~cz=r=)Pk$L!NE#Z5koPGNG+QnzTXdc% zXPwtMtENX9gJH*WkG1NDEjAeSEQ*d5?$DGK1LPD-`41+~A+t_zWq*GI1xG;QwF_!P z9n&9m^DY$MW4fhXhj4#oLq^Ge1p_j15aZb+5vJ!fMc@YmzEve|o6|9yky@)iOa5?w ze+vML)jI!3Irom;Mp+wbRyjJKd=!ZnQ-U3X--UmF1;bO~11xz?^3da8xzby~*Gp1V zQ}=UEf-V1le*i|B)qT5?tW7XlvI`l58`LOJ?NO&i!O!l82Y-JB&X)%j_NWHW6wjv? z*mpx7e+F~g+){`c=2==Jf(HgS)5Fb&wFKT?LQ)*{_VKct zj+C6D7*0@(KV2dB$^k^X%|IC(q8LALEk-C@pcHN$7=Aoca+px~@lnRgu2GB^D8&l) z-$ssc2%~ZGmxl!tkC=R;6g@&E-J6O0c&FpVKOQOhMmU_K5MI7hYm@^G$^k{|#Wwwe zjg;QtS(HJz>5au0F3R%fdbXf5byt za#qsYr&}B+=kLj7{ai@23nn8gx2(T4jd|dx14H?Q>!`gjB{)+;YmcDkS! zI$1k-4W`FB^37sfUz|3de+Y7IYSh%AotUG?bo2z9Q~e{!BUlz{9}qt|1{GrxpZ{(a zb5T(cez(uzq8KZj)lAY(UX&Zw=Wuu($bS(yvfkf2XPijPPg(ZQ_Oar`vSly5iv0PE z(O}5DFee0q9vuSGKSn_tPYTP=W%gHpd@{Rl!M?~BtuCY`5kU%UNc>GA{zc3@QNg5m zzdKAKWo#cRfr+<+BNt4JHb&cV7(Oq{_GBX-VnY-Z%F)ULkZCqJSJBHYo~N>~JBO|i zlIkZ-KmIsuT}VtIJrg$&5A2mJm}B~N>VmsItUF$PKWyHTkz!xKS07ct%HI-D;p+wl z|7upLf1ii#EAT(+ql(XioUJ)3mu;(a#4~?pu~}5Vg}(EbxtHSC_yOGl%>wF6+s)0* za0+k06^VMJTCvtA)Wf98FXHbxM!iI)aSMi%9cQkl5>#EtCG^K_Q|`_>->x|Lml&+` zn2A>&)`E<~@`^E>PiDxS5uqQ~AMlhKP20EVF%uT6IpB&%gf&=4TkUp#E^aKtPC!aK zoozyia}C~yudUt;osZ1rFw2uM{|vajt#ApRE22*+51d*tq7Sbit6E8|L3L%2(SIx{ z$6guoNkp&<1{5F)cA`)!O&-`O&PsCKgD3DTtM->TJ$(OCCW8`xVz-BBL;rtXE-Qu05TBuo#h(uQFeyQw zw4qdo3*OF~Fy2|o=$fwAq6_9Tc-&39%|o3Jqkjl|EJ;Jmrs7?f`m`jL**41ma-4M$ za*a*HA{a@nH&{2ep|QbpbS~qkkdb2zP~H7I$ZcEq48qSE#9(b5aQLI-MevOzB_A(F zT~kiKukho~UwZ|;KzUC>PR98*NDa>QOoI=N*>k;2GaD>y__WucsF6Q_5u0?$&NM&9 zNb;#i51Tyt%!6C{u*?rh0_YD5(Ff5t;)CBgBCKyawwKQ z&B%Ev;3(_7J(+wM8`y$?WL=4iFyeoqhDnp;>{wqvrOz%AD@8|{X+33OMn>)Szc}i) zSb*;bl}$IDqUHS4uS9Cs!bfi%nwF7Xo6;`VGY^e zYjSDb;8k(AX#Xt~4^D2YcLZFdg!=-wp_}`I&(IgLHUVVH1~=(>+8w)T%!!gstagqM z+kqA*nwG;nei)MX5z$NU-*bC?#1E5qRvgP4isJL=?M@&C>^o=Wc33Io*q?7YgfX(t z|9`IpUvTw$%z}^ zAQ^kWYmLG^X~zbf@BtU4&j?|0yf1Bgt4yj5xUx9zo4-mQoJseWSL6=hXM!IA)$)7M zcqg7@66&p@Z1mEQocx&1&G1m+;lcB8<$U=f#;vkHXH)*ZPBFL@0w-d^hMOwT!LYsZ zYegFO-oif`sCmtKC1X5r$gwT;%N%t8f?Y%Kl%$K3^Ou+@ulc{78@DVP^?|r;Z%+vz z>(MvDmWHG3*g9CEL0rj3K=d1>EK#3ZTA~%MjtxkT*PQ8%qXf4I3Fxw8BsrRIRqX14 zXyvbG!sgh&{o;)1?HLldAHA)A2ghr<$SJoQfi_99_$UDDNOZDOJLxzxZVZRRXTFTR zcM-h(PnTg?S0}c@^S?efv%WVq!fTIzGoq(i?zVr9&3#q)?DP1FK8oHhI%!$clL_{K z0Rx(E1=?5$jy)`K-vq*J+3NmiHZ=l95<>9xaroF#XKBa&56+zbf9B2~^Wzrm!TR4h z$}`0^mv}|`YHo}i=Cc^%lg5mrd77n6)p>cJOIwEn1+p8;)^J+=*IK<{>l2CVeEaN;hF zGkgMfbEiApB!!cmS&!pq)YnWq#`u2@3mACcUA)HHdpuZQ;M36g^t?6QARKxB1=9S; z+k?V&=C)T%d_rF|L&qzM^aU!?yfnd-{O zi~+j^=L@43B3kbGRoH?5e|Ap<=JC`9;9oZO1d_Cj{J1CdF(6Y*y!&;DN> zb5Bp+?a_TNu<5L#hx~tg*a~}P1m1D5-}Q?+A$+N;lZPY?1>v@RwuPs-v>!81gJcD^ zI$03A?yrV_q2(q}=YBA-G&N}Lf)%@*1}D95vyL5iZ@)lT03{3sD}~KbS`Q(ACu=_# ze-tU~3z+1NP8>tu=+4geSHS*#NIZ)lNr{7@Hq3Dl63xz64DChW@Mp$I$5FqjNwXkN OC&}z62Hw1xzyH}gS2r2} literal 0 HcmV?d00001 diff --git a/crates/j2k-native/fixtures/multi_segment/bypass_termall_rgb_53.j2k b/crates/j2k-native/fixtures/multi_segment/bypass_termall_rgb_53.j2k new file mode 100644 index 0000000000000000000000000000000000000000..316fd342481f044a7bf7b8b7ba48628abde22cd1 GIT binary patch literal 8206 zcmXX~bx;-T)4g;z(r^jsxO53zQbG^`0i}@!X=(7ebT>$MODjloK`H5!6r@AC5rNNl zzTeL7%>K2r&+Id2&N&a74_Xiy|n)`j1Ju7 zyFOJy0u-aZ+|>XEb%j6g6JQ(5>XkMT@kJ1?YjB1wI6m+Iz>`1mq2+8V+OZS8d-^Xj z37MsnX z$z-@d!G)~+krWD=z!@M*`5AZY#fHnpw6`wum!_(i&h~qo$Fj^iI&7JANIknzWSu># z-u*N-m>7sc)j1mUt|J4GPQ)QOABbUDxQGYvx&Ml1?-KtP-}usd`n{)}CVgbDKN5fY zMuFXPP3M)lnb{xmrmqt>=0kgP(F^_B7FNsyY%QwmBbq!;*DI>-CNi}&KFM&5_@{;q zegK9~=JCqk4-kb>$aw6b)&MAqosIxZI8sD?RufRx@V?!5oUKakEH7t3Q0(oY3ldDg z^KL)OeOMG)#FS+Xj28}JuJKqS^cikJMT?q`D*ZrU#l{JwMD9sE>oY7j#sfK$f{=iC zSWF?us#^);57YG5cxIXNbp&Dy<`P7^W@V_irZVdQ@d=1_1^{y3dz_f1^F+Fx8syLE zhsjN%`|Wt;B_PQQ!g1*uviXi~w*?HbWbwM2*@`axr5OC6qO6dEhzDf{{%QJ94fGU1 z-RC^_+m~*0w;=fH3|oL(Q7}kCaTg9L92w*ROZ2=qN3bKAU6t8g z{p+<=pX2~kSD;P5AHL`D9Y<+Ja3hng%K0#+g;HQsf2(IhA>St%(qbuK5=MF`^}xIW z6Brv?<-)Msr>XZGmMih)Ek`v&{@aX6iUDEV!gxh6m+gaOa4&n?WYbX+rMv;G%er|y zCw2?pTjeb!=RvCNpK#e8H7YOb#>^DDT6_NFc$&kf^jaPyzG}bd9e)$ah#@qN%TKYs z(p)$HViYSW_~$E`ulO8vU6!mYU7uZWunlynoo?*9I-Puj1_rVuKagieY9i6Dr5yt~ z9L?DRiRepHs*iQl|B_fPZ}YenK>DTDnJYiN+AO6>7kC>OhS+k?FA=_&su{#%;Mmso|2nb-Jllck`e&*KUWbRykzT54qe} zLqN>e2aWJ=@%H}ra?)jysp16%bozLleNTc;@CdJySkOMR^AqJnZ0n}D{Lp7BLxEAQ86 zY`Bm$(T)^Q-bd14Yd%>1V!IAALMNhRdMTv32xA#*-T&n@v|+`l&|swQU6b?hp5qjg z>K;!1RfE%G`P9uZL;2@e(CCauceF&#k=Bi_>xS%W4F96nNAJ++kEn7#w#<8#YYCh$ z^<31A->>jw1o}uugjxTgCJ|RcWxoO}v4tPNVJ@daVnS~@f*OHf@kz+>rmN_ziWbkj z;nRasOokE$N7Q^bMYs|#cq$lS`*#SY6m%GVDlx5#mpAHNfP|ARmPUWCA5L_=jSiGb zCpg6S5`sJcA|aHpL5SigclYH(@HF~K#CVt}Y4hVBCSH93C{GLAbtO}T%g?-?d~Kk4 zlZT}%mocoD%%<1BgCtb*^~!ctX2tp3SIw3#56>GtnJ5zFl_&3~r`h^QfrAjwPqLOD z|B8PdoQ)kDUL+`KThXN)j~*{SpL%p;;%@i5)`Dx)tO8Ld=&Cw)Y&{ z_?PkgjSvJKTM_z0^~j0fZau=td7-F?-l$_&W|~s{{KK)pyQsKXNru?}UKMR!Zj z(3z*I6(Pm7HhZ&I*i2*1K?L^I1}3ZMLe-hGBV`SjVRh6mJ^CDUTyBMgc9jF-nv)H9 zl_V-%Y?4-{07R>x$Z(Sq=k3fh4CY49;>X4lk~-m|O)ROggLyyT0C!Y6vW*zi%rhIy zD7D>!>PYnPs|WeKxkrbZYuMjt$;3Op-9zag1brqFRlfqPw4$jRssUk_Yn0LO9YJ*@ zRsPB$@5XZ56&7uskr-wF*^h5N3~Y6gb_!fp_AfKS^1tx}&}QcH4$9BV1q>2XevN>f9Ild$ObHKmUN>S6Di%9G*}ywJcIR*Q+p*iUO2JUC+su~Wy& zDuc^BoCjePaePNBJuTTu36lm0YjgufrhSOvi@>_n1e&XIThN*8 zwZ5|JVBgzyhH`ci-rJ($*L*i&8=B^-Jk_x$71RYsu-TX~LMCu(7X#KQ_$EZ&|G8~8 zU8ZW6coilwg3ODkhdxq>c(A{yK>W`5a=L_No?Qz#6K#$GVg-u0~7*)_%h=?_)#YRPmlO z?^IMG+U3lbq$#c`%nc`i!aBlCI;E=q{m=uT6s^JOAaMAMhWW!MFbTFCy}Nybe_2a! z$Ynvd-aI$LB};3HXlv`4y9lmrK6*Y+La-nSeKI=XaB`R;Z-Q?!a*AY#pu`O=DI?#W6AMB4`dAiI<_mYoE zZjFYPE26q)JlD6GDm45+BqH|!!qz_7Vz8&XBECJq?GN&p4P0BpYVb7_GcJE#cVuO` z7lp=^mSveH#`7~WY0j9YBzZhoD0`!a;wO|%4HA7WYvSTiXr}cIuY%_^AXPLUBl3}{ zR^CMFV#Bkr5!D=V3`#-Pt{!AEy_;xJ&6_=a+eZwHga$VWsgHZGU+%wp-aY>b6FxAh9QZrfavca~;dpuR5#&*nD5GZLT?Yx zbQw^X9oJXh0}RK0*yAg7*HT@>PvPN*;@V*!H3V8Z$+=DonP=0~gLzCV`kg3`v6u8; zDIjfBbtBG)A9sxkqa1s?}*#-sl zI-yLVAs(RPdQR_}tn(#jF-e$i@Yq=IIj6E+RZP^AnHjE-f7ni@>KIV2ao^u`|7ATc zCuK_clR&2B*P3d)G+{{kPZ~E5=>3WAgeJ4@K6d#poa&f!iZ;1aA#J#L!9Wxzoc=rW z((9{om;M!mNgsR=JQCdBtU&ay2*cFbYz>udH+nx}y6fPU{;-Jv z_>&X*#eIZ+sA*Y6vfKmhinvVQxooEYF+ZaG+dM!5XCg~wF%9?F^&LVD=pspum2!?b zzch>?e|((u07(2(qCX>ssCJzSu-H+*-Q!F6B?-ZNrE3Gl?K|c!hN4`-KQ4s|^;JTg zjvo+h|32&Fo{H^&VNjS`z7^)N3*KVHSkiwYNXI<;rULvkKSWKJs<#WV8@c|$hW>9{u&s{@D z5G1aT+5FG;yY*9*jF{BUj6^b=&M{0mWcgXGgbES!z_`#~=}YEZiyzaFlV20{-PqpX zWF5(*m0BNl^ZM@7`>=f)HkTX<>hVI2m>}4!^9|epF$(3c2bg>t@4ws17H)o@$ig@r zD2CFo*QSfuu&zLnM8>5Xh>WE z%yzM?<4*>W%WnnZw|d&fbGd`I?YASvCr>#5)@>m|V!sQwCH>`;=Dy1GYyEmxBlv=| zEDqm$NBk7ZgI5V93lurE9dk42yB5B&S5d5rRn`)J%=-QJl)c9vK<@KAfFq1yrJQe+ zf$L7%8%g54m^;uIN1#$^^`8HBy^67(m;Z^n%+o^^F(-g9$zEh?Qxg(X8-{BUCV>#yl z;5JNVvJXaTu)87}?2fEu*80g)Be!t1zg=CoCgPbsceWCt1!$zd;P|O5dX=aJx-rAT zI{XR?0|U8NTX8!C?Xw{QC^X&trG%`6HF3x8cV6K4Oe78@(+25Kn3C-bFFpzMLsWl|3%hFOAaIHx)kj*YK-a=5rt}?`EfGk!wxuj`pyKc1d^$zu9L|2jM@_=sa0W6-6XHHV*v|t zc7c6D5P{L@(shiw&?6lEniQY#Betz0o%_v>+lBgrhab!+j3kxC{tHHv)4Yx$kRRu! zeYY#!sIcKaD4c|890@3D?^8z>)ZubbZhD4QFWs7yO;9(W#!?FQ)i4k!(AgGHm;J;X zW?%wc&IBOLl-h+FP9Z&_0k;nqw(^qMWv~!v5^^H@+4<7k`LWVg=*!m4M4>?_GTGUm z(s@=u>?SRUU)Q_EzSOM??yD+2sCKB-D&fQw30 z;X{yCq5Y26cHh0=A4V?$FRNpU6_|{q|GwE`>X&&R&*J(~X0Jpjsouoyc>*cdXu?qJ zS>*YrJEkm*Sg%dLi2Mg9;%|S;ng@uCz7t9Jii0CUMt%ua{dSK0ukV_-G4UCW1A~d4 z@mYu4@aaF_#w2)DIDS1R;OL^4%NU#QsQfa5Fd|g-^q6EPL2BSF9AL#Shb36-a;d5+ z(bGHl(^`qmDgRRW^~FM+UNz|gWXJ!50Dxg91fwiCqu2sWGb=3wHA^VFbzL&AFsDCY z_krv-!V6z>(rme6;fWF9)nI}G%r?;jtiEh3Wf})nfvIgY629dx7|>pugfu@` zf2I$aRJX6Iix-fr6s>sbvPxug5*o6&BvsZUR1}o%tQRS-?hB_hwBdR_R^vT%+i41T z_GQsgq)h((;m%s_Z=0?Okz+jCkiZ9LuDE}MFIU+{?8r9I^@ie zvZ`7mk};eTAdRX{a?a)lu0i4Ie3Y{U24GlP#gmr);w#-LSZ4;J>*dMgZS(Ud0Klhm z5TZ*SYAh)~iSD~*mUKhzU3s%a`{v1U$?({>d~OORqr>YD2;)e4PtZV4YHSC}Sz!R1 zDlFyBNLzoz&8wNTEJu%L{md3g*DjWKI$_kErq-WARRc&(|4nCSu1w&nO9?X4=+KB0 zm9X5~Hsf5Y>S$^+WQ*yJU~2PER>pqA8v9x;*^O7NU}Rt;v^hMDo75BK77HO9QxEKjR7lB>JK zY1A7X8uQL#IMfj)Ziar_P5`|EVE6v9T6gP|Hw z9e);%Nq`wH7_5PnPA$EPfyrdBgioxmkUuN8(oW3SV=K8O?mzDFdQR$!IPgqvjuzAH zYayA*FF6@vSrp@FN0N8|Xtt-!e$BR;Q@@SxxfZLrpEZi+nBWr-$~nM)p{Ia43*MlJ zyOU#7tdTn>xNy|c)Hl)SIig`RV^|}qXcfnR0!=(>y}rONhL2C>3p!J`*?ww{0U|8Y z!hAYePh#`VY7;&O<%oB#EVgTJq}AW=0gG11YrVt-TpWAZWyy)ID09%z6M*<`gjQLbIOcc0}2PK&nPM- zv^CR0A!f?IYGaJ^PhaFHr~rT_)X>n<8h2-Lw&Ka1ZEgWe(Z(vAk=5l(e5&b=$^vi# zTF3x7{Ol2G&e9VROyXRl+~2zFWO($awLNLZi?;3N{Q#q$=7VsTHN?9jrZ6-^kV%8Q zd4F~v1?S`+wy}!8lz0~C+;D4lbqe4%&AH@Xgm1P zEiaY-R&D_`bj9IQt5cIrk6WHeRgTDVnu@wV0pj_@MvAB{FF`WN;tx|4=U3!>(ok*% zXuJOb%fRwKb`qKH`vvFoR-H?1L|jM?-Jh;7YdIF}z?ej}#;8t4*(0|Mh`Q(Uw-tFA z1DGgBJh;+3vzOZ&d70>Tp#~$3CxbAhNJ^?Mvi&g5#0nK1OZ*T!}2*C|*5gi~o{yyqAB9g|Q} zwdtmxW%J&WEp7B7WVbZVwSK~+d>WaX+gPgEb0W2RuS$qHIkR|vy-XTDm|Z^6rp1|& zv8H>s5zSe}>|U9Za<}5F`1Y%gejqdgq94+H%nrr0LRXN^ii9|~V7(c9lmRipGs{1t zfA#)rBF?^m*lQA3S(fe9uE+z#17u~dn!0yP9$O4T)<3=lF5ExBT)1Tqr|?Z=-o| zRgebTd`TAw9q14Fxo3((VW^YQ0SCvFvgu(Z1G#Oit`U4?)75=- zO+-bbRqtoslqzk%$g>|UMO@teVY(viXJ3Wp5gb{3U^>TUbb3{A>27iIb1fUiXUsr& z4xnuu(?PwZyk({Y$luZ@z2}d^)e`#H(>AU8^AXxIe>~boC0pZzQ!dAa1T*Q3UAde; zWuPDbT%2Rh3?S6v7cE%MZE)o$tw0T}^)j~?f_u}~euPpk7n2X+0zfEHE(Y=3n^d8m z)QuH;Rpqq6*dumQ9o)x$s|kR@h)|-6t9S~#o%tZMGr+qrTIj8sQcdk~R~R{WP>KHb zm02axJ=gaXP9DyeZ;LF9#lbE!eTwCnTwbvOPqX`Ql^SyzS(!Ei6ygLL;Wk6bYASGm z)-0L36xPLy`9_d-x?*Zm?9byazh$qRZ~bVbv=72%UVg$xDx6}SIRTV-SY5hQBh|j} zt{cv>!O0>D8Yy=`Kp6Us`0mWEj$9N}OK0NeK?l*{k6S;se9o9Ti0KyFycg09%t!VT z-@YtWdxk;}#^M<hOQCbTEYQB`m`c zeEK!xXy@`e6lU@r`%-x}jJJ3yImP%j4esfSq&L%P%+R#PWcv1NX>KgGk{Xx@ sf_jd&N^7++laeh{@Mff(_W{6Bp5vLTV||)`xzidZfFMcx3LfVF4`?n}p#T5? literal 0 HcmV?d00001 diff --git a/crates/j2k-native/fixtures/multi_segment/reset_rgb_97.j2k b/crates/j2k-native/fixtures/multi_segment/reset_rgb_97.j2k new file mode 100644 index 0000000000000000000000000000000000000000..4ae893afdd9f94984ef4b1d1b61765a5a5677b96 GIT binary patch literal 6140 zcmVF8}}l004jh004jh0000000000004jh004jh00000000000|x;C z2LS;G0RjI~01N;E00RL60s;a6|6BkOLSIr)1yBW0M_1EV(^q2uWB?@q0Yh?SVRU6= zAYyqSPjF>!N>D{dAa-SPb7^mGATlmBE;Rp;015yA000+%00IA#;;QtTqU8V*4<@KB zPAUBu+)e3xS@`)4f%+s0&d;}{mw!7n=ZULNlo|cctT)%_5ia7ua4J~hY;u|-s$y%g zm2eN>;DK$BOc}6<0y*8N2(oJbNUkHSxKwIYd-LA6y$HD1Z4cKeh*A146e7BwYOr?T!nGwVd3tBe)~MJ0;Q)Cd-Za>yJvwA?fH<& z6W`bz2DjX4&!`En^Dbsr3F@7rKHVIwg$e0vu!P{ywh6ZrXxCV~@`X{UDm>9!HP30Q zpx6%c&u&q>(mo#Uygf|%FWuu#R4P<#D*d^8<1dMONk&}Ec#moQr~pYug1H_W z(N-&815nH!Gm6$?l$TvzE5j^GVN(z71N+L{{~1|rqLvNwBWa=+Y{8s%Ts^b5JS{R{ zm_k6HFwd_H;uWbp6}j_15iednzJqK=1z)qUEi>{pQIw;$w0oiU9wWC?G##yz#JQmMkO1Xs8BWv? zgUwP%s#dM>16#WAMq*V*4kY$30)kon6NDofydD<4x(@L4j51=s$(*yBZVN1_zKr6v zBNAr~vH0+|jz&3Iir6exK$&1!b;tv60Mhyt1F9Bb@K6wS&?#qr?4nQsRv@jjCukJ13%C)i z$u&WA_ZrcJ6Md$P)+2&vjQ|bc4=4bR@kST}suTiU;}fDUoKU~aN=OTVH@7W776QRn^0rKd zkEQ%0aTZPv%N2A`gsJm6Wk|awa2*?qWI^VM`!ygSZ$weR3A+a;isv$fK!yGaoYP6T zLB|gX2MydNR$gI;JwjfUy<}Wi_$Npz?2*N@XWkQu*@CuIGNm8BlAhn#dAi=1ls@n1 zis~@t@}GTX?Rw6TeMli79i_?j|9?K2iZs(r{^nMyQUk#=O-Z-F4&Ali$+ss{^_on- z)NFm#jF#5tLyX)!-W`nSCxqOEf%((U-(F*hb9K(&B)Z8#v0vWGh&cRj5_)-`4|g+( z#fpp(J7J5Cb`{>>uRoj$9B7H*_BPqV;znz5V>Q^ps3uPd8o_0jysECAFkCM!NYz)o zyZ69MOFP^~qA0dWXF%Gix042tFp3HVB)4D*qB$RLaq}zyLM+6o*yDPM*oy!_Gi#H%1BY-11jzykPuk z9|cYP=9$N2l=|wL<2BQjEwh@@vD(~xjoM^URg#&*HCJUZ}8(fW!qS&9RByy*o+N}cnM)r)h z>!5hUTP>Qg?MF18)BpvutSzFbJ*7N&jSFxvd_x1^aVvFTygv&>$8(}^3oLKyqnaB0 zn;~HcY=?(|7k+gpgM9QV_6W?(%ejg*+E0^kq>pzpc^c64eWPt>yc#ErICj*+8d03=QWlk81@BqnWK91inKSDO8}{(t^036=sd zirphS<_Mt%bX5he=QCHg>gBni1vXiFKUYZ4L0t%V(tCnDxGXGL4Ff5D-pPf#?XA^ei(P(ot%S2?`(Y75`TEz2wtQfLD2yXez=qnPA<2nDr3I+a78 zreNed$?MNIfTLgBe651djj!EGSgjfmIpL(!BC{?m6AU4x$L-z~J}%{K1PeV;A%N>z zQ%*L>?(9c2pCXwh=5jvyL1`CXOeB|b_M1sC6je>rf!|E)Vpl>F8ZUVoYA-#afpp|K zheO@7h;;HyaI^6Ndb3Yz@-fEVloGzMy7arVVDs;2(Cy|zthZjNQ9oCc9A>}f$LCLr zj6%0C@4uM+o9AJM&_^x_W1-hEr*^Ny=cZEDTUcrx$4c^a7MB9Cny=7@j9d-vgt%#j zos^3-OAa-VQ-OuzE^{Q)aBId}zoUSzn!4J=A_p5y;W_x=T(5cG;5<}4t#uFxq^%C) zWOqRC0zl^ZS`@aY4B*?hhe)#W_qu{D=W|q=zjotTFcQDs)0*`yv_0j=PM)>J!riYm z<89wmb6Q9hCiyRc&0`DntIla=3_FQ%Qd*x6_lTy8)%hdTYC((v7^hJ#ztbdHGxG2s z85YkF-RHPc0&w(+3YQ%WSx^%l%YNY>UNn4Qe3oSq_JMpjpSy~{{$sSN1n~2}IX`7n zmynof0>@nYH4;mS^3oQs|39_}w$?&WCd--n{1+4mS90H#66=k}R+eX9S4M>57n6>X zt!%PLew7mhgR+MKO^4_As;VS-t&7yo%lB9copdAGA89Rgk7%CIp3wV8w@bPWcow=p z3*!%7UueClw%#ledNEr9ssgkc&3&CSt>*llaZ`)@QMU_CM3M-&!5*7O${7a6Q&QgIqd9+fCJ zDY~#s%^X?J*B3trr`+}$Cl3VPT=eY+gd}VLmT4}i>i}l|WaI&=x#Piguo{MnR#>0~ zXv^&P;SX;11L=2*@2JU84@>2UQgT7QpBnsJlQXuf;8RjE4c{P%>daZfI_TQ|2t`B} z)J2)=o=_(7l3$~wa4^ay6ifEp9Fo9bxzb97>U38sZky6;#y{8we4Iy~u)|XSI+@ug z!@qto===4kEv$w>hL!=ttH_@}TnejX32KcSL7`UTd)Bp|j9_JhC^ooxa>y!-B?T&l z#O!j-PE+GEax2qrgvQgy$fujrSjx*}Z|?`ps-u(`Vgdmj1(^>C`FMM|h=9GiK)H%# z^kL>rJLpI(8yt@H(ATx?COQ`hqAD+(b)8nuQctIan0fI7tk2!+bQ_h$Dsxs17A3UM^BE0|k96h#htQFpGqb;*gI8Q-IJyCAVI z-l_7z8gF9G)4s`}O$RcJzq5mme5&cU67DF28Rexk?#>JO0fp0R-L$!lkS&=ihmiNa zwCkVk-kkeG>+fByv0IPYFKE@h?FY1LSN4E=MeP{%{lB^lZ)l5F+V+Rq6WS471KJ;G zK#ib%FJU_Nz%*u$z%Ms#6_s(r(|P5efBM&ITmY{mPjZaIJ$?MF#<>HT-mV4)kY1SW zl-^#PBcK0+MCGfGRgzyLYT7dvDr9X75{dX*6$CZht_NR4QGpN>O1xzB#wxUU#dI*}au22OTVQ&;4hI%7RF0Qpk*q$J@P8z|Ra!>(qinZ#>^K#fO`D(-7CLhZM!VlT3IT(+H) zE>U@ff{(3oS&E^CWAhf>Z zti!La*QRp#?zz}$<1i+SuB6Q_BjBJtjBlZW{2+JB15Bfd@>6hN|3zP^Im}v)V;1ml zgC!zRtNw#v!UAW!g$+KX)cSV~ZJ1Yx=)7bH8y{Yz>_M4geNO;$U z_Lc1xGNZ?UhHxwT*k8QMdIU^zL3zuS)~RV4=JsQ{2%sr~Ye zs#mC(2b@-4jUhx1(|p4|@i8;>u5MBr$qltLZXUuJ{|3N~ zjxEU%{}38xLjZ2Cp0v)NRGAR8ttPevbSiQOwREd(qNwXwt;lfWF7K~E42C{}{Xzy9 z12B@x&6$j;ji_h${#)(}$f=$mMzNkW4B3DE%q2b`0~r~Y6_C9g?%R4FfA18%X&Q1` zs>-p#2SB_SjmW)lv3sk59?UK{AR5Yj$AYT8r_|X*j1n@@(*;f>&aLwTCvl72j+T z$RWuuyrLu3;rjS3?Im7ArKY!c$Wtl(=XF3>GX4P0!th}gL5{t)NR)8c($7d^VMT+8 zj$2s)fm)P~n{hrBcP#hV6vV@(yzgLR>598ms(um$Mg%|(-M}v z^(cX+sxN=XA;)32_G&(*3Xg%=qQkU6R9UG(v4JtE<`~f{pn;*TEE5Vn#pX?~2z`xa z3=PO@zK^x|?jrH3fi}m5^z>j#*Az~Fi0TBy}&dWwuX$#~( O9|O=C1#%scP5;^Ox2*gC literal 0 HcmV?d00001 diff --git a/crates/j2k-native/fixtures/multi_segment/segmark_gray_53.j2k b/crates/j2k-native/fixtures/multi_segment/segmark_gray_53.j2k new file mode 100644 index 0000000000000000000000000000000000000000..2f53bc63299af9358dccc1d5d714ea52fa61ab3e GIT binary patch literal 5447 zcmV-N6}amEPybN>DF6Tf004jh004jh0000000000004jh004jh00000000000S5sA z|55-900IC500II6AOZhe017}rNJvmfNKpS|03`qcLvm$dbY)~9VtF7>aAj^vP(?=| zc4cyNX>V>IGA=bPH2;tQ3IG5A02J2%0soWdtN4#s^{^5)a5ryS-yJNEm?02r`0Qb1 z&Q*I;t0uKwef_5WVI&J04j-gkDf{K{O>~h-4(iSWNGA9spX-Yw@*k0p` z<|G;4ve*kOX?tp_fDqWsGpbG5^xP@;N$nq+K54+Z{*nqHYAwT`sx)uYXF{2ilQyC7hixLow%PaDEYkR3*m;n@i4Ap`-u7W0a)HTc0h*8z z`8MoWe1h}}TE|CDzs7)@>J*`e3LLbYF7Ny2FHmp-HoiP z`(WW1Pz~V?Nz4TvBN&a{)H8TC+TvpsM-2aD%-K>_d>4P$L`cwZC_O>l75_AjFJ<7O z4YW{e=_M~W9j>?)l?NRrY zV>M`;S)I2^5vNO$R#4G)rp7_zV1o-V&q#Vk+|HnBxVfa%SiHWejL20Zz`JoQG&Hb( z6M+Q_+Z+}}?zb5l`weomBr1%jjN{ruX9(b!+5TYlC7n-feqq@Ic)|Uax zBl&E$TjA?U30$@pa55o(#6ZXcc*Ga`(Y*CJ*^XS{>TvAV78#RWm~a8e;8%iW64ypM zcv98w3qr&+^WX}zjCE5pFK5C!>RH`w)LnV2`FWL3)HC*g1;r&9Ri)y~*M?pNHUmYX zN(45CUK<7MbOBku<$#x~up}uf0IV=Ti5ok0GXAW$f*nkJY8bn+?DERgn5ZyaX;~%@ z@Vo&2k|FAwQJF9*pKb#r@*kG9tZKMT6qWDj-r?wM?+^S$vj~A^G6$B^Mz1E0@d7fe z5`%YSqm1k^FjeNP&CRn!!GB_f>BZ2V`NZpLY|B(R?Gfe1`gx$Bbt94#@0I@ZB?Ku` z%Qbs%?;hD>0#7Kwt0gLz^^7_vl_a?cv~+0IYJ0tBUR)z&i>>o!SveO>X{H+`he3*@ zz2gAZbBbKmGWJ>|=%L#+6~qKO9#0lFYYnDYh5#_z1PL4+-d+R_75vEU%4*{(dK1Tn z5>IbMjf@m6uU(q-p)2Kq=k=7|&cQ0Tm<*Qv6@tVs@64c^`s$3Wz3HOl0BLyF1`02R z-2tKGMS0a9A)^xy0B7o0d(!0SO%4(_A>^j6L*;d0{UhU0gB@DG23Ww5(gCx37Gg4 zUYdLAM{$aY+JRu64QcWyTfI2#Qc_;$`fw4<)Lwi&CQsrBi07Ol+bx9YM#!^>OmKLQ)0-6i>fy=(~)MnZc z1)ryB&eWqi7wnp;07SrqqkW0lf|@2_PkbyU1-sRIuu{{2b6xn+h;-g5C-lr$w_0G8sVj`0CjP zF8=lEnzcVawy^sm?C3=qX)r@|7Qqyj=aU${$VaYo?B#RdX3#-AUl-jCKDprHNsm1a zgqYnN8gg|X{_$1OPa;+92&mu*t&H>*yFryRQa?DM$ciaU6&=YU+*$Y0{nZ~G=dU~% zEhWEvd!s=e@jD_>;lSfjUGcmT<6CGo?kk9ET4BV9V9ITEBhNnQ)wfjVSO!k(#D(M!_w2x^m z?z$JW_V={j((9)eZtjEw9NJq*Vw*yw02Y{}s41~QcJJgItoPTvHNQ>0hvUK7vIalM z(oOes^+Prh;hIy*apR;b-OzPK7(p9B{P-|yQlR_(_l_SB?mRU?O^P@t2r+_q94ES?a4x9>Y1Hjn%PIwSBx{cFAfD=!Konj zgyfW?f^xNb6jng2ep)bIs5s;nJ%~wr#N{}wDz;@&xG?%DCd9Gnop&ozsvYQ{4tp>H zu4WZ9)B6%N7FFzAR}c1^&;m=Jn|3u7vs@v3$UZqnpK3)mSI!Dp_t{su2+%w2sJ0kP z_2(V{c+mhkdRXfe-c`2HmYEL7j;_^$mq;#+$C?Wv<>rf&nz)-*lEL#6+t70nV2?)Q zFXB zg3o0^EItnrtarv_~p^kZrCn zjqYXhAGT20>MikeM8zMz%9_}1B4}e=0ls?f*Ao|N7%_>s@ad08cUL(%jQ(OhC{0KC zTV&-l>{`vnBR|E$?nWVGB4p2!GT*1ePStBGZRi&!SSOf>pe3aGttm-vhyK>j!36P6 zNy@9bXW-L$Q+G$@UkbQW!k#MSUkZ6s#dxOfmCC*-hWR>P}bGZZ+yR+TRg?V%f7FK|z%^E|l?-@qH{8+u$1aU!Q}H z{~FH`45q9>1hdTNUZ0zRZ)_9XWFVXWOvKS7`^!}#%Z)!K^rL&ZlH5gmw>&| z@Sf0*p`y!|u@s7*=l0cIeU^bFM2(%fjDZt*!9Zx^QRzGp9G-O*>adNq@aq$mTc>7T7{e z&GNsm{UXMs7v|b-nveRHW+3R`k8z2v4rZeVqrIN2mUxmi0MN6M)YdrMjOhRyX0LX$cgxuc&1;rubPtMH|4P&}q=KbEQ@R7R)$7%6%$AzP zjeKFOrXV?>+r_tl_A&Ox4hrDMMFflc*+_KQR zLBcUvkU$2@s1&-tc(N5)FWWiQgspg}3@wmzKvMK4Jkz3VK8D+bpGdHDB1fXmQqq-c z9i)vWXI-g{cx_4WpQRo5rNY$XM^67~kla|{_q%H(a?u-Y6*?;Xd}6)ER54`sBSYsI zv}&A6@@u6g>lCyBD3~4ac7ABqa9mR@4t^n5r{)xc4&+Dd3Ml)c z33VEvykA8bi$lK<;yw8@MiQo*B z7wuS(rP>yquf5pS1nFaXjrWQrM66~x`e4U=>P9w>cMVLT!kbt#9Eb87mEQ=G7dadk zF6O$+S&wU-%^XXrTBsnNdL_=7{UQ{o6H4arnkqdf} z+oFzU=R|O%P74Dj+3`(*|7|c$nb&-wV4#Egr`VMN7!898*VaPBCvze(HIQdQo}cqd zeAi%wT~A4>j21@%Bjdsml$u1V{C<`^gD;^y0K+uWU^#S&Fzn|$^hEdXqVeJT~O!V z23)1x34TzemJ7nY35f6tw)z$kASg|ii}ZSrghP5t(2d>^_ejIK-9peANXOu%xdeC* zi#l@LYv?*Aw=kcFeEH=?)l?48aU<>r`|6JXok7wOE!j}dF706L>1=lo5 z2N$2Cxzm14$vL`dN>Q!vd1$!~yXcy? zj!;n4a)Rz3j3+nh{4-6h=Vw%vx=Qs9G+^L9|2ZZ{owTdwCP#DN{~1;F^{sZ7ejgb zd<}Q-xi^UxN}uvzBWqQt*)Tmeb;1igyrzR>iqUh^*Zh~+Vo`0CJUKjMWx zB3kS!sompj5vzvVTT%B$PMFDV^sdsv0&lK%G7p1|&L(6I^4yt806=yY^G0mb39CHAjZ&P8b)NUtlravv?;YRJ zYi<$DcQEKzBnlsgIeLu>#wMk3=d(+Tz4lTjdS2H~mk-eVEMyd>$MJ3Xuo)HsTS9uJ z$P{tsG%}Txq)=&!B>k2tjY*LgP8Dv?joA$KmMgDpLLh-ji` zO(&B&3>>y}J^>hQrP3&e?Ne5e2?N53UYaYJ-8Snzk434ymI6O;@Zh;DBv|?FO1QN) zvc%q3PGsoqPUNBz^Jf!j9p<$Q--fBsyZ;n3jGdb?DT=DhgNoZ{ugFzXU2F_jkLqxe z>k0;r<%FBjdR>Y5`fZ?KE-vM95^N=a9mqV)l{_#vT#p+PJtb^i9rq;x1d5dV zs0Zmw7B)gMH;-DmuLQzvpuU~dxQ>N4yTeP9v5p<%1YS&9)Ow9ZQT6rbD{szJ>vn6>Y}2Ugyk@{8KQvSwMlfHSqqXkv)%s zelZ|ceYyi~u27Vgpbh(G>3#mn7gdu3b~eO2{=nA8k5Ib{0lMaQB5?e(pB1XCbqUz5 zFrFVJW1hAwUwGPTtLzP0t`QH*Y_uZbjjVp)j=C6~Jdi43B^0HgR33)mr`%|;kEFvr z;>+9n-zTiCv!;S~D-Bz28g&KwXkc_+bF-i2L(73+z}eH<(swVWk8J@xcPS(rPGfr5 x($5--fwAcE0LQ2z_?TGxy~g6&wuDVBlTzwe0iEkOLRze-r6Ln?+^4vj|Jg^bW-9;y literal 0 HcmV?d00001 diff --git a/crates/j2k-native/fixtures/multi_segment/termall_gray_97.j2k b/crates/j2k-native/fixtures/multi_segment/termall_gray_97.j2k new file mode 100644 index 0000000000000000000000000000000000000000..3e497f23d134e983641ac3a6030954b55dfd5822 GIT binary patch literal 5464 zcmV-e6{qU|PybN>DF6Tf004jh004jh0000000000004jh004jh00000000000S5sA z|55-900IC500II61OWeB01-l8Qcwj@1yDy<(^u11V*g|SB>(|Ka%Ew3Wn>^?c_2@4 zWo}APMMofZWpZZCnO5a&$p$PM%w$Se=X3ZeAUZ<`V2Znd>Db3hL3|7FzFZW1!zSSpB?fg zQOJXqBdi|h(ZTkEA5+P}GzD8mh}t{5g(=^G&|v%Dd4ussFdXn||9`|~SD!^QjD)+N zPy%jf;Ua72zZf-btztd_M8Ocx>L;z0@L|8NXQu|p2WvBll2PgzR0M;g){^{9B}eNT zcn1UTcgD5uLY*~OlX^oLGD$1^aJPRL;&TTN>9|Q6pCsZS4F7+$6#p-{Q<}(_qY%0C z|7pxav-qUbr(RMo`7jFm8^<*d3ea0VPbh2&&v7$<01eha?8{Yu=m8=<@jgjnv#r`I z^e;<8YJcKt*G5^D=)v!-g?9HDD;_f8!^($__Jo zh%o0IM=mawBzwXJ3_jwVJaMjRqZX}&yDBq1`6R-ti}92ekAD$rr9+6B=KCC{qj-}q z$K%PBy2H>aeWOMtt4lwBUql*zbM-%e1@#$RT^!JmHS%)L6idxQUv1SzE;5H}%#$^j zAVx$2Up6Ii&ieE@X1s$YN2_IuG|7z-n{l6i7yPW>x)kp@>-1?ws&?P_D;Ce9_oJyQ z1c+yG4K)wz?~m4_#Md>RQsqokkADnlChH0q`ppn6b7Gbp`dg!UPHce8w41P?2Iun# zJR5<7n1P=#g}6TOK?5HW0Os$16vCNHP{7D>NaUVpLYw~yrl~qz2ukTGA%xr7e+0B4 zQm|p5z>R@GmEG{Qfor1_Wq>09Y9-PxdqcFZKg;^}e}4(#bKT5k*&38f_{J#Df2yB(e}4 zmAy$p7BZ;#rodESS$Ynr0MQ@N810=tKtHEhxln$X_FnW;ayXtQ*o@W|kp|zSo4%HV zaerj8LfS`BQ?n=ObtCBWvOAo?jf-Jnww_1x@a=x-J+=QY+G-eCbv^_-4@-Y$A69JB zuOPlXV+JGqdR7))jhN_>Xs~M&c1Yu7py;a|hdNb)hf{|BAm`C@H@^%M>|Azt=6Vs* zU6TyR9=B0B29E6fkRaw&10-E@_RXo=y(L!@=V59Lz&t+R9h!cdGK#9269%TL!pR9U zkrM#;={DjsgT1{c&3-wRvYs zt%Xo8cSiYROcz7~7k$r}zD4^cAnr4kx#+K3of1n5nf6pH3Eu)yGJ7G9yY0Sl`nv?K z>n4lF@l@CbO}P8G6#x-tZt2JvL620^F3X6UJbw#YwmnM5m_L3+xtW*Sw-bPl$CPGA z@^gS^OF!aAW`rVqhGz5rpl#QJmN&wrh537?4<)D}!0AQ=C`_Ap8eCHwtLc3|dT$1y z-rW_$eN4`_9%Ta*3V`y^(Qn$8$llDf3z~IEUbVYIe*hI2Dn)d^df{k7{{{eAf7T5t z6nFqlSi_kMpndg|!-VCc^3~k6m$F+AKM0>8PJR1-+wra5C|30ApmNsflzDEtA8^JV zaW&$dZoR`GbXM+a0$stP+)}X(8 zk?R(vM*^A)`BEtpXv?W|lb}`5jtI4TPu)Gm~$QAAbz8PVH2y6ZWU6Qy;M(m)&q45IV%e>gky8Mz~s`tbdNDuv~_0x>I^N zOE>J`WiKNt>c1D5nm>O9%{$XYuKdZQUwtKYHr7KF0AlrU1@j5kPTIu}3DFUslq(`! zTCKFTHX8;mC1;2m?1{%d2{;?<@WVytgA{lGfVpG0&$bVLaRwdxgb?@KF$c%Erw+LF zZ@t0@eat}F?jXaTaSR>S`nnB+vtatS+XpJ&dxRj@+%gA`aD*HOxI_*;!Vq+Ogb?}e z5JSL!5rl;Ydkl8TRLjaWFqfq$v_F)1pm)S|dM;ixN0Fj{e>&_pe+YnGwbRw)a|;(1 z+8`b3?x|COEOYxH{4~tJoKLTR1URz=IDQ*vSNGUqbcPO~Q2Iba(Kx|v7tD3>XNqxu z|9^i7Cc}jW5)3-0%T!C11Y~AsJ<^4|^Yrp5JM2o`uVh5zbMI-!K)bP8e}4*P#5kj_ zg+PBfZa1Sem3!HlV(hkDr8b*|lpB9K+xUJMX(3US+byFq9Wo*tSS=oK&)9S@HE^;a z!9&lUL(JZroh=%F0|-fAo6S$&shPB_qCI~b7)u{X`g$v%qFzS>5CW^fTOsM4D=l&X2)6Ga6eDnd2NXJ? zQPJ8q-8_276fF`i-)pQsjLSsqpRwD20=Gl4d_>8^#cXmEnRbA96HogXf8&N7GzkM> z|9=)@Dv5z>3hrSK*3It>8vz?f+SJ8Nl=f~cz=r=)Pk$L!NE#Z5koPGNG+QnzTXdc% zXPwtMtENX9gJH*WkG1NDEjAeSEQ*d5?$DGK1LPD-`41+~A+t_zWq*GI1xG;QwF_!P z9n&9m^DY$MW4fhXhj4#oLq^Ge1p_j15aZb+5vJ!fMc@YmzEve|o6|9yky@)iOa5?w ze+vML)jI!3Irom;Mp+wbRyjJKd=!ZnQ-U3X--UmF1;bO~11xz?^3da8xzby~*Gp1V zQ}=UEf-V1le*i|B)qT5?tW7XlvI`l58`LOJ?NO&i!O!l82Y-JB&X)%j_NWHW6wjv? z*mpx7e+F~g+){`c=2==Jf(HgS)5Fb&wFKT?LQ)*{_VKct zj+C6D7*0@(KV2dB$^k^X%|IC(q8LALEk-C@pcHN$7=Aoca+px~@lnRgu2GB^D8&l) z-$ssc2%~ZGmxl!tkC=R;6g@&E-J6O0c&FpVKOQOhMmU_K5MI7hYm@^G$^k{|#Wwwe zjg;QtS(HJz>5au0F3R%fdbXf5byt za#qsYr&}B+=kLj7{ai@23nn8gx2(T4jd|dx14H?Q>!`gjB{)+;YmcDkS! zI$1k-4W`FB^37sfUz|3de+Y7IYSh%AotUG?bo2z9Q~e{!BUlz{9}qt|1{GrxpZ{(a zb5T(cez(uzq8KZj)lAY(UX&Zw=Wuu($bS(yvfkf2XPijPPg(ZQ_Oar`vSly5iv0PE z(O}5DFee0q9vuSGKSn_tPYTP=W%gHpd@{Rl!M?~BtuCY`5kU%UNc>GA{zc3@QNg5m zzdKAKWo#cRfr+<+BNt4JHb&cV7(Oq{_GBX-VnY-Z%F)ULkZCqJSJBHYo~N>~JBO|i zlIkZ-KmIsuT}VtIJrg$&5A2mJm}B~N>VmsItUF$PKWyHTkz!xKS07ct%HI-D;p+wl z|7upLf1ii#EAT(+ql(XioUJ)3mu;(a#4~?pu~}5Vg}(EbxtHSC_yOGl%>wF6+s)0* za0+k06^VMJTCvtA)Wf98FXHbxM!iI)aSMi%9cQkl5>#EtCG^K_Q|`_>->x|Lml&+` zn2A>&)`E<~@`^E>PiDxS5uqQ~AMlhKP20EVF%uT6IpB&%gf&=4TkUp#E^aKtPC!aK zoozyia}C~yudUt;osZ1rFw2uM{|vajt#ApRE22*+51d*tq7Sbit6E8|L3L%2(SIx{ z$6guoNkp&<1{5F)cA`)!O&-`O&PsCKgD3DTtM->TJ$(OCCW8`xVz-BBL;rtXE-Qu05TBuo#h(uQFeyQw zw4qdo3*OF~Fy2|o=$fwAq6_9Tc-&39%|o3Jqkjl|EJ;Jmrs7?f`m`jL**41ma-4M$ za*a*HA{a@nH&{2ep|QbpbS~qkkdb2zP~H7I$ZcEq48qSE#9(b5aQLI-MevOzB_A(F zT~kiKukho~UwZ|;KzUC>PR98*NDa>QOoI=N*>k;2GaD>y__WucsF6Q_5u0?$&NM&9 zNb;#i51Tyt%!6C{u*?rh0_YD5(Ff5t;)CBgBCKyawwKQ z&B%Ev;3(_7J(+wM8`y$?WL=4iFyeoqhDnp;>{wqvrOz%AD@8|{X+33OMn>)Szc}i) zSb*;bl}$IDqUHS4uS9Cs!bfi%nwF7Xo6;`VGY^e zYjSDb;8k(AX#Xt~4^D2YcLZFdg!=-wp_}`I&(IgLHUVVH1~=(>+8w)T%!!gstagqM z+kqA*nwG;nei)MX5z$NU-*bC?#1E5qRvgP4isJL=?M@&C>^o=Wc33Io*q?7YgfX(t z|9`IpUvTw$%z}^ zAQ^kWYmLG^X~zbf@BtU4&j?|0yf1Bgt4yj5xUx9zo4-mQoJseWSL6=hXM!IA)$)7M zcqg7@66&p@Z1mEQocx&1&G1m+;lcB8<$U=f#;vkHXH)*ZPBFL@0w-d^hMOwT!LYsZ zYegFO-oif`sCmtKC1X5r$gwT;%N%t8f?Y%Kl%$K3^Ou+@ulc{78@DVP^?|r;Z%+vz z>(MvDmWHG3*g9CEL0rj3K=d1>EK#3ZTA~%MjtxkT*PQ8%qXf4I3Fxw8BsrRIRqX14 zXyvbG!sgh&{o;)1?HLldAHA)A2ghr<$SJoQfi_99_$UDDNOZDOJLxzxZVZRRXTFTR zcM-h(PnTg?S0}c@^S?efv%WVq!fTIzGoq(i?zVr9&3#q)?DP1FK8oHhI%!$clL_{K z0Rx(E1=?5$jy)`K-vq*J+3NmiHZ=l95<>9xaroF#XKBa&56+zbf9B2~^Wzrm!TR4h z$}`0^mv}|`YHo}i=Cc^%lg5mrd77n6)p>cJOIwEn1+p8;)^J+=*IK<{>l2CVeEaN;hF zGkgMfbEiApB!!cmS&!pq)YnWq#`u2@3mACcUA)HHdpuZQ;M36g^t?6QARKxB1=9S; z+k?V&=C)T%d_rF|L&qzM^aU!?yfnd-{O zi~+j^=L@43B3kbGRoH?5e|Ap<=JC`9;9oZO1d_Cj{J1CdF(6Y*y!&;DN> zb5Bp+?a_TNu<5L#hx~tg*a~}P1m1D5-}Q?+A$+N;lZPY?1>v@RwuPs-v>!81gJcD^ zI$03A?yrV_q2(q}=YBA-G&N}Lf)%@*1}D95vyL5iZ@)lT03{3sD}~KbS`Q(ACu=_# ze-tU~3z+1NP8>tu=+4geSHS*#NIZ)lNr{7@Hq3Dl63xz64DChW@Mp$I$5FqjNwXkN OC&}z62Hw1xzyH}LI5!yp literal 0 HcmV?d00001 diff --git a/crates/j2k-native/fixtures/multi_segment/termall_rgb_53.j2k b/crates/j2k-native/fixtures/multi_segment/termall_rgb_53.j2k new file mode 100644 index 0000000000000000000000000000000000000000..ff43b510e16956d25b10d43c149045211e8d3e2b GIT binary patch literal 6127 zcmXX~bx;)i^FF$}jy$^IKpKGq>5>L%1d)=K20wIn2+}F=(h4GZfRr>+g2>S&(vmOm z^PS&!=h>NO_K)4!*=J^V_Ce=C4+4k$XQBV||M-6v`al1VLm(J9XlVbB2Ym=31Ra8b zhK`PoiS}Rt!Izg;Q`7y6hZhhI2%4go{qxuMFPLoum~}kt-86KSRhfP5y}X^=-IyN< ziV2E7q(bojOGy&|L3_y9mMI)z`&Wxx+?}(gH)wGbvP#hW*&@XZyi8i&}FGu%g;cFYd|JibYZs^UcHCruCRo9N~~ znY;G80zi2)_MNL|{nz>q7afNba$G zo5R-Vw+w`E$shSAM>1#_!e@XY^Jnt0A16L9%l?MgzjU?b431wrd{*Q(FySlYqgweb zVjEoXP2Q(K9HzzHYm7-$a90WMRQbZ$J>wVUV1IF6WG2$2F-@hfY=i~u!LJFT} z!9jo|np)0hAH5#H(3}i};i7S3S_?XWre5IHf$KtTMt4;eQz-TRKIS8$X++WOM}@bQ zp9n)`#O}#_o8Gzd@9Yc-tvO3*w zn}Co6cK{&w1IKB329ISsX+iOKlW2u+m_fUKMX5-#Yw?3;!QtolW<#n^k%OT~%90p- zhUR?1quU)}a~yes-p`z67r`=2K`?O+$nV$(RTrUI#wcyfOh7y6{{2@V!`5C&#N`>T zuoufz9paG}buHtI|3*mEVg8OhOKkDP^)P2LzxFw6kK?6a*Y@3#Q|ND@0dbGpg15^)?bhtVXFIG9PqAf5l*nI65y zU_62r{?oSTSEVO>zTE$(aq1q$|1K;*IyU;{4K10ZGCJ2KU`r}}3l0l;6w^`%szP5p z8N&?jdX!nxW&WYUx-sR6#lR9IhAkkqz@MSIyEZ{G_gc^xM*S^{I3l+GtM$7kbo1o6 zbh=e}qNfT!bM9L!UyZsp42nn(zncEK$9GkE-lk`rAV=}zeZtqKu{6)ygfN*LqC-MI z5y%4|5kZR{fhbMz^ka|4ooi zyA#|yl$|FUw34b_4YAitfHv0w;vUaQG>FrQ;HZzR^O0X6S7GO8^sR zW1N#Buf4P3yXfLC{2}zYg#sf_78OE9NNM_FArWXfnZJg_H}rW!v77gfTl~q`Oa@zX zCIIc2e|^nKc?CgK*c^x5G&`@!$mF}|2kt34T)>W1_D(xQ03yNZ!U{s2+56*w8Lw~*dbh?c5Jih_( zq&XaQcP~`$$ZVkYT$~4b&d|^a%O}CGL6^pBV>bGa9Zrm&FyFF6==4=p7YkO8NEpAeE}LFX-_5}ocN%~I2QoOQOo8#_F{|EM z&xW4m5!r^hqu3!|tc%(6{#YBNLbX^I_(X{>_;itwMO{XouH?ZZTQkcAP2ekxddnxy=9itLf|hiFT^jB%YW|CJKkl zUyz=-O!L==JQ4G~XK~}7E*{)SBj$Jgu)&a1=eHhb$vW#y#=;!aR|Z{W$#r`1uJE6=5qN ztIisv)JRx|r$qjhyFmoTQmmp%r-|7z zX_|xFYKT%#B!D50K%PG!e3t2Yxn#&7NI<}+USIqOD`?usg!T;GaM;Kmt79&xQGX`X z+wy8U_Vo%6*ZH@Z9i4n@bEaCfsFWNDWBFR3Ep-X7J0yb?~??(di?+(XtG`b)HcUWHTM9^bqLOWiRt;?$oxZ0%%P-y z^gr6dZQYc-r=_e505jLHTkJ=k?oju9JU*2v2c8Il5-$!xS_)Bf{f%1nkQ>c0?y|`< zMWlneQSA8`d(VV$kz*EVjh(<0*3k@IRFEDT2ZeDc?_G_upudONU}tZc@Xp zQY#gkQvfEu@9dt{hG0e>iyzr)LY_bkrK>ggs58*>!p+no$(*RX26_raF;>dPqw z8EcxKL~?Db>*^`8#F05a>AZYk_a{cvI;=(qxK*oob&2QH9ST_@`UuIA;dmZ|(g3V` zB=@^3tD~I-T6f?fipCi#9QXNnFGbDIL`o|0-Wshn%!*<=5VCxRLp1Xt4Y0nLOn-!` z{PJ|ph0k?|buFp(&1!Nxv_K~1+l8q5*rBV^peK^2@jwbx;d7u`y zvJ0Q?cVY3-W#;Jd+Ww?sS5?E%=eY9a0Kr1ve)tby)}OA(s*Q%l630fVpB|I#rpgSz zLKvm{WgiRMQE~|%5CgA2S`R)!xd9Y1y@8${en7fLyjsO$jl|(3D)Ps&Ol7}FS$K}S zaoiKFKm^T7n(&W0m|RP@IDlJ3lV!0vf5K{YTQo2SqUqR{aX=y7%hMqvQDP~h=v)T}a*V*AhM$bzKWXP;K3iM!r^|Add?w34G zRIh(Y@|orjBs~e0ZYIOylO3^-l##vNw96yYdMII`>wqlfO27*hS9}|;SL(d$w=;PE z=!V%(*w6l$+<1ELJe8bxB6T$BEbjcn9ZMcol3!iQSqwII#DkPx-&;T5`oi{oKbHBmN`@#QT;O7USp5kGwj*7 zSb_gA{~HY1b|b*RWyN|KT1&4q8ME%D({{o119pb-n2J`Jz>0H!>XLc~i z1Xn&`!3G>svxDG69Cjlhf;2*)4Pf`Z?}26<&EDygSb2rWoyw9j;Y?J$ej^#X(rYs| zuBU~fF?Zm7I$7^OdfROc_!UT$I-ZX4EH0Kyob1B(=S19Ey|bwndIyh6=DuMu_6dFL zIyRtWjgwc`8<$SxkpkJ!M(OVk^Y~_^Ym4y=&6L_B!)yBb>2SQ%7Hd9eWolBfEprdc zV|OVmz_h$Z03eb3?rla!sY(^n^fDZ(i#s3Id~uDind#SZ%9Gng`($N51d@;P%sb-g+z0wzaeU-B)4<|RUB}D3ysdav(Psg z_wxHpURhwwzj5Y>WayMAI-R!Y&DI#ogw_Lc>%Zl>SZh*w8#BW#w7ayE#id^C@7VCH z*LJmbm~$rf#Gq4^ow#GnsV<0N`(b~UZtS#@snrx`^joUE3n9LY`)CX zkH`S=EL5}Sf|P^Jzs&V!+4FZi1~Kh`+6DR%O|62yepV#XMs>9z(X#kdw?I)1Xm5uW zAZQKxwjwLM(b)iANE-#9M4VBf7c4&(!zL}XDEwu_MM1!L+R&eE`NOgE`4E$;4m!@`MYLSqzOX*ree_;GHWQ3w81TyPkh`BX~2PganKxEyh>x=UA#z z-?FEuQ2`D~Ln+NL_s)x$Wod1DmP@pe2AC-Ykyc@YP-^k$PiDLH{-A`mJ+yN`9<-nU zH8brr&5_6?Zn643S`PrgkIM=Cg!gf~(IY7~IkMn#+R?SyjP)Rbg4d1&$>67v*r6lE3x`7)@{Cnb z1N@Q=y}tC5JyI5ER&MJ7l1n|^#vhPWm%1))H%g3Ck<_ij>;YC3x#PDg2=%AW?jqjA z9$-n2I+H(|p21|FCmImn!bX>{AlH~}{aLx_FWuI`C`NHh=iU$`N-n6KyS2kIWQnpV zK8sC}jsb7_(vo!HUUc*U$^?5A`7(Za=ks13&L;&7T?4v)+gcbiHqHwB67H_;OX7bw zp1rxJR6PCs6hz9m-0;MY-@bFd2)=g9dr}dEEOI2MSI+suk1RGWD|+RzEq|pYqk#k> zIVB1@f#16BQ5XUjhw?vWFvLx=K?HS1LR1hy<=btmlst73{`*J>{5qdOWnGYVwVQ+e zuEN?0TYmAziOW!d35rVqfVH|R{P|2!2D4PM*XbWY#dVKYTW8$8J@=Dn^t;5WD=4K8 zB$U8;;(h3!+Z&ck@*%D@SP{{Y-CLG(TxPdtB^TayCqLKo(E=vD(W6)|CNUF^c~+JB z@bZD;xJa6sr(`CV)AJElcR=vRVriguMm?+FTXFo{MJ?e=AFEoLcVXZug0i#OetCPU z>|JkT*nok)V6JQ|O82utGyq{xD5}{t+v)9PR^n+~P4*l>rTTJ)*kbsRp$3h}Q#B~D z5g@Mc@bwk89tb1nc2d5M*KGDmj7Q?VA!=u+^g`!Gj7Ja?D{?^hFX|n7f{Y_dqemYA zd;33aAK%UWPVAM~x)(7DD@G2G-WtBw&_rXT@!6PKHg;ZR;)D>-WLFBD|0(YJ z2E$R$|1Q>G{H%hMhbNDJ)i8bR_rl7FK0V%?+)IYnn+ZI%tll*RnRh66rB{6hCSf$& z?ZNnROAgtj4yhs&-qJ1R;#*0~Eu{!pEW{-8^D!3;+a6OKe1!Xz-L|FRTMV%S#P1R-=~qhEB&=aA>iY?;DNCwI@l zp%O-5wJdh&CzamzPkGq4*!C`=>F$peZCz>g$R3P26GP>LQ`!)sL&L14+s+8+bV%2| zaf(q=i46%bwi95^RL*}CrWq5ec>i1uX8QO|=aA-m$4W|{IKVJjrZ5jji&y8-kGZbM zb71>3>s4QSd}lrB0tk}IZ~5YH!fDA2yKHl_n|zsi#swM@oGs}I5vCem`62-DHr1p! zKLjBY_7`w^I}CE&bjnZr-W}~;Tt&eF9-R5T#=KFij{oMsY)T3CM-zkd$Mc693a*c5-(XPZ1xl)uSSn*@O$no zf&c{aXku+(qDEQ1b{hzjrx;_LEF*ITWwd@ zxS%I+=9W1}+<0F48Ym_Uw;&uwGc^9n7vr;Z;c|430JiNT`q}E4Er}cUo}d-K>n}qX zrP~(y4E(oC^!09o9j@JbudaijYN4}L&)5&-PBI2koq<3ddjg;b`PBxdY2sVO= zpyY2WW>5IX;+xHRGlT3_4xJ5#-lW{%K?9gfVaATq`KGAPxH4-XM0E0pW|0=U+d_A3 zx1HY1LKNPAwEez84?8{MMJB( bail!(TileError::Invalid); } ensure_tiles_share_sample_type(tiles)?; - let (structural_workspace_bytes, reused_baseline) = choose_region_idwt( + let (structural_workspace_bytes, reused_baseline, cpu_decode_parallelism) = choose_region_idwt( header, tiles, ctx, @@ -348,7 +348,6 @@ fn decode_parsed_tiles<'a>( let retained_decode_base_without_scratch = retained_decode_baseline .checked_sub(reused_baseline.scratch_capacity) .ok_or(ValidationError::ImageTooLarge)?; - let cpu_decode_parallelism = ctx.cpu_decode_parallelism; let (tile_ctx, storage) = (&mut ctx.tile_decode_context, &mut ctx.storage); // Under Auto, the tiles may decode at once on worker contexts. On any @@ -571,7 +570,12 @@ fn region_decode_without_hook_or_special_samples( /// if it `wants_rows`, whether it streams them (see [`streams_rows`]). A /// decode that streams gives up the context's retained capacity first (see /// `release_retained_before_streaming`), and the decode goes on with the -/// returned structural bytes and baseline. +/// returned structural bytes, baseline and parallelism. +/// +/// Rows that stream only from serial bands, under `Auto` because the pool's +/// bands can't take a tile, stream serially when the region's planes would +/// pass the decode budget even in a fresh context: the planes decode would +/// be refused, so this costs no decode that succeeds today its parallelism. fn choose_region_idwt( header: &Header<'_>, tiles: &ParsedTiles<'_>, @@ -579,7 +583,7 @@ fn choose_region_idwt( (hooked, wants_rows): (bool, bool), structural_workspace_bytes: usize, reused_baseline: reuse::ReusedDecodeBaseline, -) -> Result<(usize, reuse::ReusedDecodeBaseline)> { +) -> Result<(usize, reuse::ReusedDecodeBaseline, CpuDecodeParallelism)> { let line_region_idwt = ctx.line_region_idwt && ctx.tile_decode_context.line_region_idwt_permitted && region_decode_without_hook_or_special_samples( @@ -588,21 +592,57 @@ fn choose_region_idwt( hooked, ); ctx.tile_decode_context.line_region_idwt = line_region_idwt; - let stream_rows = wants_rows - && line_region_idwt - && streams_rows( + let streams = |tile_ctx: &TileDecodeContext, parallelism| { + wants_rows && line_region_idwt && streams_rows(header, tiles, tile_ctx, parallelism) + }; + let mut parallelism = ctx.cpu_decode_parallelism; + let mut stream_rows = streams(&ctx.tile_decode_context, parallelism); + if !stream_rows + && parallelism == CpuDecodeParallelism::Auto + && streams(&ctx.tile_decode_context, CpuDecodeParallelism::Serial) + && planes_pass_budget( header, - tiles, &ctx.tile_decode_context, - ctx.cpu_decode_parallelism, - ); + structural_workspace_bytes, + reused_baseline, + ) + { + parallelism = CpuDecodeParallelism::Serial; + stream_rows = true; + } ctx.tile_decode_context.stream_rows = stream_rows; ctx.tile_decode_context.streamed_samples = 0; - if stream_rows { - ctx.release_retained_before_streaming(structural_workspace_bytes, reused_baseline) + let (structural_workspace_bytes, reused_baseline) = if stream_rows { + ctx.release_retained_before_streaming(structural_workspace_bytes, reused_baseline)? } else { - Ok((structural_workspace_bytes, reused_baseline)) - } + (structural_workspace_bytes, reused_baseline) + }; + Ok((structural_workspace_bytes, reused_baseline, parallelism)) +} + +/// Whether the output region's component planes would pass the decode +/// budget beside `structural_workspace_bytes` without the context's +/// retained capacity, as `reset` charges them; planes too large to count +/// pass it too. +fn planes_pass_budget( + header: &Header<'_>, + tile_ctx: &TileDecodeContext, + structural_workspace_bytes: usize, + reused_baseline: reuse::ReusedDecodeBaseline, +) -> bool { + let (width, height) = tile_ctx.output_region.map_or( + ( + header.size_data.image_width(), + header.size_data.image_height(), + ), + OutputRegion::dimensions, + ); + let fresh = structural_workspace_bytes + .saturating_sub(reused_baseline.channel_capacity) + .saturating_sub(reused_baseline.scratch_capacity); + planes_floor::region_planes_bytes(header, width, height).map_or(true, |planes| { + planes.saturating_add(fresh) > tile_ctx.planes_budget() + }) } /// Whether a region decode on the row-streamed IDWT can hand its samples to diff --git a/crates/j2k-native/src/j2c/decode/planes_floor.rs b/crates/j2k-native/src/j2c/decode/planes_floor.rs index 684a47e6..7fde899a 100644 --- a/crates/j2k-native/src/j2c/decode/planes_floor.rs +++ b/crates/j2k-native/src/j2c/decode/planes_floor.rs @@ -75,10 +75,17 @@ pub(crate) fn full_decode_planes_floor( /// What `reset` charges for every component's plane of the output image. fn planes_bytes(header: &Header<'_>) -> Result { - let samples = checked_decode_sample_count( + region_planes_bytes( + header, header.size_data.image_width(), header.size_data.image_height(), - )?; + ) +} + +/// What `reset` charges for every component's plane of a `width` x +/// `height` output. +pub(super) fn region_planes_bytes(header: &Header<'_>, width: u32, height: u32) -> Result { + let samples = checked_decode_sample_count(width, height)?; let padded = SimdBuffer::::padded_len(samples).ok_or(ValidationError::ImageTooLarge)?; let components = header.component_infos.len(); diff --git a/crates/j2k-native/src/j2c/decode/reuse.rs b/crates/j2k-native/src/j2c/decode/reuse.rs index ea05e8cb..6effc87a 100644 --- a/crates/j2k-native/src/j2c/decode/reuse.rs +++ b/crates/j2k-native/src/j2c/decode/reuse.rs @@ -159,6 +159,17 @@ impl DecoderContext<'_> { } impl TileDecodeContext { + /// The decode budget the component planes are charged against. + pub(super) fn planes_budget(&self) -> usize { + #[cfg(test)] + return self.planes_budget.unwrap_or(DEFAULT_MAX_DECODE_BYTES); + #[cfg(not(test))] + { + let _ = self; + DEFAULT_MAX_DECODE_BYTES + } + } + pub(crate) fn retained_channel_bytes(&self) -> Result { retained_channel_bytes_with_cap(&self.channel_data, DEFAULT_MAX_DECODE_BYTES) } @@ -245,7 +256,8 @@ impl TileDecodeContext { let fresh_baseline = retained_baseline_bytes .checked_sub(retained_channel_bytes) .ok_or_else(allocation_overflow)?; - let mut budget = ContextCapacityBudget::from_live_bytes(retained_baseline_bytes)?; + let cap = self.planes_budget(); + let mut budget = ContextCapacityBudget::with_cap(retained_baseline_bytes, cap)?; let reused = reset_channel_data( &mut self.channel_data, &initial_tile.component_infos, @@ -258,7 +270,7 @@ impl TileDecodeContext { Ok(deferred) => (deferred, budget.bytes()), Err(error) if retained_channel_bytes != 0 && is_capacity_error(&error) => { self.release_channel_allocations(); - let mut fresh_budget = ContextCapacityBudget::from_live_bytes(fresh_baseline)?; + let mut fresh_budget = ContextCapacityBudget::with_cap(fresh_baseline, cap)?; let deferred = reset_channel_data( &mut self.channel_data, &initial_tile.component_infos, @@ -514,10 +526,6 @@ struct ContextCapacityBudget { } impl ContextCapacityBudget { - fn from_live_bytes(bytes: usize) -> Result { - Self::with_cap(bytes, DEFAULT_MAX_DECODE_BYTES) - } - const fn with_cap(bytes: usize, cap: usize) -> Result { if bytes > cap { return Err(DecodeError::AllocationTooLarge { diff --git a/crates/j2k-native/src/j2c/decode/state.rs b/crates/j2k-native/src/j2c/decode/state.rs index 78a8f7a3..e186c29c 100644 --- a/crates/j2k-native/src/j2c/decode/state.rs +++ b/crates/j2k-native/src/j2c/decode/state.rs @@ -300,6 +300,12 @@ pub(crate) struct TileDecodeContext { /// their per-thread counters see the decode. #[cfg(test)] pub(crate) enter_pool: bool, + /// Test-only decode budget for the component planes, in place of the + /// default: `reset` charges them against it, and a region decode + /// compares them with it when it chooses whether its rows stream from + /// serial bands (see `choose_region_idwt`). + #[cfg(test)] + pub(crate) planes_budget: Option, /// The raw, decoded samples for each channel. pub(crate) channel_data: Vec, /// Optional output window for region-local decode storage. @@ -365,6 +371,8 @@ impl TileDecodeContext { let serial_small_decodes = self.serial_small_decodes; #[cfg(test)] let enter_pool = self.enter_pool; + #[cfg(test)] + let planes_budget = self.planes_budget; *self = Self::default(); self.output_region = output_region; self.round_irreversible_output = round_irreversible_output; @@ -384,6 +392,7 @@ impl TileDecodeContext { self.staged_rgb8_pack = staged_rgb8_pack; self.serial_small_decodes = serial_small_decodes; self.enter_pool = enter_pool; + self.planes_budget = planes_budget; } } diff --git a/crates/j2k-native/src/tests.rs b/crates/j2k-native/src/tests.rs index 120b9109..31b79b64 100644 --- a/crates/j2k-native/src/tests.rs +++ b/crates/j2k-native/src/tests.rs @@ -11,6 +11,8 @@ mod ht_refinement; mod idwt_into_channels; mod idwt_window_storage; mod line_region_idwt; +#[cfg(feature = "parallel")] +mod multi_segment; #[cfg(feature = "std")] mod packet_directory; #[cfg(feature = "parallel")] diff --git a/crates/j2k-native/src/tests/multi_segment.rs b/crates/j2k-native/src/tests/multi_segment.rs new file mode 100644 index 00000000..31262952 --- /dev/null +++ b/crates/j2k-native/src/tests/multi_segment.rs @@ -0,0 +1,309 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Classic code-blocks of several codeword segments (selective bypass, +//! termination on each pass, or both) and the other code-block styles +//! (context reset, segmentation symbols, vertically causal context, +//! predictable termination) decode on the worker pool as on one thread: in +//! place on the tile route, and band by band ahead of the streamed IDWT, +//! whose rows then need no component planes. +//! +//! The fixtures in `fixtures/multi_segment/` were encoded with `OpenJPEG` +//! 2.5.4's `opj_compress` from two synthetic images, 128 x 128 and 256 x 256. +//! The unsigned 8-bit red, green and blue samples at `(x, y)` are +//! +//! ```text +//! r = (7x² + 13y² + 5xy + 31x) mod 256 +//! g = (3x² + 11y² + 9xy + 29y + 64) mod 256 +//! b = (13x² + 5y² + 3xy + 7(x xor y)) mod 256 +//! ``` +//! +//! and the gray sample is `(r + 2g + b) / 4`, rounded down. Written as a +//! binary PPM (`P6`) and PGM (`P5`) of maximum 255, `rgb128.ppm`, +//! `gray128.pgm` and `rgb256.ppm`, they encode with three resolution levels +//! below the full one, 16 x 16 code-blocks, RPCL order and three or four +//! quality layers: +//! +//! ```sh +//! c="-n 3 -b 16,16 -p RPCL" +//! opj_compress -i gray128.pgm -o bypass_gray_53.j2k $c -M 1 -r 16,8,4,2 +//! opj_compress -i rgb128.ppm -o bypass_rgb_97.j2k $c -I -M 1 -r 32,16,8 +//! opj_compress -i gray128.pgm -o termall_gray_97.j2k $c -I -M 4 -r 16,8,3 +//! opj_compress -i rgb128.ppm -o termall_rgb_53.j2k $c -M 4 -r 32,16,8 +//! opj_compress -i rgb128.ppm -o reset_rgb_97.j2k $c -I -M 2 -r 32,16,8 +//! opj_compress -i gray128.pgm -o segmark_gray_53.j2k $c -M 32 -r 16,8,3 +//! opj_compress -i gray128.pgm -o bypass_termall_gray_97.j2k $c -I -M 5 -r 16,8,3 +//! opj_compress -i rgb128.ppm -o bypass_termall_rgb_53.j2k $c -M 5 -r 40,20,10,6 +//! opj_compress -i gray128.pgm -o all_modes_gray_97.j2k $c -I -M 63 -r 16,8,3 +//! ``` +//! +//! and, as archival images are often coded, with selective bypass, five +//! decomposition levels, 64 x 64 code-blocks, precincts, packet lengths and +//! seven layers: +//! +//! ```sh +//! opj_compress -i rgb256.ppm -o bypass_rgb_97_64x64.j2k -n 6 -b 64,64 -p RPCL -PLT -I -M 1 \ +//! -c "[256,256],[256,256],[128,128]" -r 160,120,90,70,55,45,36 +//! ``` +//! +//! `-M` adds 1 for selective bypass, 2 for context reset, 4 for termination +//! on each pass, 8 for the vertically causal context, 16 for predictable +//! termination and 32 for segmentation symbols. The samples contain no +//! externally sourced image content. + +use alloc::vec::Vec; + +use crate::{ + ComponentPlane, ComponentRowSink, CpuDecodeParallelism, DecodeSettings, DecoderContext, Image, +}; + +const FIXTURES: [(&str, &[u8]); 10] = [ + ( + "bypass gray 5/3", + include_bytes!("../../fixtures/multi_segment/bypass_gray_53.j2k"), + ), + ( + "bypass rgb 9/7", + include_bytes!("../../fixtures/multi_segment/bypass_rgb_97.j2k"), + ), + ( + "termall gray 9/7", + include_bytes!("../../fixtures/multi_segment/termall_gray_97.j2k"), + ), + ( + "termall rgb 5/3", + include_bytes!("../../fixtures/multi_segment/termall_rgb_53.j2k"), + ), + ( + "reset rgb 9/7", + include_bytes!("../../fixtures/multi_segment/reset_rgb_97.j2k"), + ), + ( + "segmark gray 5/3", + include_bytes!("../../fixtures/multi_segment/segmark_gray_53.j2k"), + ), + ( + "bypass and termall gray 9/7", + include_bytes!("../../fixtures/multi_segment/bypass_termall_gray_97.j2k"), + ), + ( + "bypass and termall rgb 5/3", + include_bytes!("../../fixtures/multi_segment/bypass_termall_rgb_53.j2k"), + ), + ( + "all modes gray 9/7", + include_bytes!("../../fixtures/multi_segment/all_modes_gray_97.j2k"), + ), + ( + "bypass rgb 9/7, 64 x 64 blocks", + include_bytes!("../../fixtures/multi_segment/bypass_rgb_97_64x64.j2k"), + ), +]; + +/// Collects the sample bits a sink receives into region-sized planes, and +/// checks that each sample arrives once. +struct Rows { + width: usize, + planes: Vec>, + arrivals: Vec, +} + +impl ComponentRowSink for Rows { + fn write_rows(&mut self, x: u32, y: u32, planes: &[ComponentPlane<'_>]) { + if self.planes.is_empty() { + self.planes = alloc::vec![alloc::vec![0; self.arrivals.len()]; planes.len()]; + } + let (width, height) = planes[0].dimensions(); + for (component, plane) in planes.iter().enumerate() { + for row in 0..height as usize { + for column in 0..width as usize { + let at = (y as usize + row) * self.width + x as usize + column; + self.planes[component][at] = + plane.samples()[row * width as usize + column].to_bits(); + if component == 0 { + self.arrivals[at] += 1; + } + } + } + } + } +} + +/// What a decode wrote, and where it ran. +struct Decoded { + samples: Vec>, + streamed: bool, + pool_components: usize, + channel_bytes: usize, + /// The bytes the decode's budget held after `reset`. + baseline_bytes: usize, +} + +fn context(parallelism: CpuDecodeParallelism) -> DecoderContext<'static> { + let mut context = DecoderContext::default(); + context.set_cpu_decode_parallelism(parallelism); + context +} + +/// The region's rows, streamed to a sink where the decode can. +fn rows(image: &Image<'_>, roi: (u32, u32, u32, u32)) -> Decoded { + rows_with(image, roi, |_| {}) +} + +/// [`rows`] from a context `configure` sets up. +fn rows_with( + image: &Image<'_>, + roi: (u32, u32, u32, u32), + configure: impl FnOnce(&mut DecoderContext<'_>), +) -> Decoded { + let mut context = context(CpuDecodeParallelism::Auto); + configure(&mut context); + let mut sink = Rows { + width: roi.2 as usize, + planes: Vec::new(), + arrivals: alloc::vec![0; (roi.2 * roi.3) as usize], + }; + image + .decode_region_rows_for_integer_output_with_context(roi, &mut context, &mut sink) + .expect("decode region rows"); + assert!( + sink.arrivals.iter().all(|&count| count == 1), + "every sample arrives once" + ); + let counters = context.tile_decode_context.debug_counters; + Decoded { + samples: sink.planes, + streamed: context.tile_decode_context.stream_rows, + pool_components: counters.pool_band_components, + channel_bytes: counters.workspace_peak.bytes().channels, + baseline_bytes: counters.decode_baseline_bytes, + } +} + +/// The region's component planes from a serial decode. +fn planes(image: &Image<'_>, roi: (u32, u32, u32, u32)) -> Vec> { + let mut context = context(CpuDecodeParallelism::Serial); + let components = image + .decode_region_components_for_integer_output_with_context(roi, &mut context) + .expect("decode region planes"); + components + .planes() + .iter() + .map(|plane| { + plane.samples()[..(roi.2 * roi.3) as usize] + .iter() + .map(|sample| sample.to_bits()) + .collect() + }) + .collect() +} + +/// The image's native samples, whole or of `roi`, with Auto parallelism. +fn native(image: &Image<'_>, roi: Option<(u32, u32, u32, u32)>) -> Vec { + let mut context = context(CpuDecodeParallelism::Auto); + match roi { + Some(roi) => image.decode_native_region_with_context(roi, &mut context), + None => image.decode_native_with_context(&mut context), + } + .expect("decode native samples") + .data +} + +fn pool(threads: usize) -> rayon::ThreadPool { + rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("build pool") +} + +fn regions(image: &Image<'_>) -> [(u32, u32, u32, u32); 3] { + let (width, height) = (image.width(), image.height()); + [ + (0, 0, width, height), + (width / 5, height / 7, width / 2, height / 3), + (width / 3, 1, width / 4, height - 2), + ] +} + +#[test] +fn multi_segment_blocks_decode_on_a_pool_as_on_one_thread() { + let one = pool(1); + for (label, bytes) in FIXTURES { + for reduction in [0_u8, 1] { + let image = Image::new_with_reduction(bytes, &DecodeSettings::default(), reduction) + .expect("parse fixture"); + let regions = regions(&image); + let expected_full = one.install(|| native(&image, None)); + let expected: Vec<_> = regions + .iter() + .map(|&roi| one.install(|| (native(&image, Some(roi)), rows(&image, roi)))) + .collect(); + for (&roi, (_, rows)) in regions.iter().zip(&expected) { + assert_eq!( + rows.samples, + planes(&image, roi), + "{label} 1/{} {roi:?}: rows on one thread against the planes", + 1 << reduction + ); + } + for threads in [2, 4, 12] { + pool(threads).install(|| { + assert!( + native(&image, None) == expected_full, + "{label} 1/{} on {threads} threads: full image", + 1 << reduction + ); + for (&roi, (expected_native, expected_rows)) in regions.iter().zip(&expected) { + let context = format!("{label} 1/{} {roi:?}", 1 << reduction); + assert!( + native(&image, Some(roi)) == *expected_native, + "{context} on {threads} threads: region" + ); + assert_eq!( + rows(&image, roi).samples, + expected_rows.samples, + "{context} on {threads} threads: rows" + ); + } + }); + } + } + } +} + +/// A full-image rows decode on a pool whose tiles only serial bands can +/// stream (here the pool's routes are switched off, as for a tile they +/// can't decode) streams from serial bands when its planes would pass the +/// decode budget, rather than fail on them, and keeps the planes within it. +#[test] +fn rows_stream_from_serial_bands_when_the_planes_pass_the_budget() { + for (label, bytes) in FIXTURES { + let image = Image::new(bytes, &DecodeSettings::default()).expect("parse fixture"); + let roi = (0, 0, image.width(), image.height()); + let expected = pool(1).install(|| rows(&image, roi)); + pool(4).install(|| { + let planes = rows_with(&image, roi, |context| { + context.tile_decode_context.force_per_sub_band = true; + }); + assert!(!planes.streamed, "{label}: planes within the budget"); + assert_eq!(planes.samples, expected.samples, "{label}: planes"); + // `reset` charged the structural bytes and the planes: the + // planes pass this budget, which leaves room for a packet cache + // of up to half their size, which a decode past the budget + // drops to try again. + let budget = planes.baseline_bytes - planes.channel_bytes / 2; + let streamed = rows_with(&image, roi, |context| { + context.tile_decode_context.force_per_sub_band = true; + context.tile_decode_context.planes_budget = Some(budget); + }); + assert!( + streamed.streamed && streamed.pool_components == 0 && streamed.channel_bytes <= 256, + "{label}: serial bands past the budget: streamed {}, pool components {}, plane \ + bytes {}", + streamed.streamed, + streamed.pool_components, + streamed.channel_bytes, + ); + assert_eq!(streamed.samples, expected.samples, "{label}: serial bands"); + }); + } +} From d548621307b7ba84469890afa191adefca27c0e2 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 23:15:58 -0400 Subject: [PATCH 57/94] fix(j2k-native): reserve each pool thread's payload scratch once, as charged The tile route and pool bands charge each thread the plan's largest gathered payload, but each thread grew its scratch as it gathered: amortized growth from one payload to the next could leave its capacity above the charge (a 1000-byte payload, then a 1500-byte one, held 2000 bytes against 1500), and a pool band worker started a new scratch every time the executor sent it, so the scratch was allocated again for each send. Each thread's scratch is now reserved for the largest payload before the threads start, with any capacity the allocator gives beyond it charged to the decode budget, and gathering a payload never grows it: one past the reserved capacity is a planning error. The band executor hands each pool worker and each reader its own scratch with its workspace, so a worker the executor sends again keeps its scratch. Bytes are unchanged. A unit test gathers a 1000-byte and then a 1500-byte payload, each from two fragments, through one thread's scratch: its capacity stays the 1500 bytes charged. --- .../src/j2c/decode/parallel_bands.rs | 65 ++++---- .../src/j2c/decode/subband/tile_parallel.rs | 151 ++++++++++++++++-- 2 files changed, 169 insertions(+), 47 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index 2f6782d9..ed91804f 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -227,10 +227,12 @@ fn decode_route( )?; } // The executor's records live beside the route's plan and scratch while - // the bands decode. - route - .budget() - .include_bytes(executor_bytes(&rings, &bands)?)?; + // the bands decode, and so does each thread's scratch, the route's + // workers' and this thread's. + let mut budget = route.budget(); + budget.include_bytes(executor_bytes(&rings, &bands)?)?; + let mut scratches = + BlockScratch::for_threads(route.workers() + 1, route.gathered_len(), &mut budget)?; tile_ctx.idwt_output.placement = crate::j2c::idwt::IdwtPlacement::Coefficients; tile_ctx.idwt_output.coefficients = Vec::new(); tile_ctx.idwt_output.rect = IntRect::from_xywh(0, 0, 0, 0); @@ -262,7 +264,7 @@ fn decode_route( &rings, &bands, slots, - &mut tile_ctx.classic_task_workspaces, + (&mut tile_ctx.classic_task_workspaces, &mut scratches), |index, output, workspace, scratch| { route.decode_classic(index, output, workspace, (scratch, source)) }, @@ -276,7 +278,7 @@ fn decode_route( &rings, &bands, slots, - &mut tile_ctx.ht_task_workspaces, + (&mut tile_ctx.ht_task_workspaces, &mut scratches), |index, output, workspace, scratch| { route.decode_ht(index, output, workspace, (scratch, source)) }, @@ -424,6 +426,13 @@ pub(super) fn decode_tile_rows<'a>( } let mut output = RowOutput::new(tile, sink, tile_ctx, copy, region); let workers = route_workers.min(bank.saturating_sub(readers)); + // Each thread's scratch serves every component's bands. + let gathered_len = routes + .iter() + .map(|route| route.gathered_len()) + .max() + .unwrap_or(0); + let mut scratches = BlockScratch::for_threads(workers + readers, gathered_len, &mut live)?; // One reader for every component of a classic tile leaves the pool's // threads to Tier-1, and decodes bands itself while it waits. let helps = readers == 1 && coding == CodeBlockCoding::Classic; @@ -471,7 +480,7 @@ pub(super) fn decode_tile_rows<'a>( &rings, &bands, slots, - &mut tile_ctx.classic_task_workspaces, + (&mut tile_ctx.classic_task_workspaces, &mut scratches), |strip, output, workspace, scratch| { let (route, strip) = route_strip(&routes, &strip_starts, strip)?; route.decode_classic(strip, output, workspace, (scratch, source)) @@ -486,7 +495,7 @@ pub(super) fn decode_tile_rows<'a>( &rings, &bands, slots, - &mut tile_ctx.ht_task_workspaces, + (&mut tile_ctx.ht_task_workspaces, &mut scratches), |strip, output, workspace, scratch| { let (route, strip) = route_strip(&routes, &strip_starts, strip)?; route.decode_ht(strip, output, workspace, (scratch, source)) @@ -1147,7 +1156,7 @@ struct State<'c, 'w, W> { decoded: Vec, running: usize, freed: usize, - workspaces: Vec<&'w mut W>, + workspaces: Vec<(&'w mut W, &'w mut BlockScratch)>, /// Workers on the pool. active: usize, error: Option, @@ -1328,9 +1337,9 @@ enum Task<'c> { /// Decode `bands` into `slots` on up to `workers` pool threads while `run` /// drives the IDWT through `readers` views of them, then decode the bands it /// didn't read. Each view reads its own sub-bands and decodes a band nobody -/// has started with a workspace of its own, so views may run on different -/// threads. Views that `help` decode other bands while they wait. Returns -/// `false` when a band was stranded. +/// has started with a workspace and scratch of its own, so views may run on +/// different threads. Views that `help` decode other bands while they wait. +/// Returns `false` when a band was stranded. #[expect( clippy::too_many_arguments, clippy::too_many_lines, @@ -1341,7 +1350,7 @@ fn run_bands( rings: &[Ring], bands: &[Band], slots: Slots<'_>, - bank: &mut [TaskWorkspace], + (bank, scratches): (&mut [TaskWorkspace], &mut [BlockScratch]), decode: impl Fn(usize, BandOutput<'_, '_>, &mut W, &mut BlockScratch) -> Result<()> + Sync, run: impl FnOnce(&mut [&mut (dyn SubBandRows + Send)]) -> Result<()>, profile_enabled: bool, @@ -1351,14 +1360,15 @@ fn run_bands( #[cfg(not(test))] let _ = strand_once; let workspaces_needed = workers + readers.max(1); - let mut workspaces: Vec<&mut W> = Vec::new(); + let mut workspaces: Vec<(&mut W, &mut BlockScratch)> = Vec::new(); workspaces .try_reserve_exact(workspaces_needed) .map_err(|_| DecodingError::HostAllocationFailed)?; workspaces.extend( bank.iter_mut() - .take(workspaces_needed) - .map(|slot| &mut slot.workspace), + .map(|slot| &mut slot.workspace) + .zip(scratches.iter_mut()) + .take(workspaces_needed), ); if workspaces.len() != workspaces_needed { return Err(DecodingError::CodeBlockDecodeFailure.into()); @@ -1413,13 +1423,13 @@ fn run_bands( views .try_reserve_exact(reader_workspaces.len()) .map_err(|_| DecodingError::HostAllocationFailed)?; - for workspace in reader_workspaces { + for (workspace, scratch) in reader_workspaces { views.push(PoolRows { shared, scope, decode, workspace, - scratch: BlockScratch::default(), + scratch, current: try_filled_with(rings.len(), || None)?, next: try_collect(rings.iter().map(|ring| ring.bands.start), rings.len())?, helps, @@ -1529,11 +1539,10 @@ fn work( let Ok(mut state) = shared.lock() else { return; }; - let Some(workspace) = state.workspaces.pop() else { + let Some((workspace, scratch)) = state.workspaces.pop() else { state.active -= 1; return; }; - let mut scratch = BlockScratch::default(); loop { let next = if state.error.is_some() { Ok(None) @@ -1560,7 +1569,7 @@ fn work( } return; } - if run_task(shared, decode, task, workspace, &mut scratch).is_err() { + if run_task(shared, decode, task, workspace, scratch).is_err() { return; } state = match shared.lock() { @@ -1568,7 +1577,7 @@ fn work( Err(_) => return, }; } - state.workspaces.push(workspace); + state.workspaces.push((workspace, scratch)); state.active -= 1; drop(state); shared.decoded.notify_all(); @@ -1581,7 +1590,7 @@ struct PoolRows<'s, 'p, 'c, 'w, W, D> { scope: &'s rayon::Scope<'p>, decode: &'p D, workspace: &'w mut W, - scratch: BlockScratch, + scratch: &'w mut BlockScratch, /// Each ring's band being read, with its slot. current: Vec>, /// Each ring's next band to read. @@ -1695,13 +1704,7 @@ where /// will take, and hand it over as a worker does. fn decode_for_pool(&mut self, task: Task<'c>) -> Result<()> { let start = profile::profile_now(self.profile_enabled); - let result = run_task( - self.shared, - self.decode, - task, - self.workspace, - &mut self.scratch, - ); + let result = run_task(self.shared, self.decode, task, self.workspace, self.scratch); self.tier1_us += profile::elapsed_us(start); result } @@ -1713,7 +1716,7 @@ where self.shared.bands[index].strip, BandOutput::Whole(slot), self.workspace, - &mut self.scratch, + self.scratch, ); self.tier1_us += profile::elapsed_us(start); result?; diff --git a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs index 5a9439b4..4124ac80 100644 --- a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs +++ b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs @@ -39,8 +39,8 @@ use crate::j2c::ht_block_decode::selected_code_block_segment_lengths; use crate::j2c::tile::{ComponentTile, ResolutionTile}; use crate::scalar::decode_ht_code_block_scalar_with_workspace_midpoint; use crate::{ - decode_ht_code_block_scalar_with_workspace, try_resize_decode_elements, - HtCodeBlockDecodeWorkspace, J2kCodeBlockSegment, + decode_ht_code_block_scalar_with_workspace, try_reserve_decode_elements, + try_resize_decode_elements, HtCodeBlockDecodeWorkspace, J2kCodeBlockSegment, }; /// Below this many coded bytes per tile (about 45 us of serial entropy decode @@ -160,6 +160,32 @@ pub(in crate::j2c::decode) struct BlockScratch { payload: Vec, } +impl BlockScratch { + /// Scratch for each of `count` threads, its payload buffer reserved + /// before they start for the largest payload any of them gathers, + /// `payload_len` bytes, which the route charged per thread: gathering + /// never grows it. Charges `budget` for the list, and for capacity the + /// allocator gave beyond `payload_len`. + pub(in crate::j2c::decode) fn for_threads( + count: usize, + payload_len: usize, + budget: &mut DecodeAllocationBudget, + ) -> Result> { + let mut scratches = Vec::new(); + budget.reserve_new(&mut scratches, count)?; + for _ in 0..count { + let mut payload = Vec::new(); + try_reserve_decode_elements(&mut payload, payload_len)?; + budget.include_capacity_overage::(payload_len, payload.capacity())?; + scratches.push(Self { + clipped: Vec::new(), + payload, + }); + } + Ok(scratches) + } +} + struct BlockJob<'a> { payload: Payload<'a>, code_block: usize, @@ -415,7 +441,10 @@ fn record_route_peak( plan.jobs.capacity() * size_of::>(), plan.strips.capacity() * size_of::(), plan.band_sub_bands.capacity() * size_of::(), - workers.saturating_mul(plan.max_gathered_len), + workers.saturating_mul( + plan.max_gathered_len + .saturating_add(size_of::()), + ), counts.tasks * size_of::>(), counts.shared * size_of::>(), workers @@ -452,13 +481,17 @@ fn decode_with_bank<'a, W: TaskScratch>( ) else { return Ok(None); }; + let Ok(mut scratches) = BlockScratch::for_threads(workers, plan.max_gathered_len, budget) + else { + return Ok(None); + }; let mut coefficients = core::mem::take(&mut storage.coefficients); let stale = storage.stale_coefficients; let result = decode_strips( plan, &mut coefficients, stale, - &mut bank[..workers], + (&mut bank[..workers], &mut scratches), counts, PayloadSource::new(storage), |job, payload, output, stride, slot| { @@ -959,9 +992,10 @@ fn gather_payload<'s>( ) -> Result<&'s [u8]> { let expected = job.payload_length as usize; scratch.clear(); - scratch - .try_reserve(expected) - .map_err(|_| DecodingError::HostAllocationFailed)?; + // The thread's scratch holds the plan's largest gathered payload. + if expected > scratch.capacity() { + return Err(DecodingError::CodeBlockDecodeFailure.into()); + } let code_block = source .code_blocks .get(job.code_block) @@ -1222,13 +1256,13 @@ fn push_task(tasks: &mut Vec, task: T) -> Result<()> { Ok(()) } -/// Decode the strips on one worker per workspace, each block through -/// `decode_block`. +/// Decode the strips on one worker per workspace, with its own scratch, +/// each block through `decode_block`. fn decode_strips<'plan, 'coefficients, W: Send>( plan: &'plan TilePlan<'_>, coefficients: &'coefficients mut [f32], stale: bool, - workspaces: &mut [W], + (workspaces, scratches): (&mut [W], &mut [BlockScratch]), counts: TaskCounts, source: PayloadSource<'_, '_>, decode_block: impl Fn(&BlockJob<'_>, &[u8], &mut [f32], usize, &mut W) -> Result<()> + Sync, @@ -1290,9 +1324,9 @@ fn decode_strips<'plan, 'coefficients, W: Send>( // without forking a Rayon job and borrowing/returning scratch for each row. workspaces .par_iter_mut() + .zip(scratches) .with_max_len(1) - .try_for_each(|workspace| -> Result<()> { - let mut scratch = BlockScratch::default(); + .try_for_each(|(workspace, scratch)| -> Result<()> { while !failed.load(Ordering::Relaxed) { let next = queue .lock() @@ -1315,7 +1349,7 @@ fn decode_strips<'plan, 'coefficients, W: Send>( strip, output, workspace, - (&mut scratch, source), + (&mut *scratch, source), &decode_block, ) } @@ -1327,7 +1361,7 @@ fn decode_strips<'plan, 'coefficients, W: Send>( stale, counts.scratch_samples, workspace, - (&mut scratch, source), + (&mut *scratch, source), &decode_block, ), }; @@ -1766,6 +1800,11 @@ impl BandRoute<'_> { self.workers } + /// Bytes of the largest payload a thread gathers for this plan. + pub(in crate::j2c::decode) fn gathered_len(&self) -> usize { + self.plan.max_gathered_len + } + pub(in crate::j2c::decode) fn strips(&self) -> impl Iterator + '_ { self.plan.strips.iter().map(|strip| { let band = self.plan.jobs[strip.jobs.start].band; @@ -1905,9 +1944,89 @@ mod tests { use alloc::vec::Vec; use super::{ - route_fallback, split_runs, tile_route_fallback, CodeBlockCoding, Strip, TaskCounts, - TileRoute, + gather_payload, route_fallback, split_runs, tile_route_fallback, BlockJob, BlockPlacement, + BlockScratch, CodeBlock, CodeBlockCoding, DecodeAllocationBudget, Layer, Payload, + PayloadSource, Segment, Strip, TaskCounts, TileRoute, }; + use crate::j2c::rect::IntRect; + + /// A thread gathers payloads into scratch reserved for the plan's + /// largest before it starts, as charged, and gathering a payload larger + /// than the last never grows it: 1000 bytes in two fragments, then 1500. + #[test] + fn gathered_payloads_stay_in_the_scratch_charged_for_them() { + let data = [7_u8; 1000]; + let segment = |len: usize| Segment { + idx: 0, + coding_pases: 1, + data_length: len as u32, + data: &data[..len], + }; + let segments = [segment(500), segment(500), segment(1000), segment(500)]; + let layers = [0..1, 1..2, 2..3, 3..4].map(|range| Layer::with_segments(range).unwrap()); + let block = |layers| CodeBlock { + rect: IntRect::from_xywh(0, 0, 4, 4), + x_idx: 0, + y_idx: 0, + layers, + has_been_included: true, + missing_bit_planes: 0, + number_of_coding_passes: 2, + ht_total_coding_passes: 0, + ht_first_cleanup_pass: None, + ht_selected_set: None, + coding: Some(CodeBlockCoding::Classic), + l_block: 3, + non_empty_layer_count: 2, + }; + let code_blocks = [block(0..2), block(2..4)]; + let source = PayloadSource { + code_blocks: &code_blocks, + layers: &layers, + segments: &segments, + }; + let job = |code_block, payload_length| BlockJob { + payload: Payload::Gathered, + code_block, + payload_length, + refinement_length: 0, + placement: BlockPlacement::new( + IntRect::from_xywh(0, 0, 4, 4), + IntRect::from_xywh(0, 0, 4, 4), + ), + width: 4, + height: 4, + missing_bit_planes: 0, + number_of_coding_passes: 2, + band: 0, + }; + let mut budget = DecodeAllocationBudget::from_live_bytes(0).unwrap(); + let mut scratches = BlockScratch::for_threads(1, 1500, &mut budget).unwrap(); + let charged = budget.live_bytes() - size_of::(); + let scratch = &mut scratches[0].payload; + for (code_block, len) in [(0, 1000), (1, 1500)] { + let gathered = gather_payload( + CodeBlockCoding::Classic, + &job(code_block, len), + source, + scratch, + ) + .unwrap() + .len(); + assert_eq!(gathered, len as usize); + assert_eq!(scratch.capacity(), 1500); + } + assert_eq!(charged, 0, "the route charged the 1500 bytes; no overage"); + // A payload past the reserved scratch is a planning error, not growth. + let mut short = BlockScratch::for_threads(1, 1000, &mut budget).unwrap(); + assert!(gather_payload( + CodeBlockCoding::Classic, + &job(1, 1500), + source, + &mut short[0].payload + ) + .is_err()); + } #[test] fn split_runs_cover_the_strip_in_balanced_parts() { From f3b255c9ecd44ebd297a4ec0525ca96a5a6ba4e2 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 23:16:20 -0400 Subject: [PATCH 58/94] fix(j2k-native): charge the pool band threads' scratch once per streamed tile A tile whose rows stream from pool bands plans every component's route in one budget, and each route charged its own threads' payload and block scratch, although the executor runs all the routes on the same threads, each with one scratch that serves every component: a three-component tile charged that scratch three times. Its workspace peak, on the other hand, sampled each route alone, so it held one component's plan where three were live. Routes planned for that executor no longer charge their threads' scratch. The executor charges it once, for its pool workers and readers, as large as any route's, and declines to the serial bands when it does not fit, as a route that declines does. Its peak sample now holds every live plan beside that scratch. A route decoded alone charges and records as before. A test streams a three-component region from pool bands: the plans its peak counts add up to the three a region decode into planes holds one after another, and the peak holds all of them beside the retained workspace bank (without the change it held 41,688 bytes of Tier-1 workspace beside a 21,200-byte bank and 45,384 bytes of plans). The combined plan and executor budget test charges the routes' threads' scratch once, as the executor does. --- crates/j2k-native/src/j2c/decode.rs | 4 + .../src/j2c/decode/parallel_bands.rs | 72 ++++++++---- .../src/j2c/decode/subband/tile_parallel.rs | 111 +++++++++++++----- crates/j2k-native/src/tests/parallel_bands.rs | 50 ++++++++ 4 files changed, 181 insertions(+), 56 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode.rs b/crates/j2k-native/src/j2c/decode.rs index a1f7eacc..2a59b3dc 100644 --- a/crates/j2k-native/src/j2c/decode.rs +++ b/crates/j2k-native/src/j2c/decode.rs @@ -771,6 +771,10 @@ pub(crate) struct DecodeDebugCounters { /// bands held beyond the IDWT samples its ROI plan charged. #[cfg(all(test, feature = "parallel"))] pub(crate) uncharged_line_samples: usize, + /// Bytes of the pool band plans recorded in the workspace peak, summed + /// over components and tiles. + #[cfg(all(test, feature = "parallel"))] + pub(crate) pool_band_plan_bytes: usize, #[cfg(all(test, feature = "parallel"))] pub(crate) parallel_coefficients: ParallelCoefficientStats, } diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index ed91804f..72b0e010 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -393,6 +393,23 @@ pub(super) fn decode_tile_rows<'a>( } else { 1 }; + let workers = route_workers.min(bank.saturating_sub(readers)); + // Each thread's scratch serves every component's bands, so it is + // charged once, as large as any route's; the serial bands need none. + let mut thread_scratch = 0usize; + let mut gathered_len = 0usize; + for route in &routes { + thread_scratch = thread_scratch.max(route.thread_scratch_bytes()?); + gathered_len = gathered_len.max(route.gathered_len()); + } + let threads = workers + readers; + let scratch_bytes = threads + .checked_mul(thread_scratch) + .ok_or(ValidationError::ImageTooLarge)?; + match live.include_bytes(scratch_bytes) { + Err(error) if super::reuse::is_capacity_error(&error) => return Ok(false), + result => result?, + } // Rows reconstruct in batches, charged by the region's width (see // `roi_idwt_peak_elements`): two of them with a reader per component of // an HTJ2K tile, so one finishes while the next reconstructs; a classic @@ -425,14 +442,7 @@ pub(super) fn decode_tile_rows<'a>( .max(held.saturating_sub(storage.roi_idwt_charge_elements)); } let mut output = RowOutput::new(tile, sink, tile_ctx, copy, region); - let workers = route_workers.min(bank.saturating_sub(readers)); - // Each thread's scratch serves every component's bands. - let gathered_len = routes - .iter() - .map(|route| route.gathered_len()) - .max() - .unwrap_or(0); - let mut scratches = BlockScratch::for_threads(workers + readers, gathered_len, &mut live)?; + let mut scratches = BlockScratch::for_threads(threads, gathered_len, &mut live)?; // One reader for every component of a classic tile leaves the pool's // threads to Tier-1, and decodes bands itself while it waits. let helps = readers == 1 && coding == CodeBlockCoding::Classic; @@ -523,9 +533,12 @@ pub(super) fn decode_tile_rows<'a>( profile_timings.idwt_us += elapsed.saturating_sub(tier1_us); let result = result.and_then(|completed| { if completed { - for route in &routes { - route.record(tile_ctx, storage)?; - } + BandRoute::record_live( + routes.iter().map(|route| &**route), + threads, + tile_ctx, + storage, + )?; } Ok(completed) }); @@ -1899,9 +1912,10 @@ mod tests { } } - /// Every live component plan and its clipped scratch must fit beside - /// the executor, including the shared executor that streams RGB rows. - /// Simulated retained memory leaves room for either owner, but not both. + /// Every live component plan and the scratch of the threads that serve + /// them must fit beside the executor, including the shared executor that + /// streams RGB rows. Simulated retained memory leaves room for either + /// owner, but not both. #[test] #[expect( clippy::too_many_lines, @@ -1964,7 +1978,6 @@ mod tests { ); let (route_bytes, executor) = { let mut routes = Vec::new(); - let mut route_bytes = 0; let mut live = None; for component in 0..usize::from(components) { let BandRouting::Route(route) = plan_band_route_beside( @@ -1980,18 +1993,27 @@ mod tests { panic!("control fixture must take the pool route"); }; live = Some(route.budget()); - // Reuse the existing peak accounting for the - // route's owned tables and clipped scratch, - // excluding the retained bank. - let retained = tile_ctx.tier1_capacity_bytes().expect("bank bytes"); - tile_ctx.debug_counters.workspace_peak = WorkspacePeak::default(); - route - .record(tile_ctx, storage) - .expect("record route owners"); - route_bytes += - tile_ctx.debug_counters.workspace_peak.bytes().tier1 - retained; routes.push(route); } + // Reuse the existing peak accounting for the routes' + // owned tables and their threads' scratch, excluding + // the retained bank: the workers and one reader. + let threads = routes + .iter() + .map(|route| route.workers() + 1) + .max() + .unwrap_or(0); + let retained = tile_ctx.tier1_capacity_bytes().expect("bank bytes"); + tile_ctx.debug_counters.workspace_peak = WorkspacePeak::default(); + BandRoute::record_live( + routes.iter().map(|route| &**route), + threads, + tile_ctx, + storage, + ) + .expect("record route owners"); + let route_bytes = + tile_ctx.debug_counters.workspace_peak.bytes().tier1 - retained; let (rings, bands, _, _) = plan_tile_rings(&routes, storage, &image.header) .expect("plan control rings"); ( diff --git a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs index 4124ac80..aaf8d2c3 100644 --- a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs +++ b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs @@ -426,8 +426,8 @@ fn count_decoded_blocks( } /// Record the decode's workspace peak while the plan, its tasks and each -/// worker's clipped-block scratch, as budgeted, are live next to the retained -/// workspace bank. +/// worker's scratch, as budgeted, are live next to the retained workspace +/// bank. fn record_route_peak( tile_ctx: &mut TileDecodeContext, storage: &DecompositionStorage<'_>, @@ -436,20 +436,10 @@ fn record_route_peak( counts: TaskCounts, ) -> Result<()> { let route_bytes = [ - plan.bands.capacity() * size_of::(), - plan.classic_bands.capacity() * size_of::(), - plan.jobs.capacity() * size_of::>(), - plan.strips.capacity() * size_of::(), - plan.band_sub_bands.capacity() * size_of::(), - workers.saturating_mul( - plan.max_gathered_len - .saturating_add(size_of::()), - ), + plan_bytes(plan), + workers.saturating_mul(thread_scratch_bytes(plan, counts.scratch_samples)), counts.tasks * size_of::>(), counts.shared * size_of::>(), - workers - .saturating_mul(counts.scratch_samples) - .saturating_mul(size_of::()), ] .into_iter() .fold(0, usize::saturating_add); @@ -458,6 +448,28 @@ fn record_route_peak( Ok(()) } +/// Bytes the plan's own tables hold. +fn plan_bytes(plan: &TilePlan<'_>) -> usize { + [ + plan.bands.capacity() * size_of::(), + plan.classic_bands.capacity() * size_of::(), + plan.jobs.capacity() * size_of::>(), + plan.strips.capacity() * size_of::(), + plan.band_sub_bands.capacity() * size_of::(), + ] + .into_iter() + .fold(0, usize::saturating_add) +} + +/// Bytes a thread's scratch holds at most for `plan`, as budgeted: the +/// largest payload it gathers, block scratch of `scratch_samples`, and its +/// entry in the threads' list. +fn thread_scratch_bytes(plan: &TilePlan<'_>, scratch_samples: usize) -> usize { + plan.max_gathered_len + .saturating_add(scratch_samples.saturating_mul(size_of::())) + .saturating_add(size_of::()) +} + /// Prepare one workspace per worker in the retained `bank`, then decode the /// plan's strips into `storage`'s coefficients in place through /// `decode_block`, which leaves a stale buffer current. Returns the bank's @@ -1634,7 +1646,11 @@ pub(in crate::j2c::decode) enum BandRouting<'a> { /// path: a classic component with fewer rows of code-blocks than threads /// stays on the route, whose bands split into parts for the idle threads, /// rather than decode its bands serially. A 512 x 512 RGB image at 6.6 bits -/// per pixel decoded 4.5 times as long on 18 threads as on 12 that way. +/// per pixel decoded 4.5 times as long on 18 threads as on 12 that way. Its +/// executor runs every component's routes on the same threads, and charges +/// their scratch once for all of them (see +/// [`BandRoute::thread_scratch_bytes`]); a route charges its threads' +/// scratch otherwise. pub(in crate::j2c::decode) fn plan_band_route<'a>( tile: &Tile<'a>, component_idx: usize, @@ -1734,12 +1750,14 @@ pub(in crate::j2c::decode) fn plan_band_route_beside<'a>( // Each thread gathers payloads into scratch of its own as it // decodes, and allocates its block scratch on first use: a clipped // block, or any block of a band it decodes a part of. - include_payload_scratch(&mut budget, &plan, workers + 1)?; - budget.include_elements::( - (workers + 1) - .checked_mul(band_scratch_samples(&plan)?) - .ok_or(ValidationError::ImageTooLarge)?, - )?; + if !lockstep { + include_payload_scratch(&mut budget, &plan, workers + 1)?; + budget.include_elements::( + (workers + 1) + .checked_mul(band_scratch_samples(&plan)?) + .ok_or(ValidationError::ImageTooLarge)?, + )?; + } let (width, height) = (plan.max_width, plan.max_height); let structural = &mut storage.structural_workspace_bytes; match coding { @@ -1805,6 +1823,15 @@ impl BandRoute<'_> { self.plan.max_gathered_len } + /// Bytes a thread's scratch holds at most for this plan, as a route + /// that is not `lockstep` charges it for each of its threads. + pub(in crate::j2c::decode) fn thread_scratch_bytes(&self) -> Result { + Ok(thread_scratch_bytes( + &self.plan, + band_scratch_samples(&self.plan)?, + )) + } + pub(in crate::j2c::decode) fn strips(&self) -> impl Iterator + '_ { self.plan.strips.iter().map(|strip| { let band = self.plan.jobs[strip.jobs.start].band; @@ -1920,21 +1947,43 @@ impl BandRoute<'_> { } /// Count the plan's blocks and record the decode's peak with the plan - /// and every thread's clipped-block scratch live. + /// and every thread's scratch live. pub(in crate::j2c::decode) fn record( &self, tile_ctx: &mut TileDecodeContext, storage: &DecompositionStorage<'_>, ) -> Result<()> { - // Band slots, not route tasks, hold this plan's outputs. - let counts = TaskCounts { - tasks: 0, - shared: 0, - scratch_samples: band_scratch_samples(&self.plan)?, - part_work: None, - }; - record_route_peak(tile_ctx, storage, &self.plan, self.workers + 1, counts)?; - count_decoded_blocks(tile_ctx, &self.plan, self.workers + 1, self.growths); + Self::record_live(core::iter::once(self), self.workers + 1, tile_ctx, storage) + } + + /// Count every route's blocks and record the decode's peak with all + /// their plans live, beside the scratch of the `threads` threads that + /// serve them all. + pub(in crate::j2c::decode) fn record_live<'r>( + routes: impl Iterator + Clone, + threads: usize, + tile_ctx: &mut TileDecodeContext, + storage: &DecompositionStorage<'_>, + ) -> Result<()> + where + Self: 'r, + { + let mut plans = 0usize; + let mut scratch = 0usize; + for route in routes.clone() { + plans = plans.saturating_add(plan_bytes(&route.plan)); + scratch = scratch.max(route.thread_scratch_bytes()?); + } + let bytes = plans.saturating_add(threads.saturating_mul(scratch)); + let sample = WorkspaceBytes::sample(tile_ctx, storage)?.with_tier1(bytes); + tile_ctx.debug_counters.workspace_peak.record(sample); + #[cfg(test)] + { + tile_ctx.debug_counters.pool_band_plan_bytes += plans; + } + for route in routes { + count_decoded_blocks(tile_ctx, &route.plan, route.workers + 1, route.growths); + } Ok(()) } } diff --git a/crates/j2k-native/src/tests/parallel_bands.rs b/crates/j2k-native/src/tests/parallel_bands.rs index 5f32e32d..8f44781f 100644 --- a/crates/j2k-native/src/tests/parallel_bands.rs +++ b/crates/j2k-native/src/tests/parallel_bands.rs @@ -243,6 +243,56 @@ fn rgb_fixture() -> Vec { .expect("encode rgb fixture") } +/// Discards the rows a decode streams. +struct DiscardRows; + +impl crate::ComponentRowSink for DiscardRows { + fn write_rows(&mut self, _x: u32, _y: u32, _planes: &[crate::ComponentPlane<'_>]) {} +} + +/// A tile whose rows stream from pool bands holds every component's plan at +/// once, beside the scratch of the threads that serve them all, and its +/// workspace peak counts them together: the three plans add up to the ones +/// a region decode into planes holds one after another, and the peak holds +/// all of them. +#[test] +fn streamed_pool_bands_record_every_live_plan() { + let bytes = rgb_fixture(); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("parse rgb fixture"); + let roi = (0, 0, WIDTH, HEIGHT); + pool(4).install(|| { + let mut context = DecoderContext::default(); + context.set_cpu_decode_parallelism(CpuDecodeParallelism::Auto); + image + .decode_native_region_with_context(roi, &mut context) + .expect("decode region planes"); + let one_at_a_time = context.tile_decode_context.debug_counters; + let mut context = DecoderContext::default(); + context.set_cpu_decode_parallelism(CpuDecodeParallelism::Auto); + image + .decode_region_rows_for_integer_output_with_context(roi, &mut context, &mut DiscardRows) + .expect("decode region rows"); + assert!(context.tile_decode_context.stream_rows); + let together = context.tile_decode_context.debug_counters; + assert_eq!(one_at_a_time.pool_band_components, 3); + assert_eq!(together.pool_band_components, 3); + assert_eq!( + together.pool_band_plan_bytes, + one_at_a_time.pool_band_plan_bytes + ); + let bank = context + .tile_decode_context + .tier1_capacity_bytes() + .expect("bank bytes"); + let tier1 = together.workspace_peak.bytes().tier1; + assert!( + tier1 >= bank + together.pool_band_plan_bytes, + "peak Tier-1 bytes {tier1}: the bank's {bank} and the plans' {}", + together.pool_band_plan_bytes + ); + }); +} + /// With the pool's other thread held by unrelated work, workers can't run /// until the decode is over, so the IDWT must decode every band itself. #[test] From 1e856a9625a35916b5f32091b031570c1bcd67af Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 23:24:08 -0400 Subject: [PATCH 59/94] perf(j2k-native): decode multi-segment classic blocks on the pool routes The tile route and pool bands took classic tiles only when each block was one arithmetic codeword segment, so tiles with selective bypass or per-pass termination, common in archival images and in this crate's own multi-layer encodes, decoded on the per-sub-band path on a pool, which copies every payload, and region decodes held them over whole windows; full-image decodes that stream their rows could not stream them on a pool at all. Such blocks now take both routes. A block's job is unchanged: when it decodes, its worker collects the block's codeword segment descriptors (byte range, passes, raw or arithmetic) into its own scratch, in the walk over the segment records that gathers a payload of several fragments, or beside the payload borrowed from the codestream, and decodes through the block decoder's multi-segment entry. Each thread's segment scratch is reserved before it starts for the plan's block of the most segments, and charged with its payload scratch; the plans hold no per-segment records. The walk is the one the per-sub-band and batch collectors use: one allocation-free walker now serves the three of them. Tests: on pools of 4 and 12 threads the multi-segment fixtures decode their planes on the tile route, and stream their rows from pool bands with no component planes and no uncharged line-buffer samples, some bands in parts; with the pool's routes on, a full-image rows decode under a planes budget it would pass streams them from pool bands. The tests that compare pool bands and streamed rows with whole windows now also cover the terminated multi-layer blocks of this crate's encodes, which they skipped. Measured on an 18-core ARM64 machine, with the scratch fix that follows, against the base of this series, one fresh process per decode, build and round, seven rounds of three warm decodes (medians): full-image 8-bit decodes of archival RGB and gray images with selective bypass (6061 x 4252, 5428 x 4460 and 5100 x 6600, seven and eight layers) take 0.39-0.60 of their time on 18 threads and 0.52-0.68 on 6, their peak resident memory beyond input and output falling from 298-463 MiB to 17-71 MiB. One of 17311 x 13396 that failed with a resource limit on 6 and 18 threads takes 587 and 301 ms, and a 47-megapixel RGB image with selective bypass that failed there takes 202 and 98 ms, 0.91-0.93 of its twin without bypass, in the same memory. Their 1024 x 1024 regions take 0.22-0.46 of their time on 18 threads and 0.45-0.68 on 6, a 1024 x 1024 window at half scale 0.37-0.52 and 0.65-0.74, quarter- and eighth-scale overviews 0.41-0.63 and 0.69-0.74, each in 13-61% of the peak heap. One-thread decodes, the twin without bypass and a 16-image decode benchmark set on 6 and 18 threads keep their bytes, and their times within 1.1% in 15-round reruns, where two runs of the same build differ by up to 0.8%. --- .../j2k-native/src/j2c/decode/direct_plan.rs | 4 +- .../src/j2c/decode/direct_plan/classic.rs | 288 ++++++++---------- .../src/j2c/decode/parallel_bands.rs | 15 +- .../src/j2c/decode/subband/tile_parallel.rs | 287 ++++++++++++----- crates/j2k-native/src/scalar.rs | 4 +- crates/j2k-native/src/tests/multi_segment.rs | 60 ++++ crates/j2k-native/src/tests/parallel_bands.rs | 15 +- crates/j2k-native/src/tests/row_sink.rs | 9 +- .../j2k-native/src/tests/workspace_reuse.rs | 6 +- 9 files changed, 414 insertions(+), 274 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode/direct_plan.rs b/crates/j2k-native/src/j2c/decode/direct_plan.rs index 99f9d47a..64367a50 100644 --- a/crates/j2k-native/src/j2c/decode/direct_plan.rs +++ b/crates/j2k-native/src/j2c/decode/direct_plan.rs @@ -22,7 +22,9 @@ use crate::{ mod classic; #[cfg(feature = "parallel")] -pub(super) use self::classic::classic_allocation_counts; +pub(super) use self::classic::{ + classic_allocation_counts, extend_reserved, push_reserved, walk_classic_segments, +}; pub(super) use self::classic::{ collect_classic_code_block_data, collect_referenced_classic_code_block_data, }; diff --git a/crates/j2k-native/src/j2c/decode/direct_plan/classic.rs b/crates/j2k-native/src/j2c/decode/direct_plan/classic.rs index e297ab9b..74179f9a 100644 --- a/crates/j2k-native/src/j2c/decode/direct_plan/classic.rs +++ b/crates/j2k-native/src/j2c/decode/direct_plan/classic.rs @@ -4,7 +4,7 @@ use super::{bail, DecodeAllocationBudget, DecodingError, DecompositionStorage, Result}; use crate::error::ValidationError; -use crate::j2c::build::CodeBlock; +use crate::j2c::build::{CodeBlock, Layer, Segment}; use crate::j2c::codestream::CodeBlockStyle; use crate::{try_reserve_decode_elements, J2kCodeBlockSegment}; use alloc::vec::Vec; @@ -12,7 +12,8 @@ use alloc::vec::Vec; #[derive(Clone, Copy)] pub(crate) struct ClassicAllocationCounts { pub(crate) data_bytes: usize, - segments: usize, + /// Codeword segments. + pub(crate) segments: usize, /// Segments that hold data. pub(crate) fragments: usize, /// Coding passes over all segments. @@ -47,67 +48,17 @@ pub(crate) fn collect_classic_code_block_data( collected_segments.capacity(), )?; - let mut last_segment_idx = 0u8; - let mut segment_start_offset = 0usize; - let mut segment_start_coding_pass = 0u8; - let mut coding_passes = 0u8; - let is_normal_mode = style.single_codeword_segment(); - - for layer in &storage.layers[code_block.layers.start..code_block.layers.end] { - let Some(range) = layer.segments() else { - continue; - }; - - for segment in &storage.segments[range] { - if segment.idx != last_segment_idx { - validate_next_segment_index(last_segment_idx, segment.idx)?; - if coding_passes > segment_start_coding_pass - || combined_data.len() > segment_start_offset - { - collected_segments.push(classic_segment( - *style, - segment_start_offset, - combined_data.len(), - segment_start_coding_pass, - coding_passes, - )?); - } - segment_start_offset = combined_data.len(); - segment_start_coding_pass = coding_passes; - last_segment_idx = segment.idx; - } - - combined_data.extend_from_slice(segment.data); - coding_passes = coding_passes.saturating_add(segment.coding_pases); - } - } - - if coding_passes > segment_start_coding_pass || combined_data.len() > segment_start_offset { - collected_segments.push(classic_segment( - *style, - segment_start_offset, - combined_data.len(), - segment_start_coding_pass, - coding_passes, - )?); - } - - if is_normal_mode { - collected_segments.clear(); - collected_segments.push(J2kCodeBlockSegment { - data_offset: 0, - data_length: u32::try_from(combined_data.len()) - .map_err(|_| DecodingError::CodeBlockDecodeFailure)?, - start_coding_pass: 0, - end_coding_pass: coding_passes, - use_arithmetic: true, - }); - } - - if coding_passes != code_block.number_of_coding_passes { - bail!(DecodingError::CodeBlockDecodeFailure); - } - if combined_data.len() != counts.data_bytes || collected_segments.len() > counts.segments { + let (data_bytes, _, coding_passes) = walk_classic_segments( + code_block, + *style, + (&storage.layers, &storage.segments), + |fragment| extend_reserved(&mut combined_data, fragment), + |segment| push_reserved(&mut collected_segments, segment), + )?; + if coding_passes != code_block.number_of_coding_passes + || data_bytes != counts.data_bytes + || collected_segments.len() > counts.segments + { bail!(DecodingError::CodeBlockDecodeFailure); } @@ -123,7 +74,7 @@ pub(crate) fn collect_referenced_classic_code_block_data( style: &CodeBlockStyle, storage: &DecompositionStorage<'_>, budget: &mut DecodeAllocationBudget, - mut append_fragment: impl FnMut(&[u8]) -> Result<()>, + append_fragment: impl FnMut(&[u8]) -> Result<()>, ) -> Result<(usize, usize, Vec)> { let counts = classic_allocation_counts(code_block, *style, storage)?; budget.include_elements::(counts.segments)?; @@ -135,140 +86,153 @@ pub(crate) fn collect_referenced_classic_code_block_data( collected_segments.capacity(), )?; + let (data_bytes, fragment_count, coding_passes) = walk_classic_segments( + code_block, + *style, + (&storage.layers, &storage.segments), + append_fragment, + |segment| push_reserved(&mut collected_segments, segment), + )?; + if coding_passes != code_block.number_of_coding_passes + || data_bytes != counts.data_bytes + || fragment_count != counts.fragments + || collected_segments.len() > counts.segments + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + + Ok((fragment_count, data_bytes, collected_segments)) +} + +/// Payload bytes, segments, data fragments and coding passes of a classic +/// block, rejecting segment indices that skip one, without allocating. +pub(crate) fn classic_allocation_counts( + code_block: &CodeBlock, + style: CodeBlockStyle, + storage: &DecompositionStorage<'_>, +) -> Result { + let mut segments = 0_usize; + let (data_bytes, fragments, coding_passes) = walk_classic_segments( + code_block, + style, + (&storage.layers, &storage.segments), + |_| Ok(()), + |_| { + segments = segments + .checked_add(1) + .ok_or(ValidationError::ImageTooLarge)?; + Ok(()) + }, + )?; + Ok(ClassicAllocationCounts { + data_bytes, + segments, + fragments, + coding_passes, + }) +} + +/// Walk the codeword segments of a classic block without allocating, as +/// [`collect_classic_code_block_data`] collects them: pass each nonempty +/// data fragment, layer by layer, to `fragment`, and each codeword segment, +/// its contributions across layers joined, to `segment`, at its offset into +/// the fragments joined. A segment that holds coding passes and no bytes is +/// kept; one that holds neither is not. A block of one codeword segment is +/// one arithmetic segment over all its passes. Returns the payload bytes and +/// data fragments, and the coding passes, rejecting segment indices that +/// skip one. +pub(crate) fn walk_classic_segments( + code_block: &CodeBlock, + style: CodeBlockStyle, + (layers, segments): (&[Layer], &[Segment<'_>]), + mut fragment: impl FnMut(&[u8]) -> Result<()>, + mut segment: impl FnMut(J2kCodeBlockSegment) -> Result<()>, +) -> Result<(usize, usize, u8)> { + let single = style.single_codeword_segment(); let mut data_bytes = 0usize; - let mut fragment_count = 0usize; + let mut fragments = 0usize; let mut last_segment_idx = 0u8; let mut segment_start_offset = 0usize; let mut segment_start_coding_pass = 0u8; let mut coding_passes = 0u8; - let is_normal_mode = style.single_codeword_segment(); - - for layer in &storage.layers[code_block.layers.start..code_block.layers.end] { + let layers = layers + .get(code_block.layers.clone()) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + for layer in layers { let Some(range) = layer.segments() else { continue; }; - - for segment in &storage.segments[range] { - if segment.idx != last_segment_idx { - validate_next_segment_index(last_segment_idx, segment.idx)?; - if coding_passes > segment_start_coding_pass || data_bytes > segment_start_offset { - collected_segments.push(classic_segment( - *style, + let records = segments + .get(range) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + for record in records { + if record.idx != last_segment_idx { + validate_next_segment_index(last_segment_idx, record.idx)?; + if !single + && (coding_passes > segment_start_coding_pass + || data_bytes > segment_start_offset) + { + segment(classic_segment( + style, segment_start_offset, data_bytes, segment_start_coding_pass, coding_passes, - )?); + )?)?; } segment_start_offset = data_bytes; segment_start_coding_pass = coding_passes; - last_segment_idx = segment.idx; + last_segment_idx = record.idx; } - - if !segment.data.is_empty() { - append_fragment(segment.data)?; - fragment_count = fragment_count + if !record.data.is_empty() { + fragment(record.data)?; + fragments = fragments .checked_add(1) .ok_or(ValidationError::ImageTooLarge)?; } data_bytes = data_bytes - .checked_add(segment.data.len()) + .checked_add(record.data.len()) .ok_or(ValidationError::ImageTooLarge)?; - coding_passes = coding_passes.saturating_add(segment.coding_pases); + coding_passes = coding_passes.saturating_add(record.coding_pases); } } - - if coding_passes > segment_start_coding_pass || data_bytes > segment_start_offset { - collected_segments.push(classic_segment( - *style, - segment_start_offset, - data_bytes, - segment_start_coding_pass, - coding_passes, - )?); - } - - if is_normal_mode { - collected_segments.clear(); - collected_segments.push(J2kCodeBlockSegment { + if single { + segment(J2kCodeBlockSegment { data_offset: 0, data_length: u32::try_from(data_bytes) .map_err(|_| DecodingError::CodeBlockDecodeFailure)?, start_coding_pass: 0, end_coding_pass: coding_passes, use_arithmetic: true, - }); + })?; + } else if coding_passes > segment_start_coding_pass || data_bytes > segment_start_offset { + segment(classic_segment( + style, + segment_start_offset, + data_bytes, + segment_start_coding_pass, + coding_passes, + )?)?; } + Ok((data_bytes, fragments, coding_passes)) +} - if coding_passes != code_block.number_of_coding_passes - || data_bytes != counts.data_bytes - || fragment_count != counts.fragments - || collected_segments.len() > counts.segments - { +/// Append `data` to `values` within the capacity reserved for it. +pub(crate) fn extend_reserved(values: &mut Vec, data: &[u8]) -> Result<()> { + if values.capacity() - values.len() < data.len() { bail!(DecodingError::CodeBlockDecodeFailure); } - - Ok((fragment_count, data_bytes, collected_segments)) + values.extend_from_slice(data); + Ok(()) } -/// Payload bytes, segments, data fragments and coding passes of a classic -/// block, rejecting segment indices that skip one, without allocating. -pub(crate) fn classic_allocation_counts( - code_block: &CodeBlock, - style: CodeBlockStyle, - storage: &DecompositionStorage<'_>, -) -> Result { - let mut data_bytes = 0_usize; - let mut segment_count = 0_usize; - let mut last_segment_idx = 0_u8; - let mut segment_start_offset = 0_usize; - let mut segment_start_coding_pass = 0_u8; - let mut coding_passes = 0_u8; - let mut fragment_count = 0_usize; - - for layer in &storage.layers[code_block.layers.start..code_block.layers.end] { - let Some(range) = layer.segments() else { - continue; - }; - for segment in &storage.segments[range] { - if segment.idx != last_segment_idx { - validate_next_segment_index(last_segment_idx, segment.idx)?; - if coding_passes > segment_start_coding_pass || data_bytes > segment_start_offset { - segment_count = segment_count - .checked_add(1) - .ok_or(ValidationError::ImageTooLarge)?; - } - segment_start_offset = data_bytes; - segment_start_coding_pass = coding_passes; - last_segment_idx = segment.idx; - } - data_bytes = data_bytes - .checked_add(segment.data.len()) - .ok_or(ValidationError::ImageTooLarge)?; - if !segment.data.is_empty() { - fragment_count = fragment_count - .checked_add(1) - .ok_or(ValidationError::ImageTooLarge)?; - } - coding_passes = coding_passes.saturating_add(segment.coding_pases); - } - } - - if coding_passes > segment_start_coding_pass || data_bytes > segment_start_offset { - segment_count = segment_count - .checked_add(1) - .ok_or(ValidationError::ImageTooLarge)?; - } - if style.single_codeword_segment() { - segment_count = 1; +/// Push `value` onto `values` within the capacity reserved for it. +pub(crate) fn push_reserved(values: &mut Vec, value: T) -> Result<()> { + if values.len() == values.capacity() { + bail!(DecodingError::CodeBlockDecodeFailure); } - - Ok(ClassicAllocationCounts { - data_bytes, - segments: segment_count, - fragments: fragment_count, - coding_passes, - }) + values.push(value); + Ok(()) } fn validate_next_segment_index(previous: u8, current: u8) -> Result<()> { diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index 72b0e010..b59b0231 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -232,7 +232,7 @@ fn decode_route( let mut budget = route.budget(); budget.include_bytes(executor_bytes(&rings, &bands)?)?; let mut scratches = - BlockScratch::for_threads(route.workers() + 1, route.gathered_len(), &mut budget)?; + BlockScratch::for_threads(route.workers() + 1, route.payload_scratch(), &mut budget)?; tile_ctx.idwt_output.placement = crate::j2c::idwt::IdwtPlacement::Coefficients; tile_ctx.idwt_output.coefficients = Vec::new(); tile_ctx.idwt_output.rect = IntRect::from_xywh(0, 0, 0, 0); @@ -395,13 +395,10 @@ pub(super) fn decode_tile_rows<'a>( }; let workers = route_workers.min(bank.saturating_sub(readers)); // Each thread's scratch serves every component's bands, so it is - // charged once, as large as any route's; the serial bands need none. - let mut thread_scratch = 0usize; - let mut gathered_len = 0usize; - for route in &routes { - thread_scratch = thread_scratch.max(route.thread_scratch_bytes()?); - gathered_len = gathered_len.max(route.gathered_len()); - } + // charged once, for the largest of the routes'; the serial bands need + // none. + let (thread_scratch, payload_scratch) = + BandRoute::shared_scratch(routes.iter().map(|route| &**route))?; let threads = workers + readers; let scratch_bytes = threads .checked_mul(thread_scratch) @@ -442,7 +439,7 @@ pub(super) fn decode_tile_rows<'a>( .max(held.saturating_sub(storage.roi_idwt_charge_elements)); } let mut output = RowOutput::new(tile, sink, tile_ctx, copy, region); - let mut scratches = BlockScratch::for_threads(threads, gathered_len, &mut live)?; + let mut scratches = BlockScratch::for_threads(threads, payload_scratch, &mut live)?; // One reader for every component of a classic tile leaves the pool's // threads to Tier-1, and decodes bands itself while it waits. let helps = readers == 1 && coding == CodeBlockCoding::Classic; diff --git a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs index aaf8d2c3..92c8bca3 100644 --- a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs +++ b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs @@ -10,9 +10,10 @@ //! them with work stealing. HT cleanup-only and single-segment classic //! payloads are borrowed from the codestream; a payload that spans several //! segments is gathered from the block's segment records into its worker's -//! scratch just before the block decodes. Classic blocks take this route when -//! each decodes as one arithmetic segment: no selective bypass or per-pass -//! termination. +//! scratch just before the block decodes. A classic block of several +//! codeword segments (selective bypass, per-pass termination) collects their +//! descriptors from the same records into its worker's scratch as its +//! payload is borrowed or gathered. use alloc::boxed::Box; use alloc::vec::Vec; @@ -32,12 +33,16 @@ use super::{ use crate::error::{DecodingError, ValidationError}; use crate::j2c::build::{CodeBlock, CodeBlockCoding, Layer, Segment}; use crate::j2c::codestream::CodeBlockStyle; -use crate::j2c::decode::direct_plan::classic_allocation_counts; +use crate::j2c::decode::direct_plan::{ + classic_allocation_counts, extend_reserved, push_reserved, walk_classic_segments, +}; use crate::j2c::decode::peak::WorkspaceBytes; use crate::j2c::decode::workspace::{task_workspace_bytes, TaskScratch, TaskWorkspace}; use crate::j2c::ht_block_decode::selected_code_block_segment_lengths; use crate::j2c::tile::{ComponentTile, ResolutionTile}; -use crate::scalar::decode_ht_code_block_scalar_with_workspace_midpoint; +use crate::scalar::{ + decode_ht_code_block_scalar_with_workspace_midpoint, internal_j2k_code_block_style, +}; use crate::{ decode_ht_code_block_scalar_with_workspace, try_reserve_decode_elements, try_resize_decode_elements, HtCodeBlockDecodeWorkspace, J2kCodeBlockSegment, @@ -152,35 +157,43 @@ impl<'s, 'a> PayloadSource<'s, 'a> { } } +/// A block's payload, and the codeword segments of a classic block of +/// several, which other blocks leave empty. +type BlockPayload<'p> = (&'p [u8], &'p [J2kCodeBlockSegment]); + /// A worker's scratch for the blocks it decodes: a block its sub-band stores -/// only part of, and a payload gathered from several segments. +/// only part of, a payload gathered from several segments, and the codeword +/// segments of a classic block of several. #[derive(Default)] pub(in crate::j2c::decode) struct BlockScratch { clipped: Vec, payload: Vec, + segments: Vec, } impl BlockScratch { - /// Scratch for each of `count` threads, its payload buffer reserved - /// before they start for the largest payload any of them gathers, - /// `payload_len` bytes, which the route charged per thread: gathering - /// never grows it. Charges `budget` for the list, and for capacity the - /// allocator gave beyond `payload_len`. + /// Scratch for each of `count` threads, reserved before they start for + /// the largest payload any of them gathers, `payload_len` bytes, and the + /// most codeword segments of a block, `segments`, which the route + /// charged per thread: decoding never grows it. Charges `budget` for the + /// list, and for capacity the allocator gave beyond those. pub(in crate::j2c::decode) fn for_threads( count: usize, - payload_len: usize, + (payload_len, segments): (usize, usize), budget: &mut DecodeAllocationBudget, ) -> Result> { let mut scratches = Vec::new(); budget.reserve_new(&mut scratches, count)?; for _ in 0..count { - let mut payload = Vec::new(); - try_reserve_decode_elements(&mut payload, payload_len)?; - budget.include_capacity_overage::(payload_len, payload.capacity())?; - scratches.push(Self { - clipped: Vec::new(), - payload, - }); + let mut scratch = Self::default(); + try_reserve_decode_elements(&mut scratch.payload, payload_len)?; + try_reserve_decode_elements(&mut scratch.segments, segments)?; + budget.include_capacity_overage::(payload_len, scratch.payload.capacity())?; + budget.include_capacity_overage::( + segments, + scratch.segments.capacity(), + )?; + scratches.push(scratch); } Ok(scratches) } @@ -222,6 +235,9 @@ struct TilePlan<'a> { /// Bytes of the largest payload gathered from several segments. Each /// worker gathers such payloads into scratch of this size. max_gathered_len: usize, + /// Codeword segments of the classic block of several with the most. + /// Each worker collects their descriptors into scratch of this size. + max_segments: usize, skipped: usize, coded_bytes: usize, max_width: u32, @@ -233,6 +249,14 @@ struct TilePlan<'a> { band_sub_bands: Vec, } +impl TilePlan<'_> { + /// What each thread's payload scratch holds for the plan: the largest + /// gathered payload's bytes and the most codeword segments of a block. + fn payload_scratch(&self) -> (usize, usize) { + (self.max_gathered_len, self.max_segments) + } +} + /// How [`plan_tile_route`] wants the tile's code blocks decoded. #[cfg_attr(test, derive(Debug, PartialEq, Eq))] pub(in crate::j2c::decode) enum TileRoute { @@ -245,9 +269,9 @@ pub(in crate::j2c::decode) enum TileRoute { } /// Decode every code block of the tile in parallel when all of its -/// components use HT block coding only, or classic coding with one arithmetic -/// segment per block, and the tile carries enough coded data to amortize a -/// fork-join. Returns without side effects for the non-`Decoded` routes. +/// components use HT block coding only, or classic coding only, and the tile +/// carries enough coded data to amortize a fork-join. Returns without side +/// effects for the non-`Decoded` routes. pub(in crate::j2c::decode) fn plan_tile_route<'a>( tile: &Tile<'a>, components: Range, @@ -462,10 +486,11 @@ fn plan_bytes(plan: &TilePlan<'_>) -> usize { } /// Bytes a thread's scratch holds at most for `plan`, as budgeted: the -/// largest payload it gathers, block scratch of `scratch_samples`, and its -/// entry in the threads' list. +/// largest payload it gathers and the segments of its classic block of the +/// most, block scratch of `scratch_samples`, and its entry in the threads' +/// list. fn thread_scratch_bytes(plan: &TilePlan<'_>, scratch_samples: usize) -> usize { - plan.max_gathered_len + payload_scratch_bytes(plan) .saturating_add(scratch_samples.saturating_mul(size_of::())) .saturating_add(size_of::()) } @@ -481,7 +506,8 @@ fn decode_with_bank<'a, W: TaskScratch>( bank: &mut Vec>, storage: &mut DecompositionStorage<'a>, budget: &mut DecodeAllocationBudget, - decode_block: impl Fn(&BlockJob<'_>, &[u8], &mut [f32], usize, &mut W) -> Result<()> + Sync, + decode_block: impl Fn(&BlockJob<'_>, BlockPayload<'_>, &mut [f32], usize, &mut W) -> Result<()> + + Sync, ) -> Result> { let Ok(growths) = prepare_uniform_task_workspaces( bank, @@ -493,7 +519,7 @@ fn decode_with_bank<'a, W: TaskScratch>( ) else { return Ok(None); }; - let Ok(mut scratches) = BlockScratch::for_threads(workers, plan.max_gathered_len, budget) + let Ok(mut scratches) = BlockScratch::for_threads(workers, plan.payload_scratch(), budget) else { return Ok(None); }; @@ -523,7 +549,7 @@ pub(in crate::j2c::decode) fn routes_tile(tile: &Tile<'_>) -> bool { } /// The block coding of every component in `components` when the route can -/// decode it: HT only, or classic with one arithmetic segment per block. +/// decode it: HT only, or classic only. fn tile_route_coding(tile: &Tile<'_>, components: Range) -> Option { let mut styles = tile .component_infos @@ -537,9 +563,7 @@ fn tile_route_coding(tile: &Tile<'_>, components: Range) -> Option( jobs: Vec::new(), strips: Vec::new(), max_gathered_len: 0, + max_segments: 0, skipped: 0, coded_bytes: 0, max_width: 0, @@ -661,7 +686,7 @@ fn plan_tile<'a>( { continue; } - let job = plan_block( + let (job, segments) = plan_block( code_block, code_block_idx, stored_placement(code_block, sub_band), @@ -669,6 +694,7 @@ fn plan_tile<'a>( storage, classic_style, )?; + plan.max_segments = plan.max_segments.max(segments); if matches!(job.payload, Payload::Gathered) { plan.max_gathered_len = plan.max_gathered_len.max(job.payload_length as usize); @@ -793,8 +819,10 @@ fn required_code_blocks( Ok((count, segment_bytes, rows)) } -/// A block's job. A payload in one codestream segment is borrowed; others -/// are gathered when the block decodes. +/// A block's job, and the codeword segments whose descriptors it collects +/// when it decodes: those of a classic block of several, or none. A payload +/// in one codestream segment is borrowed; others are gathered when the +/// block decodes. fn plan_block<'a>( code_block: &CodeBlock, code_block_idx: usize, @@ -802,13 +830,15 @@ fn plan_block<'a>( band: usize, storage: &DecompositionStorage<'a>, classic_style: Option, -) -> Result> { - let (payload_length, refinement_length, single) = match classic_style { - Some(style) => classic_payload(code_block, style, storage)?, - None => ht_payload(code_block, storage)?, +) -> Result<(BlockJob<'a>, usize)> { + let (payload_length, refinement_length, single, segments) = if let Some(style) = classic_style { + classic_payload(code_block, style, storage)? + } else { + let (payload_length, refinement_length, single) = ht_payload(code_block, storage)?; + (payload_length, refinement_length, single, 0) }; let payload = single.map_or(Payload::Gathered, Payload::Borrowed); - Ok(BlockJob { + let job = BlockJob { payload, code_block: code_block_idx, payload_length: u32::try_from(payload_length) @@ -821,7 +851,8 @@ fn plan_block<'a>( missing_bit_planes: code_block.missing_bit_planes, number_of_coding_passes: code_block.number_of_coding_passes, band, - }) + }; + Ok((job, segments)) } /// An HT block's payload and refinement lengths, and its cleanup segment when @@ -840,13 +871,14 @@ fn ht_payload<'a>( } /// A classic block's payload length, checked as -/// `collect_classic_code_block_data` checks it, and its segment data when at -/// most one segment holds any. +/// `collect_classic_code_block_data` checks it, its segment data when at +/// most one segment holds any, and its codeword segments when it has +/// several. fn classic_payload<'a>( code_block: &CodeBlock, style: CodeBlockStyle, storage: &DecompositionStorage<'a>, -) -> Result<(usize, usize, Option<&'a [u8]>)> { +) -> Result<(usize, usize, Option<&'a [u8]>, usize)> { let counts = classic_allocation_counts(code_block, style, storage)?; if counts.coding_passes != code_block.number_of_coding_passes { return Err(DecodingError::CodeBlockDecodeFailure.into()); @@ -860,7 +892,12 @@ fn classic_payload<'a>( .find(|data| !data.is_empty()) .unwrap_or_default() }); - Ok((counts.data_bytes, 0, single)) + let segments = if style.single_codeword_segment() { + 0 + } else { + counts.segments + }; + Ok((counts.data_bytes, 0, single, segments)) } /// The block's cleanup segment when it is the only selected segment. @@ -979,8 +1016,8 @@ fn strips_are_disjoint(strips: &mut [Strip], coefficient_len: usize) -> bool { previous_end <= coefficient_len } -/// Charge the payload scratch each of `threads` threads grows to as it -/// gathers the plan's payloads. +/// Charge the payload scratch each of `threads` threads holds to gather the +/// plan's payloads and collect their codeword segments. fn include_payload_scratch( budget: &mut DecodeAllocationBudget, plan: &TilePlan<'_>, @@ -988,11 +1025,19 @@ fn include_payload_scratch( ) -> Result<()> { budget.include_bytes( threads - .checked_mul(plan.max_gathered_len) + .checked_mul(payload_scratch_bytes(plan)) .ok_or(ValidationError::ImageTooLarge)?, ) } +/// Bytes of a thread's scratch for the plan's largest gathered payload and +/// its classic block of the most codeword segments. +fn payload_scratch_bytes(plan: &TilePlan<'_>) -> usize { + plan.max_segments + .saturating_mul(size_of::()) + .saturating_add(plan.max_gathered_len) +} + /// Gather the payload of `job`, a block whose payload spans several /// segments, into `scratch`: for a classic block every segment, for an HT /// block the selected set's cleanup and refinement segments. @@ -1277,7 +1322,8 @@ fn decode_strips<'plan, 'coefficients, W: Send>( (workspaces, scratches): (&mut [W], &mut [BlockScratch]), counts: TaskCounts, source: PayloadSource<'_, '_>, - decode_block: impl Fn(&BlockJob<'_>, &[u8], &mut [f32], usize, &mut W) -> Result<()> + Sync, + decode_block: impl Fn(&BlockJob<'_>, BlockPayload<'_>, &mut [f32], usize, &mut W) -> Result<()> + + Sync, ) -> Result<()> { let mut shared: Vec> = Vec::new(); shared @@ -1402,11 +1448,12 @@ fn decode_part( scratch_samples: usize, workspace: &mut W, (scratch, source): (&mut BlockScratch, PayloadSource<'_, '_>), - decode_block: &impl Fn(&BlockJob<'_>, &[u8], &mut [f32], usize, &mut W) -> Result<()>, + decode_block: &impl Fn(&BlockJob<'_>, BlockPayload<'_>, &mut [f32], usize, &mut W) -> Result<()>, ) -> Result<()> { let BlockScratch { clipped: scratch, payload: payload_scratch, + segments: segment_scratch, } = scratch; let lock = || { output @@ -1453,7 +1500,7 @@ fn decode_part( if !placement.is_whole(job.width, job.height) { block.fill(0.0); } - let payload = job_payload(plan, job, source, payload_scratch)?; + let payload = job_payload(plan, job, source, (payload_scratch, segment_scratch))?; decode_block(job, payload, block, job.width as usize, workspace)?; placement.copy_from_block(block, job.width, &mut lock()?, strip.stride)?; } @@ -1466,11 +1513,12 @@ fn decode_strip( output: &mut [f32], workspace: &mut W, (scratch, source): (&mut BlockScratch, PayloadSource<'_, '_>), - decode_block: &impl Fn(&BlockJob<'_>, &[u8], &mut [f32], usize, &mut W) -> Result<()>, + decode_block: &impl Fn(&BlockJob<'_>, BlockPayload<'_>, &mut [f32], usize, &mut W) -> Result<()>, ) -> Result<()> { let BlockScratch { clipped: clipped_scratch, payload: payload_scratch, + segments: segment_scratch, } = scratch; for job in &plan.jobs[strip.jobs.clone()] { // The strip starts at the block row's first stored row. @@ -1495,7 +1543,7 @@ fn decode_strip( block.fill(0.0); (block, job.width as usize) }; - let payload = job_payload(plan, job, source, payload_scratch)?; + let payload = job_payload(plan, job, source, (payload_scratch, segment_scratch))?; decode_block(job, payload, block_output, output_stride, workspace)?; if !whole { placement.copy_from_block(clipped_scratch, job.width, output, strip.stride)?; @@ -1507,7 +1555,7 @@ fn decode_strip( fn decode_strip_block( plan: &TilePlan<'_>, job: &BlockJob<'_>, - payload: &[u8], + (payload, _): BlockPayload<'_>, output: &mut [f32], output_stride: usize, workspace: &mut HtCodeBlockDecodeWorkspace, @@ -1548,22 +1596,28 @@ fn decode_strip_block( fn decode_classic_strip_block( plan: &TilePlan<'_>, job: &BlockJob<'_>, - payload: &[u8], + (payload, segments): BlockPayload<'_>, output: &mut [f32], output_stride: usize, workspace: &mut J2kCodeBlockDecodeWorkspace, ) -> Result<()> { let band = plan.classic_bands[job.band]; - let segments = [J2kCodeBlockSegment { + // A block of one codeword segment is one arithmetic segment. + let single = [J2kCodeBlockSegment { data_offset: 0, data_length: job.payload_length, start_coding_pass: 0, end_coding_pass: job.number_of_coding_passes, use_arithmetic: true, }]; + let segments = if band_style(plan, job.band).single_codeword_segment() { + &single[..] + } else { + segments + }; let decode_job = band.job( payload, - &segments, + segments, job.width, job.height, output_stride, @@ -1574,17 +1628,68 @@ fn decode_classic_strip_block( } /// The job's payload, borrowed from the codestream or gathered into -/// `scratch`. +/// `payload`, and for a classic block of several codeword segments, their +/// descriptors, collected into `segments`. fn job_payload<'p>( plan: &TilePlan<'_>, job: &BlockJob<'p>, source: PayloadSource<'_, '_>, - scratch: &'p mut Vec, -) -> Result<&'p [u8]> { - match job.payload { - Payload::Borrowed(data) => Ok(data), - Payload::Gathered => gather_payload(plan.coding, job, source, scratch), + (payload, segments): (&'p mut Vec, &'p mut Vec), +) -> Result> { + if plan.coding == CodeBlockCoding::Classic + && !band_style(plan, job.band).single_codeword_segment() + { + return collect_segments(plan, job, source, (payload, segments)); } + let data = match job.payload { + Payload::Borrowed(data) => data, + Payload::Gathered => gather_payload(plan.coding, job, source, payload)?, + }; + Ok((data, &[])) +} + +/// The payload and codeword segments of `job`, a classic block of several: +/// one walk over its segment records collects the segments into +/// `segments` and, when its payload spans several fragments, gathers them +/// into `payload`. The thread's scratch holds both for the plan's largest. +fn collect_segments<'p>( + plan: &TilePlan<'_>, + job: &BlockJob<'p>, + source: PayloadSource<'_, '_>, + (payload, segments): (&'p mut Vec, &'p mut Vec), +) -> Result> { + let code_block = source + .code_blocks + .get(job.code_block) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let gathered = matches!(job.payload, Payload::Gathered); + payload.clear(); + segments.clear(); + let (data_bytes, _, coding_passes) = walk_classic_segments( + code_block, + band_style(plan, job.band), + (source.layers, source.segments), + |fragment| { + if gathered { + extend_reserved(payload, fragment)?; + } + Ok(()) + }, + |segment| push_reserved(segments, segment), + )?; + if data_bytes != job.payload_length as usize || coding_passes != job.number_of_coding_passes { + return Err(DecodingError::CodeBlockDecodeFailure.into()); + } + let data = match job.payload { + Payload::Borrowed(data) => data, + Payload::Gathered => payload, + }; + Ok((data, segments)) +} + +/// The code-block style of band `band`'s blocks in a classic plan. +fn band_style(plan: &TilePlan<'_>, band: usize) -> CodeBlockStyle { + internal_j2k_code_block_style(plan.classic_bands[band].style) } /// Where a band decodes: into its slot, or as run `part` of its blocks split @@ -1649,8 +1754,8 @@ pub(in crate::j2c::decode) enum BandRouting<'a> { /// per pixel decoded 4.5 times as long on 18 threads as on 12 that way. Its /// executor runs every component's routes on the same threads, and charges /// their scratch once for all of them (see -/// [`BandRoute::thread_scratch_bytes`]); a route charges its threads' -/// scratch otherwise. +/// [`BandRoute::shared_scratch`]); a route charges its threads' scratch +/// otherwise. pub(in crate::j2c::decode) fn plan_band_route<'a>( tile: &Tile<'a>, component_idx: usize, @@ -1818,18 +1923,36 @@ impl BandRoute<'_> { self.workers } - /// Bytes of the largest payload a thread gathers for this plan. - pub(in crate::j2c::decode) fn gathered_len(&self) -> usize { - self.plan.max_gathered_len + /// What each thread's payload scratch holds for this plan: the largest + /// gathered payload's bytes and the most codeword segments of a block. + pub(in crate::j2c::decode) fn payload_scratch(&self) -> (usize, usize) { + self.plan.payload_scratch() } - /// Bytes a thread's scratch holds at most for this plan, as a route - /// that is not `lockstep` charges it for each of its threads. - pub(in crate::j2c::decode) fn thread_scratch_bytes(&self) -> Result { - Ok(thread_scratch_bytes( - &self.plan, - band_scratch_samples(&self.plan)?, - )) + /// What each thread's scratch holds to serve all of `routes` at once, + /// the largest of each part over them: its bytes, as budgeted, and the + /// payload scratch to reserve (see [`BlockScratch::for_threads`]). + pub(in crate::j2c::decode) fn shared_scratch<'r>( + routes: impl Iterator, + ) -> Result<(usize, (usize, usize))> + where + Self: 'r, + { + let (mut payload_len, mut segments, mut samples) = (0, 0, 0); + for route in routes { + payload_len = payload_len.max(route.plan.max_gathered_len); + segments = segments.max(route.plan.max_segments); + samples = samples.max(band_scratch_samples(&route.plan)?); + } + let bytes = [ + payload_len, + segments.saturating_mul(size_of::()), + samples.saturating_mul(size_of::()), + size_of::(), + ] + .into_iter() + .fold(0, usize::saturating_add); + Ok((bytes, (payload_len, segments))) } pub(in crate::j2c::decode) fn strips(&self) -> impl Iterator + '_ { @@ -1909,7 +2032,7 @@ impl BandRoute<'_> { output: BandOutput<'_, '_>, workspace: &mut W, scratch: (&mut BlockScratch, PayloadSource<'_, '_>), - decode_block: &impl Fn(&BlockJob<'_>, &[u8], &mut [f32], usize, &mut W) -> Result<()>, + decode_block: &impl Fn(&BlockJob<'_>, BlockPayload<'_>, &mut [f32], usize, &mut W) -> Result<()>, ) -> Result<()> { let plan = &self.plan; let strip = plan @@ -1968,12 +2091,10 @@ impl BandRoute<'_> { where Self: 'r, { - let mut plans = 0usize; - let mut scratch = 0usize; - for route in routes.clone() { - plans = plans.saturating_add(plan_bytes(&route.plan)); - scratch = scratch.max(route.thread_scratch_bytes()?); - } + let plans = routes.clone().fold(0usize, |plans, route| { + plans.saturating_add(plan_bytes(&route.plan)) + }); + let (scratch, _) = Self::shared_scratch(routes.clone())?; let bytes = plans.saturating_add(threads.saturating_mul(scratch)); let sample = WorkspaceBytes::sample(tile_ctx, storage)?.with_tier1(bytes); tile_ctx.debug_counters.workspace_peak.record(sample); @@ -2050,7 +2171,7 @@ mod tests { band: 0, }; let mut budget = DecodeAllocationBudget::from_live_bytes(0).unwrap(); - let mut scratches = BlockScratch::for_threads(1, 1500, &mut budget).unwrap(); + let mut scratches = BlockScratch::for_threads(1, (1500, 0), &mut budget).unwrap(); let charged = budget.live_bytes() - size_of::(); let scratch = &mut scratches[0].payload; for (code_block, len) in [(0, 1000), (1, 1500)] { @@ -2067,7 +2188,7 @@ mod tests { } assert_eq!(charged, 0, "the route charged the 1500 bytes; no overage"); // A payload past the reserved scratch is a planning error, not growth. - let mut short = BlockScratch::for_threads(1, 1000, &mut budget).unwrap(); + let mut short = BlockScratch::for_threads(1, (1000, 0), &mut budget).unwrap(); assert!(gather_payload( CodeBlockCoding::Classic, &job(1, 1500), diff --git a/crates/j2k-native/src/scalar.rs b/crates/j2k-native/src/scalar.rs index 851491b6..a7dafb0d 100644 --- a/crates/j2k-native/src/scalar.rs +++ b/crates/j2k-native/src/scalar.rs @@ -55,7 +55,9 @@ fn internal_j2k_sub_band_type(sub_band_type: J2kSubBandType) -> j2c::build::SubB } } -fn internal_j2k_code_block_style(style: J2kCodeBlockStyle) -> j2c::codestream::CodeBlockStyle { +pub(crate) fn internal_j2k_code_block_style( + style: J2kCodeBlockStyle, +) -> j2c::codestream::CodeBlockStyle { j2c::codestream::CodeBlockStyle { selective_arithmetic_coding_bypass: style.selective_arithmetic_coding_bypass, reset_context_probabilities: style.reset_context_probabilities, diff --git a/crates/j2k-native/src/tests/multi_segment.rs b/crates/j2k-native/src/tests/multi_segment.rs index 31262952..d4763591 100644 --- a/crates/j2k-native/src/tests/multi_segment.rs +++ b/crates/j2k-native/src/tests/multi_segment.rs @@ -133,6 +133,8 @@ struct Decoded { samples: Vec>, streamed: bool, pool_components: usize, + parted_bands: usize, + uncharged_line_samples: usize, channel_bytes: usize, /// The bytes the decode's budget held after `reset`. baseline_bytes: usize, @@ -174,6 +176,8 @@ fn rows_with( samples: sink.planes, streamed: context.tile_decode_context.stream_rows, pool_components: counters.pool_band_components, + parted_bands: counters.pool_band_parts, + uncharged_line_samples: counters.uncharged_line_samples, channel_bytes: counters.workspace_peak.bytes().channels, baseline_bytes: counters.decode_baseline_bytes, } @@ -270,10 +274,56 @@ fn multi_segment_blocks_decode_on_a_pool_as_on_one_thread() { } } +/// On a pool, these tiles decode on the pool's own routes: the tile route +/// for the planes, and pool bands for streamed rows, which then hold no +/// planes and leave no line-buffer samples uncharged. +#[test] +fn multi_segment_tiles_take_the_pool_routes() { + let mut parted = 0; + for threads in [4, 12] { + pool(threads).install(|| { + for (label, bytes) in FIXTURES { + let image = Image::new(bytes, &DecodeSettings::default()).expect("parse fixture"); + let mut context = context(CpuDecodeParallelism::Auto); + image + .decode_native_with_context(&mut context) + .expect("decode fixture"); + assert!( + context + .tile_decode_context + .debug_counters + .classic_task_workspace_growths + > 0, + "{label} on {threads} threads: the planes decode on the tile route" + ); + for roi in regions(&image) { + let decoded = rows(&image, roi); + let components = decoded.samples.len(); + assert!( + decoded.streamed + && decoded.pool_components == components + && decoded.channel_bytes <= 256, + "{label} {roi:?} on {threads} threads: rows from pool bands, no \ + planes: streamed {}, pool components {} of {components}, plane bytes {}", + decoded.streamed, + decoded.pool_components, + decoded.channel_bytes, + ); + assert_eq!(decoded.uncharged_line_samples, 0, "{label} {roi:?}"); + parted += decoded.parted_bands; + } + } + }); + } + assert!(parted > 0, "some pool bands decode in parts"); +} + /// A full-image rows decode on a pool whose tiles only serial bands can /// stream (here the pool's routes are switched off, as for a tile they /// can't decode) streams from serial bands when its planes would pass the /// decode budget, rather than fail on them, and keeps the planes within it. +/// With the pool's routes on, its bands stream these tiles within the +/// budget. #[test] fn rows_stream_from_serial_bands_when_the_planes_pass_the_budget() { for (label, bytes) in FIXTURES { @@ -304,6 +354,16 @@ fn rows_stream_from_serial_bands_when_the_planes_pass_the_budget() { streamed.channel_bytes, ); assert_eq!(streamed.samples, expected.samples, "{label}: serial bands"); + // The pool's bands take these tiles, within the budget too. + let pooled = rows_with(&image, roi, |context| { + context.tile_decode_context.planes_budget = Some(budget); + }); + let components = pooled.samples.len(); + assert!( + pooled.streamed && pooled.pool_components == components, + "{label}: pool bands within the budget" + ); + assert_eq!(pooled.samples, expected.samples, "{label}: pool bands"); }); } } diff --git a/crates/j2k-native/src/tests/parallel_bands.rs b/crates/j2k-native/src/tests/parallel_bands.rs index 8f44781f..2e8a6cd8 100644 --- a/crates/j2k-native/src/tests/parallel_bands.rs +++ b/crates/j2k-native/src/tests/parallel_bands.rs @@ -152,15 +152,12 @@ fn pool_bands_match_whole_window_tier1_bit_for_bit() { } } } - // The route decodes HT blocks and classic blocks of one - // arithmetic segment. The encoder terminates every pass when it - // writes several layers, so those classic blocks keep whole - // windows. Classic tiles with fewer rows of code-blocks than - // threads do too, so only two threads reach every other case. - let routed = case.options.use_ht_block_coding || case.options.num_layers <= 1; - if !routed { - assert_eq!(pool_components, 0, "{}", case.label); - } else if threads == 2 { + // The route decodes HT and classic blocks, those the encoder + // terminates at every pass when it writes several layers + // included. Classic tiles with fewer rows of code-blocks than + // threads keep whole windows, so only two threads reach every + // case. + if threads == 2 { assert!(pool_components > 0, "{} never used the pool", case.label); } } diff --git a/crates/j2k-native/src/tests/row_sink.rs b/crates/j2k-native/src/tests/row_sink.rs index 6f21d17d..7dbf6c2b 100644 --- a/crates/j2k-native/src/tests/row_sink.rs +++ b/crates/j2k-native/src/tests/row_sink.rs @@ -353,12 +353,9 @@ fn pool_band_rows_match_component_planes_bit_for_bit() { } } // As in `pool_bands_match_whole_window_tier1_bit_for_bit`: the - // route takes HT blocks and classic blocks of one arithmetic - // segment, and two threads reach every such case. - let routed = case.options.use_ht_block_coding || case.options.num_layers <= 1; - if !routed { - assert_eq!(pool_components, 0, "{}", case.label); - } else if threads == 2 { + // route takes HT and classic blocks, and two threads reach every + // case. + if threads == 2 { assert!(pool_components > 0, "{} never used the pool", case.label); } } diff --git a/crates/j2k-native/src/tests/workspace_reuse.rs b/crates/j2k-native/src/tests/workspace_reuse.rs index e75effd5..b48e5554 100644 --- a/crates/j2k-native/src/tests/workspace_reuse.rs +++ b/crates/j2k-native/src/tests/workspace_reuse.rs @@ -574,8 +574,8 @@ fn classic_tile_route_matches_serial_decode() { // (case, codestream, decoded in place on the tile route). The later // components of a colour image decode into the reused coefficient buffer, // most blocks of the three-layer OpenJPEG fixture gather their payloads, - // and the crate's own layered classic encode terminates every pass, which - // keeps it on the per-sub-band path. + // and the crate's own layered classic encode terminates every pass, so + // its blocks collect their codeword segments. for (case, bytes, in_place) in [ ("lossless colour", gradient(3, 1), true), ( @@ -592,7 +592,7 @@ fn classic_tile_route_matches_serial_decode() { .to_vec(), true, ), - ("pass termination", gradient(1, 3), false), + ("pass termination", gradient(1, 3), true), ] { let image = Image::new(&bytes, &DecodeSettings::default()).expect("classic image"); let mut serial = DecoderContext::default(); From ef0793034f46dddbd7a5203cf45934373ba49d62 Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 00:12:39 -0400 Subject: [PATCH 60/94] test(j2k-native): code the multi-segment fixtures past their first raw passes Most selective-bypass fixtures stopped within a block's first ten coding passes, before any raw pass, and none held a segment of coding passes and no bytes. The bypass fixtures are now encoded at low compression ratios, so their blocks code bit-planes far below their first four: every one holds raw segments and arithmetic cleanup segments after them, the ones with per-pass termination also hold segments with passes and no bytes, and the archival-style one, with 64 x 64 code-blocks, precincts, packet lengths and seven layers, reaches 15 passes. A new fixture codes its first component in 32 x 64 code-blocks and the others in 64 x 64 ones, signalled by a COC marker; it is spliced from two encodes, and the module docs give the commands and the splice. A test reads every fixture's codeword segments from the tile's records and checks those properties, so the fixtures keep covering them. None of these encodes holds a block of several segments whose bytes are one piece of the codestream, its other segments empty; a unit test now plans one, ten arithmetic passes and then an empty raw segment of two, and checks that the pool routes borrow its payload and collect both segment descriptors. Collecting a block's segments takes its code-block style instead of the plan, for that test. --- .../multi_segment/all_modes_gray_97.j2k | Bin 5461 -> 4712 bytes .../fixtures/multi_segment/bypass_gray_53.j2k | Bin 8198 -> 17558 bytes .../bypass_rgb_53_blocks_per_component.j2k | Bin 0 -> 30477 bytes .../fixtures/multi_segment/bypass_rgb_97.j2k | Bin 6137 -> 13399 bytes .../multi_segment/bypass_rgb_97_64x64.j2k | Bin 5253 -> 32840 bytes .../multi_segment/bypass_termall_gray_97.j2k | Bin 5464 -> 4703 bytes .../multi_segment/bypass_termall_rgb_53.j2k | Bin 8206 -> 15236 bytes .../src/j2c/decode/subband/tile_parallel.rs | 105 ++++++++++- crates/j2k-native/src/tests/multi_segment.rs | 173 ++++++++++++++++-- 9 files changed, 255 insertions(+), 23 deletions(-) create mode 100644 crates/j2k-native/fixtures/multi_segment/bypass_rgb_53_blocks_per_component.j2k diff --git a/crates/j2k-native/fixtures/multi_segment/all_modes_gray_97.j2k b/crates/j2k-native/fixtures/multi_segment/all_modes_gray_97.j2k index f0a4bfee1ca343fab04f6aa92dfcf96b233958b3..6e25168ef84c02a8d863715f7cdc51caea1eadb6 100644 GIT binary patch delta 4399 zcmV+~5zy|{D(EC2|4;u>04V?f0000$0000$00000000000000$0000$ksw`v5%B;4 z|C7VH2n+xc4<@hGSk>2&xZ2A=MOFZbAXLSR303!pUGM5NQh_92`O-J8UF{0={bokY z_pYc2t@RsT+){;GXmKLVC42g)!@0&G{o%+hc|B{xXo6I!li~pHi|L0A=<^q?jSzo^ zzJ7__C+p>EebSj#?u(GX8LSe2iRN6rl{5St3?2ONC$0$%f-As<03f+kjvh9!9uAE~ zc^hQkuK6qeiQ_s-D-fpG!E|PIx#BP>8%#1aY^@o+^B~h_In8HFc)0MW; zD)8n8)3Ybp<`zXBkC7p59jV^$KVk?;=a9w+@ekb<#!`Jb#9P2MR5io&Rq*mLxgHD-Mg02C;T zB(N*{43KPL z9qUFH8d6s1Rt0cDR z8tck`4CAn!^Y`n4T_!KtrZuGfd8!Y+%2&W9P^by^z_z>EY0*5m4le+-M;uwXa;K?E zFGm4}BAG2VvElech`+@r+9T;nj>if~){NPSt5*!3K!3F?+gaZo&I|+Od z{69DnnFNOc4e&yI5JZ|!iXZqEb7y8g4@0?ss?kLU0Y?ls5eR^67%6*V)xja!fFS+| z)wZkpG1t@W7}cv*U5Fy@2+U>u-6Sw^9LnM?T}9a~O&mw^LY66kUb+%eYq4w8^6BY$oJ9EWGl8h`aYXvbC^sAr&hyJ;L zi3Z@Bwz>r=k_>SOGWz;NrE9l$Y(SJJ0->Hc@`0=S9HEJ9BDDxwOjA-;WR-ibAY$V? zau<|FBHkn_?@mC5c<DzVP`O}5CaNg7Gg8}~KqP5=LO3N=)wb$!mThGj zvqkEt^Abj;V^J0c=0v3f6COas+S3_-@WvKVB`UVp-IZoP<(+==rTI;8(Mh8Q9`Tc2 z^yk1N8}j+88v=UqnYOkf3GTecV^i=TSdaCDJ3fFso{|Z&LH3bU}Ooi3Hv?T28Z$nQnRIgg@`tgbw}p2rw*0lL-bLCt^}B zu#s_F6jmogcAn1yT95F*nDz7~%vq1XL_p0z{^|7Vo+lpelgkJkM)c(XzZhaY8yKN$ zJioi335!!$Ih;&l%0>a0$O(KgqvB)JLScSst;S2xI|rFbycnHr*T(s8_jcs^TFqq@ z9pw8e*ID6xek@t4b*-F z$Z8jOHS)5V9L%4`Dt&e^&gveQRRs-zUraO_Js;8LxDQ`iHG0nI{9^b0S#)W zW-&zblfVidlM@mlf1QjvHvkwEl$IEI;Itzj1z!)(?4JM+`LE|JYji5s_56d*Im7dq z4jy@mlm_Fy4K$D|KrO*A?UQbU^i++xbcdntk=VoAZH_byAup@*;!h}IXy&te?7)4e zHlgNspPX3eHR|#euUg-cEL>yRYB@P>uP8g1n$PVFqT4*be{IQJ=z<+hyjXrlN}zBw z0M>a);KL25O69I9IC{I${v!37E>Rz;mIND)@*s&rA#1+?#ktZKm(epE3vpKBVSVEm z^Ca}Ddbfu4va7yu$LqS`8SVuY&ru39$Kw0bqKx03G5GEU9gYmGaAoVEMr9CNC@*2n zL}5R>A+JAYlOzlse;{{&P62p4+AIroR+H0B0H#W-sD^4Op6SM_CrT$&K=M5nF9-l$ zH5a@Rh04Y3G_Yw%3#7OHfSgq`^iUrr5r)!swaF+nMFKV;@`ed`;`Z#zcB6Zfj~Kp= zVu+c6UukXjv_*h1GIB1ws59tS|23VVxv|lZV(4@Vn4N=0lkE%#TV2q?th8`-mb_Ku z5OjVga9~gQYzO2M=hHzb zHxj=UV{PHmFKv=nM{``W?ZME{CW3JJ<@g`3lO_)?e~iLYVqaf3b#`>3@EHDBzaXDw zvC`-T5a7#oyU8$w5vo}%n(2vB*BW*n-v!MSeavm6U0~Ngj6doJNmwRWjXJ0z02kTCYY^p$MupnKejbB^L_^k%=LS*8)`$TWB(S7>}5!nOW`6|Jy zgoh-;&l$(6delRTM>zmX*UmTN|M>;cphhMLy;WyNB% zPvL5ZBXgvzHRd`cDBzN2XFv*tC#OvJ@z(+4fAP06SNg;US;MMB48u3nhVip;1nC3l z6}^oYkn+DPB2Vew2qC!c8w)b#h~Q{&fd*F6*CqT9t>xnUxU`i)y+jEGVFq2V!SubB z0q)23+_F5VI||5TS)W9JLc~zp8gYJwq`n3Li$gu*4F^vBTPd4c=&Jp%Cje__02C zc`*7(9by@Q333YZ)+Jq(wzhc+5*iu!SqaKJygYIgJApfB9`XJ543`$wI1-BDg@+72+Q7g|_R*awes=23a*# zqt4%uZ5)@~)zyrwFEvV~5J|8sIxZh_u_fqG<+sO0D1OZ{; z{GK!_}LrWVvFl!ALl^w%H-g?M*YSE6!yQTmh+zFdB{UR5UztU9eeS z73~rT|6Z!6Sg2m-9Y7|jEvmLuDl;?Ua?O|KF0D~i zasCqc8uNxQ+pzxoz)o#ZLiE0qe0*>iRkp43Gdh(`2@JNjjQ}8hUWeWH`M&}h!d3i2 zf=ho`AuMKDe|7ZXmXT>*pKi8NX}bif%iUmCGsVER-6pvr;!Z$h(<(wuh`}%pNusot z8g$0*5v+|#M`X~qdcl>p-7FOw4lb^^-3S$=Crrfe`>#t&-dM+Aql#JRXiDtAx)rtv z{~9ko!F|Yp&OozB9&8G;&DkRaa?4yC`SF!_oeef>f3~*nwUXqLL^5A8$P=0?U&}En zf>=eobDmeG#8f^uR@!$7+(hXEs|JC zs4C=MA+|wE&8i1&{ZIJK=wJ6uq!XzBP!7tWw6YJ!UE9=}h1oQZwi<=}b1a2v@XPj> z?q9(l6tmBorAH?v^2z_U2L67jMwKOT}fSc%$TnHk(j)?$%sWCe-n zzJBLbi2|+O^z5U3uhjvNhC-jRkAf# zGc+w~52djK!3^isQWwg$!nKHjm7bpL@4m6Te^fNB>kj#P9#&Yln`$u> zxiO+L7Pjn10AUQs(jQjfh7~dqtX?Z*e;fgL!StW*LHDS(!$uN4_-6^W?0g!cuy3so z@>>%Pr`KbxG;DXS_K9zjb;rUukIWrUlV!QK`|0_|%5ez&(FoP$@l&4WFYV)IJq>>z^)dURFjVX!VKY?~p}4XQ*4sQeXV4e*)~bO18EGf65OI z^MEL313a)?=C@|C-3k_63mwLe*?YMeU{IM9nqJ*M2|^Aw7^Pwko~<8N`AC_MiHJDLUA|kijv>$tv}6afJ)bj5wi4A~L%w pwuq%(s0g!<9!lIXsFwNsLLFlY5efPh{q)Awri>D=uLaGv|JkfPOKkuE delta 5125 zcmV+g6#DDvB-JV)|4;u>04V?f0001h0001h00000000000001h0001hksw`v6y5*< z|C8aU|#5)US?)mYWnCa;QQ*(psV3eL~BrIz|e+WV=A*u}TRRRn^_0Fk)b%Rog| z0Qw9#M*J9|pN6l3FkiwqybAFOi6B(PiwQ^p??~xYa>b@M$6f6T^!;W=$e_zamqjyb zX1k^e%wPr8Cw&Gdin+!j@!`mSEqOg_!)St3sg#40G>@k6ycdjWylP$vz4C{06;C96w6cV9#r zDV=F=RuSj(pKT6BhxXc2y2e5sMXwgvIGo)!Z23YUMnnQ-l@tXvqcq8l5wD!AWpDuP4&RtmWP1HU#a6(E)!w}Bd0W|qxpOf_A7TCm9Z77v%8%Bg z#MkeD1VMQ|5_wDst(Gdkhot38JR@?*e*QI;ym@5HgN%?jwj95h^9Vg2;fO!jH0BU@ z2i^=IWdM`90U&>qHD-Mg02C;TB(N*{ktZT)CY#2AyYW{z|;jh}<@|0n7NBA*Aevvf{ z{}f^q%3((BZV?$!yicXJgO(ZZm z$7_$`z2DSKFEqxASgncVykXZ*(bh4z{7!6=r;C5go*nqyg(GtTBT4)gpFy(p8JjgD zKcGGs0}5-Qs_OPz>L_b6GjXiI6uJNv?>Utk7>DYx#Y`5Ut8V#9J;P7Ja23f6%boG1 zbt={T3tdf39oeUYdq|)9fh3tKjt#}9qUFH8d6#8i!0w2 ze-DZO62enZ@bpNaPZl>$-D<$VNKv-Cgkfj$`=)(%?kwsf`drqR z2c)y=tU~`{!h3PiqfZ&_)j@j(LBVVQRvud7rh!GDP{773v0^UYw(+geTl21k&08?u z;tV)t9$Bu5?ij=5>vi`Kq0w8~KH-Ev`F9%k3?cdM7(?&g3?N`mU+yVG=Z}DsI0YSl z8yKN$Jioi335!!$Ih;&l%0>a0$stP+)}X(8k?a~4rAGpq3;9wh6KPxnk_D6o!}YL- zE83W6!<3B?Xyn`p753vWcmoByfD{|1k3XN+Q~MUNKxA7abdA+!lMuljOfP_7RzR$M zZ=^po5&~5zQ<3WnaUzdjfK2znJ=V?+DrfB%#zC=P-bZ_iQhSp*1{;6NCBs)xySb@< zpKfrf01zO2?sxw91P`);lCW~*-S~YFp3TV@2O!*UiBbbmqC~Azv zaRlFvMm~+^hG}Y&MAAjTWNVjCx-=9jJK;X8Tl6#SLQ83ZM8wX6GE5Nydi{;n8gg|X z{_;Jb0c18bVXOhG7>R#GxKN=2Tx`E}7!;J27elu=gS5knZf1*R?}b!xWCi^Jx8|s2yOY#y=_G#wz}g6#eZW1FZWMDF z03B227o}qL2OUf}NULy?j7vVrD%7@J8V?d&6Wg@M1cq$7Q_Ng5UJ{DxRxcwe>Z8}@ zrO7`)2u}|T0KjM3;I3mkh8}9r#7;XZtF&8T4iOuAEi{7^azGwoh-nm@y~7xP`-mX% z?ifSk+);>fSD7<{ewxML30&^^NtZ|)fbx42>s1KctP z9^r&KJ;Mll?ifR%T?moSB(}x704GCsp3eeWkR3z`B}Z;1NgH4B9h+Otz)D0X{loFA zk)nW^P1!U9h>+t-=_Y)+DICqkBoSV7&}LZ($pd&LW)}gQlY9jnfA7t);b+!2WJKh1 z=(v!ja1D`*TQjh3I)ugTO;sS@nwT-OFj7u8h!9J_-s~Y!mD??&G95A^8(1wKaL?Fu zFg0+pA;Ck>oHfy)*&%ei2QCgvHuM%b8lzp{w9Uue*7&0^zI!1^wB0Ka>-N-8V z!qc>dWn`bN5FD%r<;DGz-UT0j8yHI;N&0##prT$!0}ukMz*`~dohvPJ0SLD5AQU5U zsb3VCsb$gxUI>lY)SVWp$BZH{a(YJfnBCoElOC5<1r2~-Of(cjsz1o~g$gV<6c{V~ zY8x2$FAuO&vX{XioT5w<7Gf%ifolrxVGq{L?+hCO8%NsI#Y~j;ZY;omhW}Djk9RijEcT(3BMe za);KL1&#;dm&`^(qS zg&FMK57QHGf-aMn01Xk<-~HgJyHeYfd0p9xasWcB-h{RLfLud|2sncP0k*r&l^dbk zm=Qd#5=A|S1k^J&gB-yuak+)jS&pVcbO18hEC|}e=Nu5_xbc8f{!G?a#-a}Sn`I<_ zehJYLpOh;iTw1NPwKf|DE+uD(8|;b4J_$G*?C`@y=vwizt2kaz3OC{4!3F3=Cyo)E zq7+-Zed zMDe@a@#a3>0H(QpRbMs=UFUY#U^ybx^P+7k!WA7txf3(^gbDCNFQI9u={c;uyqZH@ z`mJE~Q>0Y&oJxj)01ehM<$qp&PiCuPE@9QY>#=rO-lrq|p{u?ycL6 z#b=7Qs`?Lw#s}fHqp%v6dO%J>u;0lTG&wDp(Alk2>rF)W{p}DcqkYea`3YeXP zMu=JYWXQu=RI?`u3q>H7yjFb>bbcprU{CpMcGJmzX|2Xf&^rg2NxT@HZP&){)y?cj zpsymUxgeXQR!5#a&Q!keu`O)p)Kk$H_ci3KnqvxozZnO4iVTKs)_o}y*7CDo20Ocm z1u7>+ywzRia9PdPN-`~ia`6Fs(NqXFK+(h#7*RHt;^w%svNdnnxTmO6OFd&N1x$^-}4`l*RjkA4ReHAn=~t?%#dLc5VOK-WRx3%7tM21zH1)<2213h zwt828(<8CErpw-Y%;98Aw&7d1|7l0ZR0|dlR#f*oK#X~pAtGvJu%>$`4@JLF)s_Dg z9Y5ik{l+S&Y3-)^t=#as>qtEzpuL@?ckqg_;CeBQUcQ1Iq&^3^`x>c4rnm(40DX=M zTz2CJ{t?2ZT=}q58RY_rXB$1BIy+1vWo!a}5HdI|g%jgEA}c<-0A`br{c=<=$Ru(!+^PI{AR zJsI>HH*!rF``7OGX~Px9&jn&VG&L-~oQEe+-_!erBytq^?`0OITl%h6SjoCA3-%L5rd0y8lAM9F6PakK4W#>Dq_(E%@lpiZKGXyswHal&*Z%-gtyRC zs{R`gj$zQiZ)U1u1w2VV2kW-XfrZ!S z>z>8GX_-8J7?bY@i8g~SILDy+WSyw}d0hlICtF-j6=b^kef1-SblZk*{`%lQPNTax zS*(AJo#;FKB!O%UE{(TbxaX=)=g+HJ5Dz45G^+RJ$SsJ=*l@MG%cu`F z088u#A9IRoDU&(>_6 zj^Cyllb;45e>x>7;F4x%KnjH?r^EA@4jy@w(pb>etkno0o~a@=#aF(0^C-+AB1_kC z1%}arF1?Kxkx8Bc(+HVB5$8pCW$dml@1QgC1#i+4dYk9`^pytzF@Qzmi&D7 z{z$LGbTYow4Mo&sxka7iDmawHX#p2|$v?4{Tgm_zf8O<{;@e67YvHRPxYw2yEB)(Fl-w$kNQmB zvWm69M&xMI6j<*&#cB4xUZ?0FyxPM^fKMgO@DY63uI;&{J%a_#aP}$%V#n&T41}e9 zmg7cj9oSOvC&Behn-WA}a247^y*=2T`*{G(ot275EP}RC;a2HWfAANz=J;+kL!YzEv&~HK&z={BvP?5Ki4xR{{ zNrFi?_9AfRZ;Lt>BDqRk`I#Us&_E2jEOS(2GWD^0%j1K}YzXyV2bZr@l~@{n50EH^ zBdlO-i0-PRp;nzCJqSW2%cgpm`pi#39F)%p z2_OD#I#Flw+-;_Rjdje;8z*Z&7^na_de9`>Q@Xe?caBVQh^`|ot>+yTirj|nF(drx zdFO2(Qg;Nd3E0~n(ZL%pH}F@jyEYXg@Z@4Z@Fxc31@qvQ7xXd5C4KJ~MYJKXHNTeV zSK*0_*6L|BjYrr!7tV#_DMwvK@pg!DkL-5u(_^U!>~?1QQ?EIuV8Rh<$NIIL?dmsI zk-4bMvsigKUE|3vPOe1+e!NdhPx|+HkAr2c7u9p#9_Kukv(*5BQ;($Sz`rhE#g{NG={e9Il^K#H!IX!<4Ri-jA&Jv?asO0!<1wr${PEL-9M1l<@Q+oK~8Fy za}b-9(5gNcZFlsFyYRyM&2gzm^6R6R8f?=0)v)UezS>95yQXVi%-X9VFCKW?yXI6w zVf-%F1C>WVrtLkw(05?Ri|-~v{uvr?VGmUMx@b5fd%fnye8q60D4)9u;e)TQZR$1ITKw`RJ>BPM?aVF}f zchV~PcsZXOX_kL_%LiZi6}yUtSHlg-ZMesj0-|n|Bgq0UsjHHNYZJ~DdWMOhQ{z9Q zbqvV{b6DX#qSP-a$s8{k`VmAaE?4L|>vINQ=tm>rf4zkeSjSvba5=$+zn!tXv$#i} zM+)+4#%Hcni}JTcslTq4uwz_BV8zmNN6Nw~AHugFRVn?sEPZI|q}9h$k@KjOqgkh2PF>ZSTcKv*t@p z9y2eGgUUS-6C=gI?Ar-01^GCKnYj6V3b^ZBVrp4gE+EEX``SQi$6UAsx zqj~+w*HP9jyS~tN?tyno`6IB}%3auc_c6hFfrbeMVQX)vrB^!TTiZ|`#c89kJh^iQ zq>!D&9_j~^R@HGshPIML~d)3TU1#Rc``*X(E-dSU6~=qMKff%P7aJsU_+`69{wKRP z3)r6?cdK8ARmpcEusQD?4>c5VTFdfb~Y7D-s1zfUhZcfnuJ z^_9%xE(we9y;)aW@DIXI-7#Y+XR+VgTXs4ayVBQHE!<(6I_2I7dpsgk+*FQ6dEI*^ zV^PRAfWVo&Gw~i3;T%;$Z@;Q|yT}FQbxraauPU5PZ+a@#@PJDjjJfh4qcFlv4&8Tk zcW-)q9aE7Du~SxelCg{$A|;+mkeq{B(%@+M(V z4`|Za*ak~0wkCXewA)q6%%p@~vAQhK9B*!)iNK+ve^lo#Jax~DitgLzL=xU>{OZkq zNBel;w;z{e-}5_|zosFugZ6BRLBi6KLGv|~fWc3LBm@pZbB3H}FxNq?-z*iGw z{6_m@xASJA(o8881J|4`!2EZ8kge&h66*AD8_qW?pAq;R_gS9JT~p`;l(5hq4*M!q z?eeZd>85W%)R&9)`x4R=5ZG~x_!zdM^4O^3ebLaW_Qfz!O- zns;yIq(pUx-u<0|G9lR^4`cZ6?Gw3Wst5MX153?OCpdJj3p@ztSBO$c&>vp4dm;95 zWpb}L+WlZ&4UiSx`@Da@arD}3ifAw{zLNje*2yh~PvIUVDyF7dT^Birz=l?Ts_9SR zB}7l2w?V5!=!2=#w*8-V^yT8UJOeQ{93O`I_lZ~I zn`4FC>gO^sg&|96o*?`~c3jGrMW;lRWU zR{iguzO19w4AJnZQ;OLqxYq{mX1T!V`MBwmpE*l`Wb-FRPCLF?8d=L<%eU`a@B5Q2 z^+BL|d1lj$c)i2}i3gW``jpQ){`e~4ArWb~#GH|y;jOFxqxa#__wc3orN|FLXT`!Q zmqwOe3^|@@s972savZvH2KHX71~2cP(Z0X=T0xO~h0gie&tj#^Zysqh9&U&wjy1o% z&wk)pzj06abTn$zP4=ApDJE)@2y-`&1XFaAB-2|^oOz7r0CVpqIc6%4B-445EVG*D zG&5$C3{#v}npwO_imAgZi4AQ?%6iT0-XtyQevMwgrjd@Qo_>ZItu#V^+K07&e&(0$ z#|39kdDmX%B2$QyBsU|x$AvAE`_YMDURMckr;p`aXQ{>}@21gj0^g$I-bT<)5|>+A zn6m&%4ljv`WhuKV9q?uwUx<6kY;*vk(hD>4<>_4j(LTkut}%Faainof27ySX68LEab_Js zg87_FmN`g}WX^NRZ1eJS-Cu2Um3`#aYf*!d+%(OV!@jalBB zo?`w2CWsiCdUzK2ST zTZM&rIrYs1`rVgip5rEw7$%vkxPu}icP@=po!h~ojID})@-+7I_+2#zfzRd_hM6!} zazCF8GnJdfT;h{t=5R_eh4`hJEu6~CJ^Zo)O0tciA^AF6f9{ns|I{>QZvXu$(~)0> zmv8LEX^roSbT3=yR=QI+Gm>A1OXrS816)nSD<63)#=OlhPvnv`-bWG5Mf}oDQ@v11 z9TNP*Av)Hb;T&iT1zWo-FX{R zOhAUH_(yRFKC`yX|EpGb@8b5&saKTGFew6(ye?m@PYa5Zcx-$|K$k8KhrSDzTSM8KP zyKL5>RcO0>D2XA_4)zSX>ztH2`Qqx1k=sv>40tK}O#EcF2yEktmJ-(J&h`tl4xM*Y z(q&EwkOgs;-)w9G_V7IxnDsUI=4M-}yX|}glaDCNE%hm}^wP)6^}^ZvlD08-5fyn4 zi|d$$=kd0q##?SI`7SV>i84pGA3()lVsCuuaBbzO;*@W!w(u&>5tV-ly&Nh#BC_x_ zUxjDxa>m(*1IpQCfB?(`6(j zH%;*8bK;D2e;r~KoR5!TH@fgI|r&!?mbXBrg)h&S01d3Tcib{rRJ`1AOP8Sx8jcOP7_+%f0HS$Qp z1`M?M#@+qEWMIs=%-^Zw`l6y_B*XZxGWbA+ykOE!HX!5*^=$(vUFeSbr7m=H7%Uo8 z%YT0lD(HwPNBx3`D02q*8mP24zGm|GVvK24n)L zzkkbu3+vZclhp7!x+M9mRH0Y*uNQj;{E7Z3Zp=LNqxnzs;8=gpd-(QCn&2@TSlqJU z_+{^H+1Y%<=zZ%JnC5g;k=?jR}wGeQm$O?^2pAv$IjBOySBED!RmD`2wZ3`^}|N}K$mc3f$Xz@ zaLF6$$HK*v2T~FE*Zt_{uzyFNd9*;c%+IE}ZCY=AR@VY-@RQ5=!3RGNU7o^QA+SCY zjMF~u{R1`HO4Ft3VD+I#Q90X%#i(PHs3;B>UUsl84cQ&Gv(wD1a~W3{{W=a1bq}-| zfZo<^)%N6CNjw9A7i4O_>J&CJr!?bF?=3Kj&T(kk6V-$VC7(#XUiZqBMc#GYHUrMA zlU09i@3^<^$bMhy7UDG8jUaQ$9z4m1#0a9|v88Bo@M*rNNVKAqtXNj)_8gri8eb%b zvkAbtM6J;9_xD%dn6v~;8lO>;D0JA)22fEKs4zMT3ZADfK2f(va<%_Bq@@3BIyZHt z4ZiI`t1F`a%I=+5z9qSL&V)Z#;#v+{;E1UTskMRmgj+$jH&@i*7?n8e#v9)B#qQbO z{umoU1ooc}G)&;xiR&{qq3H&6@=MA~p5i~IbQx!NaWj=dGHU8XV3Lr4wX(93okd-$ z|2aU5Rrf$UK*cDzRwJbBQm#-8ic61j-@2<+1$XA}C*1s`ak}BToVO>hkV_CwM`I^> z3&U_P?Z7d41pdG+dy^jaTIPFcYF>Ma?zutXj89c`9_VCTTbv4K`Q&L?aGr8_`zDJm z4(x|`Fm#2F-v|X1l0jN)PEtBg8=lIKspUi96yOiw5gV2_&IJnk`@;dAqj}?=qb#M` zy1iIRed(!ru1YX|e$!E9r*r>SIc%Ww${bp)gEA9wuKd9;&*`qPI|B&(B&3s&xVq0F zBmL~EmhrP9p=+OQn}xn%*$v&Eo4xV8QvjaCl(8;H;@LMxDV^5*0Xc`0BqOBI!nT&K z{jX{4Njw-k@Sp-u2P(Uli`ivvc(%TdQQ_L%B`Bm0h;u35&ayEAN2Jr!Tiy8MFs=QM z!c_L&YOi^_ZN%`zth?a#JqM|A`$x~h@4Z9kHrcxJ-mrQZRDwau=vz30tqV+yuE9Sm=}g0H-VI49jR(h+mz zy5WDy34srL5}&K}8>;cR8SqGG5f8?0C8N*Vt-HKG!T!~%sAlz4A6Pza;Z`=+H6_w# z-R8kOJqtS5N3Mvx2o z1M`<268hb9X#49g512B^r)q5vdau8^mfkh!aQx17zpP9Hzl%T4&%Cs?)gte z*wdsve+mvHy#w)A=oH6xZ-6asEB^`sYPyMW5Ifc?bAxVszphBJlite#7VP2zEHXL{ z0{U4={NpxYq0ZW(hZ_EV8~j*x0oSM=Bel5KFYG8Z_WPVDX0*-akCVP#ZL*@{bzZDi zufq1rI(^o8$>EmotQ8BHZS@bo#cjH(zW*TX*>{^J8hnwb{Ne+w<`pSJu$XyCi|#Mz zC--AGL{Xb5c!D#O(=5&2BI>9_FNJpEvPcLl{)S}`caZPe>>L8|qod!t7$CA(B5ia4 zDE6k^4r{0QKVANOMfnLcE+o;FwYM>Oj@eR+;WM-3pby=MG_j<3ykoD+FtA$Fqz9D(z;itOEk>>O4R1gJNTbvEfc>|PE) zfWDD@OYJH({LKS3Tt+uy3B(gP9r+=V`{A5)bmD~!)#DZuGSO_$$%nJUF+h06Yb|_2WU2&wX1CLv#eUr6+!~nsF<)F&j4aq1E_#eQX8$11Au!&UAd_>- z@bWc#Q^>EQK)--eD9>Ah;^ftZM7b=Q_9Y{*dv^EBa%J~YmKydfQ~o6)V01hWxF(lH z#Lj&mGjxXKPOaa|#=jE0Wg&uoHrm;D;4@6QuKD_kZ>wSBejQ=rnM-BeMvp=#?43C> z_%G#swtxF01EkTPRzj6+8R1xZ!!A^eq%PWsTiAG?Eg|5Z3Dmr)0tFt3&#!|e@3ZUC ztZ=uA2Vo+#`S3=F*x9`TEWEM{2Ch=ZSlE}ZJ4v^syABd9NO~$J)5?voUv?t*Gj{JQ zkVYNZbURRWyyE!LWt%!&Xr<@?3g>>XE9BbIn4SvRB4r297rDrW*-=x(xcc(Mr$e#_x%ByTbXccC2Ye@jQHJe<0tt*Vnj@Pd zDW{mSUUTqlg!E-`@9E!~9l$B)ci&@=ND;Ebq(p7P%;EhYa!N&dTGMmF^a&0dLkuwn zY!8EP)Q_?JssIWFd%L#dRv^p4=I)*y*1PO#>xwFXRcL3rTdoeXWy#zV-(#G(cS4bo za-yMS+96P?qxS5Bd6BT#JzC|pLHo;R(RXIp#V#l#Na? z-)IQD@8Iu+`!{&LO7GhIGe-Yl$NQQW-=g(k!&JA%a;H{I!eiC&1^p1qs}6#S6rF{6 zZ}E8t( z4x}`0F*^E5Ir8@V3zIJwAM>t&qFzXwVcMQmhOgeDj?TQ&gP>cpGY1O=3l#U`#8RJd5+q)~m?>`5cZ z69$5zN1hcDQ3%YnPJw|e=zI*MqYq*E_(^z6ryLYIf<{B5AsT~@M&N6@5fQY!NG!_W z9~@*=DgX%j-v&TumOpwakQlsl;ld4TY-WfW58lMzh$4K5!F`c0O^BkkXHS<`6xcPx z?57-0QSFa%-EARN9;fZWSPuEaL2XelT7}T5&71jfH}b*yR9_Jx3IwLWV5ksAp9-M} zCU1zRO~u3BV?UHIcN+*# zk7xZ%>nVBPbMxbg&G2Zy#5@AWUyO&B?DyQ%O5xvE;b$G&6m4rOfD^$8AAex`lOfrz z!$m1TU>-D%hKJA*M6@LpB9kB*D1lLE5)MKuKsbyg1pwlABeFCil8In;mX9nV@M!u! zvH^dCWko(@e)-`=?bJW&XHaa(VUYCtc^b`56?QM2tntB{ zuZ@L?+W?crweYXRfaHvMEj@;Q?$m8wlIt$Ar!J54CeyD^)Y)_Qcx!+Z1p&=akO3B@ zqmM-ca?pevh)e~cZ$v|*h!{Y?_Rhbd&Y}avJ4+{Ug((r?251JFjwOOIzxvpg%_9-B z^2g|97d5}bzb^d#)#Nx3&|LMR-NX$WakA^@1w5QramLrveQZRz(G#r3PB^V_`I64Go3*{TkgChZD zEcc-41S)yMSO#1~ECE9S-U7$5rEhhGr?z;Wbbnzw$Te_P^@YXB#pF&=I|g@F>`Z=- za!Z))4g^l1$N%mNz-8P@UGi4HGf$SZ`@UT(DXDClA~CpW@%gDPfno%vs6s_3C^`Ye zI|(EsDOwH)dzd5@4Mh`)5L$)`jKnTwuA@sUj*sqou?Ol`FFo-wjEOqPiA7Tj1X8}7{WG44nN4ZvbAC6j@#?gZ-75|8=WMFk zhdZB(%$oZXa$fV1w>@%>1GjR)(P1=>gu|0?I7>^4J~@X%mBkP-1PY0s15yGPKy;0d zm4aBE6=HaF3YI{ng4hc;02x6aD-3QU+eisElzrWOyYwVUS{?+$jEQ@VH2y))98GQn>)pPvg7 zOYQX=d~n=HD%hYvLY^)hD0OQhI(e}Qfgu`&PQ=JKnj-#4a(v1lyrYd&46o!k0{?$+$(v`%LfzGEVR$NRItM-L1SuD ziB<@Vp|x5PAgG9xLrSJb>C!<0#}G(K5M%&hDF%=fSpGehN-zo!OijlMv-D#rm>0vv z5)I2vB!b(_1zY%fR3V2{I%hH0PtWx4n0y>!T6~AXaG@xt>R6F?T_)8AuCLEt80Kvn zPqOxsnI}^o5#^e|iU)+PEhdp$ z2{bSlkNd2PTnU${i=<^g?!aSeS73biEUnnsiX0(G{BAcM6=$S7;BQ~_((GAKf+9@HKRS_EW3c_O=itKjESvjF5iw$*cAsDSgM`7h z?_Q6*dowcM_@!_rFvlM}&-t?wv;uzj5h}^iC}KM^2t469 zHP6I2QTEwKioUyhSXU^HQ?yCdIih>^vY=%hlo;bu3AP{j*PjE!-?*E~LQ}HZ#w!{bI-u_|))-EoccsiL9vA{WU zygJ$AW7SlI(;K=j&gvL0r#!{{ql4E@eAE+O86H8y`7cLXU5_k-Yu*R`iC8a%7hUcEa$GvzTo1M3~A z460sJG9MZVc~gz-{tAZtFA@RRXeB&AxbtrVC<{zdQN#o*X;RH8eWhd5Xi8sr;g#cp zICpjpDb2nX~CCkh`IW4>KD*HKYfM6^F321FdP1zue% zt0|k!p9%RrVhh_kjv)>uW@*)Hh|g@8@ARK34>0KLKNJ6*p4tbbW;IhiQ(YF3so)5q zmN)yUfcCB5FFQtCGo7w!`=(kb&}8KBi~@`jM_;|*=dAp)_gj+MPoMDI${ds1vOZ8H z^&NpPWWcmtKV!LH-9Vk84F%Xlmy3%UYjJzU?WZQ5diR}LE(-WywYAv~GuowphL`Gx z8cl}`tCzi2%WmAAffXHwh8vU`0VTgTY__50|9Z7ko)td8fRI?yn%A@$XW7lTe5Xkh z{PkyCTVZp{DnV?31ztu{2ls@M5H-IrKQ`fX_*Sdt^5@!Pvz8I=C4vatz6DZ8abSsq zAKiR4W?c=&vyz6rJF}|Z%}tG5iCGzo1|S_X)`vn&9R+6^>Mw!6t1bJ4y!qpGH1( ziT5b9tC_`jbW`=RuYA_Nsz#0;lxdyO4!0-_U4RXE(f1J;U<^@TVw*c7;?Cv8;L8-@ zVE^i2x><=5oBgXy@Z432FCgWN{Q-rzM2(lD@1+v6>f7+s&Ik78L1n(aBivPuWrYoe zo(db11I{b}(mo)w;r}rtkh~zD7!Th}-d?ijE^=zCOyiGgH?Nd!mionc40@rl!sfW? z8-I47R)l=t2R&K0v@GFcp6wZAdvyfd{fBqYwN1BtzI-owU=ANe2Nn{mHEz`@K6OWh zD+dH`wz%_wCHUIRtm5MtBt59+K+ zdtbm2ESI;x=*kO4BXN*--d^$c!W{Ni)Vq*4>EZhAoNjG&Yu}uURiGEUYdxp7Mpm${J=G${xWx*B2W4e}jKmVFinx$^Q={Knic9#$5ih z8W=i5>YB*xQmIsHaGT@99UMlxe1v1>*iF69FrU<2MPSr*=v0Po`|Ww^S8j&2?+LUr z$!@zj_FF8sOrXf%;3+w94!_p8hMa%A{Cl9xEW~Wqch+R4;p@nQS+BkmVXwc>4PW>5 z1hh?_20O?<4F=f$lN0~J2qY4;)EBgjFW@5--x1fGlRe@~t%y0ruir^`u&(HQskv#Q z3o%6C)weV*wRGDLv1*)6pP%|}Z!Z5Vrl=gF?pnQ-&nFX6x(S?gU=4AodGz4X()?ZC zW7Ti6FTToOTBr%?S$Z2{ThmwwBMu;TzW=ww{+p!@Jry>Tv;^4)DA|47Kif7lo?#=B zH>YL%cE^s4AjN}8R=OsFoPv0%IZjf36wW3^zqUL~#>|^{sBY`w@UHs$Szwd)C+v{w zQIwiZk#vsFJ)>2-rb3x$x9ONi2Aa1E;qdrzz=`iNfEwo0p#0Xup)?MUi$2x}p zE_iit^)QYRloJ5_FLHfd#EtDlXy?mPSRExb#Tk?mIA}PM`$F!6h%9y+*%~M?GCMRp z_xfGcAGphG_)(3=ySbXR+lzO7?rLagl+CCt-UYM$-6<0Phxu7_{R_}Tg|)=Yg)LzP zzFHo}1Fw?DUe;y2h&7?hB(n)>?ud+rJ2zdT_?tb?9?^ZLSuq>_P%~dknJpoY8$65b zbLEIR*?Ni3aI$33_JCEIE4^OYGpd8#-}s(zNno=EOWhG9Y-r;4_>Hwv+lRitJZ3BH zeYLfJYcI|W9eZ3AXX@37!1e!;k+p#o{il@i_5lqB7b_5J_PjPX240@%a@%dbXDMl{ z5l(@0llVw^Mg0a=7A-jT#j)pPq4pV0&f$(9hc9Y3|8UM=b35SPuFln*@!%TG*n`ql zrMqyWK(tQ3YhRvaVZO-&YpYwiAmhCN5N{wMRUI#f9BO+4hL>w5imzOHIO2hLnVEi2 l3NiKB`~z?~`#pvMa>n1bQEyq~vdRThSFerc30C{|zW@@aAbJ1* delta 1678 zcmV;9266e8i2;T{kTd}Vku^a?C|C8sdc#l@~pb|E4D#Xq4(xUa(=yxR76LUm? z)gq)OwOxJ(823vg3mOh3hRKo!D2E=zHGH}-IuUfn_LDpT8WO5NO|&xF<6V+OnIa7*P2gl zz>^^aE`N)25d&iQNX@C8p56?<&}=KiAn5`<^gGSJkib#`*`4oZ5$7lzmVg&u5&1Uk zSbT!?3R=fUPQS*0niQdj3LLbYF7Ny2F<#LJKnuaRCZqOzJRW-=q&6k-<66qUwhj@6 z0NxPPoWM{{7>(W3Gk7-I;$sy@4F6=z*-})H7n9-zA{riZr);-py`_sbV?VP~;FddR z^wB>u&+bYWlVb)Xla>ZI47sG#QnLD{Ga*!u0F!eBBu3h6l`iI0u|4&0tMl_;&Cf=w zW!pG8_~F$yW!)f*!U7&k-Tw8hYj`g9u__IEPk|DUnvCCS9(b!+5TYlC7n-feqq@Ic zle`5PUNHs!^lv>*c4L<~dYn5oDRG%K*@plejsFweWAUgPMwfS!j#~X-~=a= z9|ay`+GU7f0}Zf1k-^>N-^R|tDz}&nmi-li#4qp6pqtJ3W*GF`7KxQkqq>9{@F=}B z_tcK#6%(}r!8{t%Amy}KAA5?u%@~M2A;lSfjUGB#b<6CGo?kk9ET4BV9V9Y_O zHNAhf6bdd**&GZZNjyLrq5p7~OOqiIf2miEEbiIE^cD# zS7;R;AP!pjkd-9G3<0-+UK8ee0NnEj4b!rbcryC_d4A~r>%5PDkyy(8YP=$3&yq6V zr^8OwYb$N%@r5E7HO!rGeOG+*UoAoiRP0z-ayk*d6Yp}6W4LML{DO&&o{fuaw_ z0p`M<_HXpo^|Sjz_M6&Qw3X3UMz(#Tdq-~d&^@NNy`}b&-1ddxcIN0F&;Z&yNMeIp z7MP@{DX~Fz@8lel=MWPaKU6be9vP)Prye>&z1;^?QG^k+AJ2mZvXeX!A(OfhCX)dY zC6kU4B$L$=B9k2xBN=6KKV-BZQF7$y&4Tx2QV)$2*UTETlIV();1eQ~NE9QJkrW-1 z(G(z)S`{W9;XSVr-@6y{6{D9hYrg4|q!u9|k+YB$GuLA(Mz0Ba_(}C6f^uB$GrLB$IR*DU-1oCX?tHCX*li_-oOZMqL>AVd95Qx_{ID zp7}qgeLLyzPP%#1&nR^B$^AR&lYtyllj9sdlU)%WvmqTb0R?=$ket?&R34wQADscS YtRM*ilf@Pyv$Y|K1hY{lQUL=0+449B1poj5 diff --git a/crates/j2k-native/fixtures/multi_segment/bypass_rgb_53_blocks_per_component.j2k b/crates/j2k-native/fixtures/multi_segment/bypass_rgb_53_blocks_per_component.j2k new file mode 100644 index 0000000000000000000000000000000000000000..0775b67a430f57db17fdc80565d88f7ef17242d7 GIT binary patch literal 30477 zcmeFYzxE&gPXhnf{=)zO7!(M||0zHf06YK`00sa72L}NGssdmD z01&YMpdJ89NJvUb3780g{cny(8vq2DeB(#@vM_+ksY7>_WOCGm;=mgmX|&Lw6BQ~z z)6j-US2iS))9csD+D7cPRv;(lem|Z844bj?lGVo)MiD9s6w^BrBrW-c9q3^9ea-(I{+uW?1qxCKNx#;*j|+r4&tO zi`8B>#OSYsjmmm}5Q8vntM4}06?IwDB{X*D$k9;FR&8$DK)OR! zAxm9WwH%3~Ps)4y(}h^p;CB^>9igW~hTw;sO$ZZN-K{&^8Iq^jgIXP3cDK%i3=?5p z&IDG_!o)9_7=tuDcZ{XuH$OMXm@o z1n=K>A(gUe*#^_`L3U`NQRTQ66^cga2C;F)ncGS;zkta z=wSaOzFpRaoYVeB7AmBa+fMV_33KKSK5#~@<-O-ol|04GVC#ncG4|$W_!#|N5dvwE ze6QJzE@P#HKqq6{zG#aOy#otkY;q(cDt7G9r|86|s!8&6aJe_;r1itY(dNa9pZ0!& z$mlwXz~}gVpthpgBY~@IIBb)B-(wUG|%CZcSx}YD!p(v0~7YsLc?I_QJ9?bx$gnFKD=?jnsVgKJ1Xyk z{yvneZQ^8)m3TRk>MI)p3HF`ftSU^2yFgThu1Oi0Oo$Fsw7y(|?t5rduka8&oK;>@ zW(^QN^zWD36%P8&FWJDf2dsGaBXgbRl)|nf1F5@E%P3Czz%=cc5IiHIH7N)eF!oH1 zUdx={MpWgm9gz^?zjpZrZC0Gp=c7x;@?w;1uk!Y8npl$vn7}j3N1?Sc+cqhZc~O@Y z+;OD1F68)&cF1`Y$twaSmn?g=kOPVll!ZLf{yKa!bWpMFKN5~4@8FH0#x!lVPpFgA z^$ZhzI`I=j&mara8_NW3cd%@Wc`PkxUz#V3?Oe?^a)U zX8G}^;kroitul%jZ`+t6#cu)3jxgS?5=CEgQ(7sF6ywMfAA1jf(tFow(t)#p2#ydX z#8jH{hO{(eHq(L+)A!*f;{C8%D4-wRf4B%CDxjs}=-9l-q(XnRRe7oQ$k3!vP9*5S z9}VsmR{`=%jwjm!?lGwjq$XrU>3n<5$ievA1mHTH9g;M$kiB$0rx$K*e`+t7%c36}e&Wg<*; zR6as7ES9&QnQ&4>uz~%OE*^4g(i9Y6b4fbU!x)flQq@=|RgYJiJWOW3R@$sG9AGRj zxdlNcM2A2$s0#z+A7+GhxYuSdb^`Q2ysj7J9M z&&`4h8%Cd3yXQEfh?if^9eyh9`J0; zk!6TAtp(k`GKHiW;`n_;b+A{2RDT&+?}j21vJ&mHxSatKYQvHm_-vuv9LA-;<~;hC zvZLE4eS$!Y7{GFsV)ZdZb^Aw3afdY?cEc!;7XA~}n*&_>R+?)ZC2%}q%`h`A2M7vJ zy6^`O$Wt$kFnDPbg124vXBmud8X1<}A?SnA8%t@j%`#E%?(+;ePm(|G>TCq7BWy#n z$|XQ5<|Q zl>%rNn+k0}!W^aL^fn2Chw=`P3pDo6lq10eDvK-XaZZ`lHp^#4Roy*}IQ1!Q89JS_ z&v%-$_NSbt?N3g76i`opk18JT@S9Q%WDxt>V^Q-B|E4ce*H17zhfsy2Nm!d1*8hym zmN4)J?3;amKa?M5R>2W1`Q+-eYfO7l==>xG6M3;Zl^>hAC7g zs%tf-OP5D`F=-kqq1!dSWxJi0F;XGz%SDMSNZCNSlOcP_S;vb#C2^85e`QlQ zozfrw#9@|n9JI5cbklp z2}7*7dP1s3#Pl)WH(yABdfDwv+pFT|n|2pIHOzKftWLDge?mG25YTae(ShIa?6FDG zNY!dcRFRvYJOuKjcHVUdTxV2Ji|{DR*L6{mTCa_nyN0o|pNn%(dO<~MD^_ocSSQ!C zAU$#{KoRJUln#57PI8>qzdCnIPJD3@nMeBcy-e&-Ro)!UL3^=(ZP1-tw-};mJ*xdq zjQk36fK6El^CPN=($iFk_GU^JNx45fA_$Dj5ZW2g4kG+kNpu8zb-jS+3v?vGN@>i8 z51M*QP}mW}HANm)AjjIljKeXMLa&urS+_}c?OLR%sBWb*8`tI%gCKzqL7!MQTKD;!8d2pT&>-bddp*?!L12d-ePaF)r;?MVXn(Zp;fCeWqFv zdw3q+c$A+FkSl>nNrjqK)+VkD1{cHk_byOZ$`0M)9_RIo;#8t>k|d7WsZ1efsAPWX7TG`}5ZPJRo5vFi;)ec9Fndl=~XB2}yxd4mLcA z1WDW$oZ95YVw6a{LxbNO|DYNE_kMWsw9;J+*smwbG*6z>8TnrqnUTv3q%my-yVX~c+>?d) z%|Z2eOfoH3-DUMIQRt~TN@d^DE?67=!#TUH>AD3hS?MdPu~fF*-%ECcJ|KFP>WX`9 z7RdIaG0DPOVrh+fVo1c7&efUkxm@b+X7WU)N|M8PO>A9O9Wn@)5sYz$Vy9$6uUi;V znn?YJu&M|r* zeAgo~b!l)f4XSIdGzpve@ctg^lCEtL-odGZYBtFiO{}&8JM8%|ME(3c(6FmI=K{pU zyACuVLmjjWKnh$bI5`ALVI}LS_#evUo@qcTTjSccvAr%@hiaGfBqIFCTKTnV${=gY{~8#f5#~1)k=iQnh#ISAJ^`@|{(VxO6vz zv6X5xxl`@f$rYc|&aCRQ&K7+9Ms5l+lkwjnIEzDtOs^tfKwfX}PD#0NFjHV}{af`q zs~cUT*6nBIFpbq708-2SPUCk5KL^w2TQIO<<|V!8iVdXn3#8(3rbwU8jEfIk#$w zC|)qK{X`$?X?aDJo=i0*Gf8LohYu79cLwg*hJoNEz%E~VVPqdYuG61hdm@ZnIlCU} z*Or$5*!rEGL5xL1j7{x8n@gVhjd2jCggYQ3Yva)_Oy3H16IXlRk)a?- z^7BmwYdBI+qfO_w)DbDZ8&#a7`JHWq*>dK7=oKMl?me=wV_Y@2v=g+wY^tx)+=PTV z|Bny(8vaieY;u#G+_-{=%T94|QHWmm&lOU;qDoCp&7w15A#flh-Mp<09S1m^>OMH8 ztbJJ;7hL+5*QO`D{hA3KRsT(O=)@flq@5=IkhDZo2Tk@s?GX`@l*&kk;TW=Nt4(gt z=YlxKCgbqYR>=Scy)!XLis;hd^8`4}$uGv;xy*9i)bvE6_uWL)z|xn*)W}5`ff?+>@6S-n<~xrpU_&ym>(KF`whLs-J)$HL<@;0#tjk@OpD9 za!ZiH*PGK?*phd^mgD}QYm+CCXe6wJw_= zq0lhN%pJ7lQhp30P5r2d#oE6$9xnwt39e}WL{#E7ufZ|B{@0+iE04~cE_Z_P15!hJ zZB23Y1y|LjoeORxszl!Aw-WEL-SOX|iW;aeSqu>88-%l*z127a2j8MCPsHYUk^QdR zj#TB^PZoRJz`{K&`v7-5y4@vhi`~=)`=h-y;eu+vEIUH^M|d2H#(1Wto@Pi--S+s- zlhYr1CbIVWaSLDN%n6=V^+X({GU@1KL`RtV#}NM~ z`u3d`w<92&s3c!#54>~~w2bTH8T_X~N|386eM@26>w6?dw`DXOZrtxE6`ss>>lTtQt1xz_{<`b**=pCMMM&<*IS0eLlkad#Bh@(Sms_lz zN1vT7<4KX^t23PWE$*vE6i%s_ap}^hOe^Pgl2I;K^sN3Fi|kseZDdsWUm`guwiJd$ zZqoLhBU!FT#dwr4GtHj!2J~v03Ui&Q=NdcYw~l=p^m-b*k1vvTbGB9dP>PrNb`9TR zgn)H%b{6O@48G#wjpzZleU+9$A`;W?;j&T^Q?G4T7{0l{`r2P$DJ*!L(v6nj95+#h zXOZ!h<9~0!LlV;{PaA4|0(u`$egEblg7dC)m3dnD#jX`(7ta{Vx8@=+a^mE>bZdcJ zCyA5S{VD25MVDYMKqasK=?H7n-8NiTrv(6|wQt zzbdn8Ihz#%G4F8!>^7(|!-`@1H{52cFv z8AWoj*4nXw{;tD)L2vov6lR|8?~Do()(;ZDjTtnx(>8dp=qDVounQC*h-0_bu$*WS zlo<4!-tuqZ8J8!GNnzq4h6p8i`nv@6{YRO63X+^LZsOE4A#$G4P-eOvMEM{EF9ejs zdlW6qTQ7pDzlo8GX65SFduX*?pfT{?)$;&23#F%w42MgjDJ9HqW-ev3v&9>qAT)nh zq`ao9FLY%kTNyP?E~3oRpN%X#p@XVAm`x_fbU4PKWZ1$|aA@Rw14@=EZF{H-3)zAY zu^B9&ku>+nET4-|zC(X7NyDtDUV2e-vr5`~E6ou2*jpKmuhQ>leqz%HHwW1P#A`Jp z*~V9Z0lqDL=ak!TCb_b`WG_@txgq!Dn*TlktaW;y9LG#{66Jo2>9*H?CT5qbHeB|5 z3|(S~sb|vm@T)kVxusHjCj-%kZrP&}RjE0@LOx1WIujA_Uqp=KwIUFmTd9WVg6A!_ zqQ9m?yIf_Ch9?#iir@CPa#p_3>1CnCUU!1PE3MT!@t;tTlB2UPGGLzz9o2|l1&B3s z2puar+9mfs&V+TNq}XkVNieyF`&8>vY+AZ_TaF8$+A>VI$Th2gG-F!}05Im6zH-f|H-`6+-)%sM~=*a_S>WffSVPN3j3*!$mtJ)c#6 ztoN#b%l6<8ySp!vFgDCkFNQ#x1X~kHhxsc1LO)9MS3-Zt@knk1c;`g{+=4L4go8L% zY2B9y`c6f5rM)1RPNC+c92*_8aONZ`)dMTf7UKsgg7U1e%oHO3$1TNuH$PepWBYt) zwu#WwT{3-jsN1WA&7B}b3p#^0V#3Qr$s8bq9Tfxu>tGe8eLEm*528@%^VXBqLQmXR_xZWS zhYWSaPx2L-3+d71U3m^#G{>73|H~O1xJvAs>zrCmX>>|2{a0ERWNA`@Ju;IQ78zk> z)Gj<^Tn}IKyye+x62XbyiUf%JxRKmizBN`a%`yd@)1uc7OvC(#*`>d?#b>0|B?R~c zQp*cKrwzNmU3c-lPPYP}As7c{3A&c$io_msGUuB7f^>64nHUl&R--`44sj zcH(D=Ud^-@ONPu2yfqNVpIGpmnGH7O+uY_yo^0)tR|3f8K|y3+HXZFD`|qT@`3Ni& zJulnA=Hn@8947Z?G~qrXN7Hq!SM9vdjWseA(?JX3{R;;>TpqU~!42DlRa(KJ1`I_P zaHK1n#=8hL^-r-^JjA0ySWq(TqxNlMia43$?V)54&EON2O*RH?O?cJM#vlPMvS{t! zWly((cf7t~FVF5c)|! z*zQZX}lq~S^{y~p8~8_SG|d%jfFo;Ix9A}S0*&&|G{oto#1xEiP)KB^7R0N2aF}oiT(+6pXhWs(87ycN>+t zxvaXQ39T=Kwe{g?q9;R&k8geeVF{Xd*T&@BCNgmome4fXh56#`m2zgPSW>{#9OX3A zx(xF`z*z0d6y3LCMw#cT6H1U_+wv%F<03{bsd#O}24Js4Ca)>+DY1TDglMw%A`7LV zuxm1_)-f^@?=Rk1AaOWBj|YM=Kjnf~A%~R_uf(*eb#9~-xz)PbB5Bx;{(;KyvOqk4 z6onQs>I{(72yNkneGP?5=26^Rx1%wO)pHE)&!96;oGyLIM5+1ef=`yeT(cdNgFvMA z*FaM$64b=5EnCRD*I1e4D2ihgf&Mh>>u0-hHf<4I0h2WoV~YYiX-KpqU2(_hNl@|3 zWYx_9$3{TT#^oNF)kxGKB$%hYB;`C0lSE^YFt4rv3X1XumkHWdyUn7Hc?&cSiu`VZ zlyE!t_|lxn9wLmb2uua(Gi_yORaVv@BTeJCftI7Z)Pjbq6yw*r9s8R%gQH1Npb-NW z2=dpFq~X|78zRzF=L{9*#?6f`?`=O4XQ~%*!H#x!e^9QoGbJ?xP1&@@{x!sxgA=DxZG5v__o8$Q;J z>{LpHHaJF|mdq!rC0<|@5!rQCRWXyA`m;g97LE>DI$pG}x_X`mYLKS9j$w+7hMuT9 z3|k-O;djr_k?Y668Ywb;xty@xbsQ+(mAzCiZ^DHlQK`xE3-QTvoYo;~+T$gf+!GA3 z(KuP!BVBR)-O&sdX(>6F86-y?5Y6m}Ijv+Ht#KhkNQ_WX3FzHnq)q{QC|F*;H$J11a2kJ`B>DjLN*2T zh-&I^Wy1n&8pqvg(AenCD(v<>)LD$iTjnx5FFTMD&2<%uJ%#)w=|a?)GP5B1gqogW z$W8E1FbzU8?*>c#XHrB;0%KVE->{XM%ojWy8FgLmaj4_fM8`J&KfeR*?Ez@M(l!nQ`jDle^Tq8kDLF4G4;Cx)fC$ zp>)dWBFsbl3qmBMI$e@wbYO5AdOXJK^7CShf9E4A=cF4CBiAdOqmFr9^r&D&NV=TR zS-YRPh|pyIy_tVTtvIk5Z)oRC0-*KljArMtUQj5MiW_02F5zRTZ}*=Le^TfPr4IMR ztCN(cdJ|#gl;FO`709PkG12|RhVc=YKE=DwrQ&HjRp^DjG3Zf_j)$Yk2Gq`~rKUgB z+1XSGi?D}b^y@c^On$&4_S!!~pCK%54<{4fp#KnD6^-2ll_wDdC4m=9rU4^xUmI}t z0s}DpIJTG+ItB*VM#Ws1?QN_ldcclvc zn{|+Q^CT-uRqTZP>*|NBQ)~KoxE%LK&vuMF3a@#X$Z5X*=lfTlU-ZM(>d!~kxzYl> zWQt|%%q0&&!v^?Y=*3jzZ&rUe>A^@ykeHQvI9)a&ou*I{=N!k%1YTfx2-!_+uyuYV z`eBcw_z*I3c+;KhL3Z(yAY|nkeFb4e~D@?v_Ha6}4S1+$Wk!x`A6mCn0{$DHVhj z!Ijq@^cDoDFf+&*15rDBCZoZRnjl#fCEZmy7!B$NJBsHqeQAFAWJ7=6o}0^aY*ehj z_1Qnh3(?4GitJYpl6(bYsO3H6dkXBVK8LeU*carv9+>(sUUjBE|W{qxEIQsQW6aA{!fzQN2Y<_a_X!Wg{}&bFZLZgBD!|pGeg2vKlBCPH`#&uL2*L1+kG;A1%h)@ z8mtr>_le>8iOnW>WG0bVmG#FX7zR4b@E!9uQ_-fzMutkl7tlc^q!m||h;|63eEPr` zf&(rgz)+l%opZJS9d7BjAja}u7szr9P$;%VxCrtwxAtnWcTa@RlGP0!KlMraiUbPF zeN^29=O$YskB;d`X0Q*cPy*kR1Suu>(p!66dMwE_UvF6yU34i|OG3$bKFcofS{%I) zP!%QAp_@Nc8rQ3*zbEDfXp-Vz-yUUPs-3jpd)+21xs-$s3-3#`9ez9^W1{oHEK1w) zXcrigGqjrR8L}wHmbrBH{ct&on&{9yP{oSbw%LCi9O+|MA}XBA1qJ$zz!f(xIm-A; zsrTp{DnxEWUx==n*9I)EKX@3*;0*UlyVLTlGY3DgKU`?J{#ir({7AcNUdqHD1Gp9I z&g@{Qw%MuMjr6bQXxq|sF+hrSc2Sy~tSAwU*9hqm!0}Mh#)3U69}m9jp(6`7;0W%9 zwJ%(#S>%|N2_e9@>?A)_=;~~Apbr=epvDDmU5$0AO(MZ$J=Hs%rSSi{hCibgh%#H9 zsiy-K=?=kyjNbX>4Kr$h*toJ(RcVYpyM@o0d+++oQ;W+Dv9`y;M}Fs(nI=zv@yRC1 zm&_&C|AI{tul74O-<53G<6Y(jEbg=k*;NHBKcqpI5<`N@F|PNmjxwX&1v^|V88IZh zcjmk6u1TuD?|0rx65sEXH0Li~A<|=8Y+K##y-M@y=6E7dEtJ3&pMq=a^j@O#A1wJ9 zJ{r%SQ&W6i4sVhXa^q>rqBKa^R0o+=-h5YN>x66Bm}N8$Mzq%j)0;M?&W}gwJ^3JI z9*@yzhMl}PegdJA`D65U7qy5O)~QH)#`49ylr+qHpW- z=!PXo)rGM$0@=#sLP0(jhY;X>AFD~MIaJ`Ob_+3S&#q-BWgyhhk^&=zTGR7Akl$&H zHm0cn+-E*-8q;@H?7HJQcfQAxKcGGJV8p9z83hqu<@n zZ@r?1_>g)+j1h-JCf>-Jcv=(obTi1HpVsY8T^xzT-VlK5k?F~*4$&} z!hRe=_hIpqd7`>zEAf}O%|0R1#Or4Npt*po!Q7Bdvp?g3VHq|Rq=aVO{elOs8F_ouNwZdKqFXNW@*lFfy^%Act-WV?>w;>7uoQqfUqci#6Tq^L9pnwqd$jlokM8MLOU(RA;%2%C^oi@2}{sTiEt)LsS!l_*jH zdS%Q^B+ou=dGQoigI*pEb0_7@t;8X?BKtq9fy0i!;N2WCELavE-;@M~ts|vs;D4a9 zVI>|fRaFNQGvg?p{AsSw#vap%)q=D5mMB8$#ke$2wqH&6om3N}@xm|?%HAl^8rHr# ze8B&A=qNd0?+-?H<*=7bzH!h}jTvM-5Kg=@eU=DhuDNJeCN@ugV`w%ey-Hf*DGc~X zT4(TeXn3(<$)ZoMT5I>tc&4#8U;%8)RV~`!0(hk&3=6@(&?(IE6}oPA4z*9%RN#H( z?u3Y=p&)}eXw26%{WV}>y##A&+ckw}92PyD2c|A^X3p~zNHA;8(1y&Pj`uJEx{>sB zKv7HCJu;+m3Yf1RwXX2=f@6Rf4OhiCrWBRaoX63KRyddp$+~umK$%gMYJ?sIQtA^y@tZL}lk; zp=3xJDGv_2y{zhr$^6X^c9C8NaDx-$mhb2|oF+m`V=TT)Lc|&9E4yntBH_CmO84)$ z0k`VcC{mVkWYe!+Y>D9EZnr9Knt!cbeTZiHl>W0TlRD zD8>=X)fKx@A^!PrjvhM~f9lx0l$c&F=y=8qMFPbn2m1@pkOhs(2vae8OaKf0 z^w)UYw@u*pYj3Fgigwy-vXR^s>nis-VS|wmeJ{d}Q*xY)cfo0~x1+HgHU#W`m!Khp z#U0{(>vjSGlXA22MOleGn zB#Dtdm`U+$?<-_ATj(79JE{9OtlyipS=t^0`5dD?f6h{L?SDI=bzq-mu5a%%u=-P1 zJWs(W_e=?y;nTVfwVfJz`>Lx|hoAORw)MU|G>qbq&qj4INhU%CDWo0LlkF1J*0pws zB40Ba3;yJeOhe*0%s<*s=H~eM8+NjYOL9Clp!GparJB2#r7_-B7dG{9fP<-z;zEOL z3ypGUe1rWJ9-U{fQ?Y}ebRfmbMmz%?cQa)i9Crt~g>$WX)i^eBUdiUG?(1%QY_#*? zFAn|u@0-creugy}HjG{cPkBF-EX3MlZc9q1g6A&4TEI^y#6dccUhoV5{#5F8|CA9+ zXf#?iza?x>g?vjJP!>nxVj#5(Y6H@TO^?c3pWsUn3BiWN#we5cVyA$CSEwJUl_FNH52iJ}JMk-$?bsd=jNG z85;t!%To}I_cUmq|C)NXc5V;g{d4`m+E!%O)qF|7Ency;+7=8^B}Y6GtGvdO%V;Q( z=z>Nj_Ev1C6tC2TT_+!n?=4!Ep;}Zi7Y!uZ=(q=0L>$+wHYZQL0;_ZDJytmeV8tUZ zauG;i{k7=QE4_4Az4D6lM0E@ybD;|dDX=U&iGj?n*IVHcE1CK&J9IP24?)(9cw|-b zUg=SlA7V6O@p6$t@$Zt~PlVjU8o!B;cUVQoa|Y z;Y&Gr*0@VQ%|?u#f2A4Twa3(XlQuBp{5VmhO|1uqh=>b< z%{GkADtp`|e-@F*7Ow5_{@*+D35oNg3~ZlyX)bg`(H{Wz2?O*3Al*{FP6RE~jcS$r zSa4gXRc(79pljD;AFaT>+szE#>w#0VHDcflV?ctomvjv?64#=~LTDLNK7?T=#;v#G zxU>+)YBTq{Rp|i1U44pQk~6$Z=DvbMkDe>Zkefv?)?*b zah!a{ZK6DWQIE3w9b4zC(OygUTSgg5RQKzCA+ae*Dq?aMQ7ne~jteF~&_z83m(i)e ze7qT7=FpTgqObjo*LuBOn1ggF4e2|&m(Hn_ebUB(#s)?>zt@#>HJiVBGbpG&QK~@2 zKH&fkU+?G<{i5Q^V>5BHMf>wzWcT&a`DxpSg)CsbA0%7!oo*Fy)r*Z7Z?+VuM z5dP#cnsFaV^2v2fm!=<+QwKSuUamHcwU7_2jl4g@$8_yqUwgU{l%>Nz)qBF`?F|t( zc}k0B7sEtTPYS6->yj8!Q7{c0nq3G2-wIV06<6>03ee77a@g^qXCF7F89mjCxj`g(!W+? zVLs>Y!uH4GFF!hJ+Ri@pZ;E@aeJFGnGNYD`U)3AE<2(^4OwmJ~s1cs#{B34Nx#34v z_#L@(hMV%<#~#w7MCvt{A+~)i?m9NM52$Wgp6le!gEwuexkZ>FFkYV-bZialRTy!6 z_)SvQzJG}ohp^XHQe#eNF&DlP(HjlL0@5s~Hue>==cTGvY>1&_=w$M3wg7p>ClI4V zIeejF0Uc4r_BDEk_J}hi+fglegyZEgkqo%5Nmsn+d#t=q>FhjS0r9eEKk-^*iCHBw z%&Wou$?`fnu&Uch=GGbzUb$?1HjV6UI^!x0heX_H^#mGQwLr9fy)aA^Zl$8uxr2F^ zo9icb{CML~XD)({ZZ;El5ajj0dqS}s8LJ4j=}BK~BJN{MBX0@A1W&472#Lc)>N9{! zTI#Gx>PvEr9ZsTKI6f3@!R&Bg!Ds3{me2c^qf@j0q7xKJtdl zyi07jl;uEhMR@ot79Ey)77{6?b5oIHb#mx0?MJ@Ux@6d}2h=ETq73xp^hJS^A`i)K zJCIDfoN>M;0=~PBVWjH~Kfz;S+CRN9A?{BXiBrtN5UJBeNHG2i!;(``f|L+W7O7V& zh#0_s%4t=`xop6!;|JlB-IyEE@=&;N%DrAhdA>`^t*;cXPY(r~F@=0Q>N-|)q!iWR zLBd8UY6o>(VF?eIJV>EU&2RVKpttGY*KPrjqp%0{J93i|m-!=A5)kRg6~Q-$(i*w~ zVA|@RRHu3Ac0vy>NK6uiW0RzRWi!@6zE-cEIOl)7WAt}~GW3NHPPY4Gq$bSWv5Q*2 zW;2eT^w0iPt7kM9_~X0+?q}ec{Uzm_7>uEzhwL20BR3J1UEn< z?iAB`jzDBXn@R6Lb1Q)jr0Tjrw<2w-BWR@a`rCf2sP`$)vezX}#HKd4P4d(;69Af2 z7#b1_<<%_)27LwoIbz}rVvLIw*6<4Uz$rSUGegQr40&-!9(FVI3H1M;5GV-rdHE0Y zzhwWb{*_%|z~|gQ*Vs4EpYQp9!=$eu(DCI1=(G9XGw1(|2mW&beg1?0%>eR$1MRqUx1$B|6Kn^Z~mvTy!kgepl$bKE}&-j#Pi#QHJf0FUtGxvbdT5^2$gbxdJh(_ z>7<;%ZQLpZ#TiDERzPRt;e7P7l8$BjcB$qIo=Iv7%YD9w3m?Emuy~=)-nw9D> zs?P-st%8R#N<49k7w8xsacG8VHf_O6@H*wUAQ>Oz^uZ0o?JR0mv(HO<8#+23sKSX@;--kme;VgB6eS%34 z)s+Gvou)Qxp#G+9N=Sr?f536y;zI@6^I%`P8PY`Tf4-q8rJ09Ac;#IyK*K>?^>)HC z{~;Gjhyzc8RrD9r(ekj7MHm6wN6Zf7MI_HF-cD8en;8HVU4^zf#Hr(6llnA3It#^3`bL3mIM3gD)rRo}@7J;^DgkEtwGgISyBHzUGYS7(d<**+U#=!y zP+DX*XO{6|)A=hn-g`O2(%#uOw|sG}WLChsnSBFIHJc}&o>YTkk$Gm4>$K#9;#j7u zIS70L9|fN(Ka~yaK>>zLOa*e`0%*ZgnqoW_iYF!=H1z^*Q@c9k=vmV8o@3U5aZc5h zgbQmz13-Eaz0eEd^>)%~$L2ZRglh)5Jui=>VZHtJAj#{-*Tt_hK3V6kMaV|IuceK# z4CMHLZ}-M2xFGcnsN0$2NWhQspXQqkPBCFj?ZDU_+2_gGPt}J-2ZQ>@sekGN^#2d| zwSW9a>HNIiQ((YL$UpZ#ayI`@Pyc`H0~+=R02TiOuKz3m8z}f6q5p&bR1rw{U)u%@ zxY+;({6k+r&$fV@uYRDW} zlJ`utDBy8-R!GWatSVD;qDB3y8j4}J2HL727qJ)X1FC+kcy0gt>uNi%#bBus-r6QN zyz0?*+zy$nIni}H_qfnd?k)|2h@+J}kI;n`)D-MPCC`5OmpK;PjqXD^DqMF#Psa(rzWY&8nrgMIv7z{yqfy#q zm6@;n*YnZqptQpmr0Dh3%ZQ*~8-D|14C@AxMGt%O?UmlWn{Q&V25Wu`q3E_lqEtlo z4rfhCz>0LpO&b0u_fy!KwS(jAFIOjX=S{!6wT%W@k_5PR3?Xsm#(+|AuiLrslN`ja zgvP5H+gh6=?#98E!!ZGEd8(;h_cy#GDV>;~>|{mlcuTX^n5jjB+`3ng64Uh*Bj)Km z(Z%T|J?(qagD{s*L2nYQB8Uw_N6&Eg25A8aPwr`?s{Jm={+ABi#^7CL$JD+Y4!iKt zydlBqe{7hVFB>%uLl^x!X%o~kMHVC0H~dX{I(J}u2Z)q7bccG*N5e;Zy+BrIaVH%8 zPZ>eZbLf!ABRf+pd@>JnZ3D4=lCB>6lR<-2Pr#W0w=Am8dY4aOCU^l2!%ed!#9L!< zxJ3xoT>@)@z+gb^?D)g$b=1NAQIKm7?8+dJ$d*>57jsM4xNa4R9siy97Mt&Nl6#Bm zxTsxa2CMoZu>^e&T%{K(?!kv3y4Rhd1oOFuQEr+YH*cKlZja^^u&7eG8+?4IVRIzc zp_iJ5WJfvz-t?}c)SwAD*Mi#Pswk`i31Oy(yKqr;d#1jq!+X#7NzwNEGq7ycy~s8{ zU6`*hA10RHA`^WoWIBD9omd(!?jeR<@9|$|6QvO$f9%t7-!9_ONa!V4+_{qlPFsjx z8mGND3$rO3M;!GgVlBkZ3Pv4Xh+wdW)Mx41{f}u{Q-a@8Jq;FyOtEGPuSkJ?6HrH{ zKR6&NS!ZJrR|CkV0uuuMERm!0MZ<^KU`q6$52Q~)b$KX*`52B6+vrqsX$uSldg#cS z@7=Y+UHY??%k!Lcf|`1g$Pds>W5b>RT9G6c)C3n#eBh_INls zZ+C^P5saCU3m3h*Ry-Dpgt_48H#OZ{z(z-o3Ju{WaE4w>SD}{2aqfLPyl{Y@XmY7R z0^!8<0pIcKFtldFg)DJs2*)y!1y7?v09&4(tbUGt=u5gjzk1U4U?-7v45nX&{&aJ%+eI7tt^0~)nWUGO*IhYi*rn0`3tJHC7O%=OY$=o~=s$!JM z;@}QCMTY~nY4gYIbJt%41VU7~j6u{Fk8qHBgZrtIxV@kLfDF=Oyy71}>mX(+Ohc94 zE+FBTpDNKtX{=CKjPbUNXGE6cL(r-oA5-89b*ouX@ znRcjXXzh@|`)=9$H+>ccq{88Nhwkikwr(|ZmZl%nlRb*oDyV^kQh$3#4I9^cV9w6k z8}{hHaC1{(BEBUgLXAF>8o0NLtcPz_T-jRuGm9X7`D22@$l$Mv{hfN?uFRcf9bMSG zJz{fd3BUUC0ucZPFug(8i6eh?N9#kO=h!k1nk>GjK|l;diJ-{nevpHyD*kcz^IkQg zFEyffGMdrz+PWLF3SH57w*4U2k`r7HD#&Sxa*;f|WAr8UeE5QM^!*}g)(zwe5rApBDsM$D zG^m(<$dOZrdJ18^A!_Detr}>AEML5SplP5?l|niU-Zl|{6<9GB)v<(8=HY%E z<1@`P`)<}rfWSvbhn1C|XA_L#T-MfNhGsvAXttE!wFb!$;ft>ZC9pz)=SZn5O7@9u zF~bij#@L>;e!A`=reo?CmYD|)=_+%FpR*_t6i+bnn)`LWnwb%kSnw^ItRTaH0fSJ- z5oY7?v)Dz&Yea1s2?X+rgiOc(+nzQB0$T`1h6HsZQ9*{oKvk#GR5`1Q`#u$zBirFpjva#qFN6dX+%b@+Y7-KzY`y`zRL~>2c$4 zyND^1ZDzT0<7QkcKIuScDmr_>Y#r&ZpzBSN9(&P+h^~UqLhzA!!e0}w zEcnU$jr+PL{rOBj6xG!%%+;lgW_)#-*@W!~dv0RHF}xd;M&f~3eew-7TKEEhF1AzT zqv{+JcEW@C(i&^geIt)9(~kU=xW_fi%O$m@aY6bAZsZeM8QLrb+5-OeV>##l*WNvL zSJFj`0*-Ab9qiav$2K~)ZQHhOqoeNFwr$(&nEU3v=fnLG=T?2FG0GS9Gi4{IrX;RoUMma3;5fQse~grPo+~>vLn|j8IZuS zb+`16U0QR!dmek4BK{k?VVRo+&Nhuu=`uBSa`#j_fLIZIGJWz8$pujbkuU8<g8+ImX*#f^oct?!rh2!e6uqj@Bl-PXh8)Qy3-d9SXHP{Z;aGCFCa;Rc+O)z*%yyMwRQz1f%ogObhT45ExKhc%l@j9M9khjM zKrN^GZRS}mcg|j8i2v8pV4sbYO66l6c#X+!e;=Jw`G`@s37;!1D)lz}4#ILB*SS}ov zF3It4Sjs_OhTY0CsdIfIf&=L}hcq^;-Xssd&}q=P*)5VD^gQf0O6O(_Kbw}CH}UwU zW}lQ*TAt=$=(L4bo2Y~nN|Bso?HDY<1W!M6Ak|E-ygzM|3e-^|BSa9{4{9UyRC+B= z$r5cGuK6F3S_OpVI`7J`3232C1!hg}>2RIyTEXXn@g(|Bdh=ks?xplv@08I&&KVO3yZWRX{H^G^Qy^ZUrR_5X zK*ywAn>~*5Q?y5)i}bCheDo{@EtvG$!it4O)3U$0)z(Z>+Jr+koxB969zqDJyNB5O zCAw#DQU3IvSJaD_rov1c4xKx4v%P9HzO>rzO|94lr!YVrb_4PWij!B~I+CGaQEr(z zy|zDm{^5AVziUzwZ>6dltURSL1sx3XXfpFpi4`npJ=65yAitn? zpy_Dt3eP>(vkv$U8A@xemu)`vyM&ENmJyWdJ2Xy!5rzR*r;rqBhX7l!;y`pZ(|5pg z@VyUD6_J))2q;btYBgvN&TDg!&soai^z#=j-9qpWLqx3>{^VJ$2&LC@eMH?tQOdR2 zNGFn%k#xY`)P%ds*<@vg`k*V|QLNhM5OjKiF}7h`_o(#{txCU;inK|X5k!yrV5dN@ z4UVN)nBJWS)n3c_>Z=y#tw%2GDF!rkRHlbP$?{mM}u_3j*`|dutCC-`B zEKRr8ADDwe3)zk@BlZ3bIZHhfV_}9&b&5XKh&Hm7vc|qR8WDOtN^lb*N+JV*4(;Xn zD3hTFIS7mhn^tQT%w7TF0;cWo*sakS&4WExz*pEISqyn0wPbX zx&Je|j|^lD#ylNYYt809)kxc+T6quj-iWO4AnPQ`=E>5FGjD1$LfT1U9!D+o=B714ct9 zXKS6LH2v;$iG|?2iv93^ZFT=|R5$yekV5i-(gFH_AirRLD5rw9bMB&N09mG9p%FXW zQNaow-x?=t$QV|GDbjh`0V%HQAw7m=_@pp`y%a@_l05`jB5!AkPjo{yVH>NNPuad$ zh$#qMB13=bK(PYli7?A_{|Rrxz|2v`=_BPcvF218 zP-HQA(!;qLIYKfl^txWy!32zkpxOo0x`d}KE;-Ycr+A*ej#z@Wh;y_neBM?nlAay> z{dA==S3HYzkj3aGrsQOi$*og}ZQW%)E2^NcOcS6)~3 zf67Ef7oN{MnmwJ%_U7>NhBE`$Ko$>W1m!4HRf(C|aS-Mt*gs2jw(R^k7NkViLphtf zaci%*y)eUqsM(AH8eim6>P8;Nv4m&2U_SP!l0l!Fxvta&2J!tfaNdg4+H*t-NYzj! z9X-$v3$W)R3ukfS<-ybE+XG@elZS#VCI&^pT`^?}x(|-A$-t2WMDFH-iJ_Np!9IJi zMj@#15xE^)lET(63Rj+6F1s9u!4GvqPPEsW7w=)XzIVv?;OaEQ5Q$ zFMbWK{QhzB`$|inG@YpysOOFf>r-yccWVAxVb9EdlD)d!wa1^yUDrdu48*xW=Obdx ze`BJBY_3kum4_X?EMje)! zJ4P|JmnCSvYb8a;BSy$d^#7Oc5f!nX$EB`{2QQlSTjCEAX3f9Znnd~Z~Om}rm+MHqsK(h&bO1*V|D3d4|rsK9`L!2n?30#L{hKp-%v zzn6StY7_|sMhO)t35qBL06+tPK>-7V2*IF1zy$&VK~Mo;NdQ5>_aFoU0}4tlDGC69 zfRsw7f{{TaLcj=u3WEriib6m@fJlcxpn;+SU}!=>P$9v<0!ad>L1;n&AOQg+m7)+o zY)M2T>GCkrvs`tJ=L$rS8&dwxo8}Q2Rk5 zN>1V*N^7B(gegUy4fm>|hjv|KDE7O`qbb$JzCBg7OU^(}^*HfC!PqsDMT$nHwM}7L z2$kZCbV*U$?u_PlWXNfb*dvT=I!9AT`#c`;BmGcBZb-i%jM>$a+EIj-w6Zsi)6;@s zQ$2}jm2xoMzFLZWmm~Ldku{PG^9)_0of4*00EydlacW6eEyMMub|p*24_vY=t01yf zyGY$Bx1EUnL7B#s2(Lq_+zzJP@FZ!KD3Z-`8ihqUba`(p>GtvKE0$pLDDlNcjTB*w zr{9@r79=d?Ybk4Z(e{^-0#{ORaO9GS^nck>$YXNHPv9_M;cMTDc>7T1R9aHjizCJS zBqJ!@Fl}qlvKwd?Z!uh3>?T9x!xX4!R5_Qqt5V)%CNWBTkjMO&9QvW03Pk$P_!5FZ zSZ0LmWjw%D22L;@pT-r_7IcF*eTe}Iiw#kNEySF4B@au z6RwkN;qV?Tg*I*dqbf1bf)0P2PBHgZMI&-nWFzg`eYjaT_s(kbY(|fTgbwzU3Oh8y z`Li~K+0#`dH0=*6)D{Oi{ zw@I`6%%kDQxV(R*S@hIB@*T*%&LV&{B-5htR8Du(T;meua$X`DvP}n>IpdZkh4G~% zF6;8R)UgH2BuVE$tf0*0F{^z?Yjt7mCNSn8FkYxMz~+z_b_E62>I451&b-q0)QOcf zr(6ex99pp-0zZk4fF|l-DQW^Ydist26i6aBJw+6Bf6c4?^Uc%+d$50*#xi zAqYdT$VV2_7?kQOzM z-{ML2;y}XBdH4xOrS?!;&ObB->&BGPMi_(aU~du;-v1O@NjL6VW#h>VMl{K2Y19DE8``2mj|Y#Te+H2lRh`{sIcz zZv*WG{$oV{Z*P3s=>qzGJLSpi|Ml~k&&$s*TcFQ2(C-Q8{Rz|)*#6HR?SW<=cP;TV zKwF@{-{;o{(7)@oYwzY`=LYEg4VvE$d9(4KVSZEne^RdxbO73ZqoyB_)gEa44V*pC z@82Z)J;(p+^K1AS=-YGueBzfpjoa%SZd`E^_~}DhT7`u^;u5k&J*aP-qZGHZ-F$ z(h4nhC!E^#V&zX=sU6x8?>}pQ6MuWPu?6CGct(|^rN+9L^*%3Mx)mOsNC^CKc!`{! z*aqTs2IXZR@NRqpV9{lYVfm~i=S=7J^KGfg zXuhIo7x8Fk#M)H=D4Ir3L$c+1QCrn2B9gE{dvl*K^sU$_Rgx&rC8wkQFr^f{ZPmC! zvy;LY5@t8D^nPXMqlOmhheuF7^l%0p|CGDrcL=)kgvQe`FrtY`p?uMDxphw^L$k!c zV^3tJ!J|A(9^fZ_ZcWsQfqkHQ+%_$No|s?yjJ1g>REv!Clu!3!TdP1Pk=8N(0eC;S zV#K`AKunNQf2kP@*2i1*`&g-J+BrWGfD!dDg4Kc?PoOXWsXi&14#*#Mz?#*R>iV2D`u)W z_&xh7yS%!lTR?d^k#_;|o*^89=p`pIaB zSV3P!3T4b9@|CSw)GC!JPc9Daq#G8x)4n>Mh$ezP^c&?q(xaY zYNIstQ_)x5m*~3UXy#+_7?9gYa~{U=gQ41pE}R|kkvvhE+U{_ddaY^perErH>?3iA zF*n-zy2s2Knz7;-SWtc~3V)F()1>U;E^3tx{!ov&DCLf&ju}V6tHIX5UArHL(7N?QT-!k&J7I(>@zcDfx-9oL2pYTdMCzNER$GfmqzvrLCH6V z9ranQjfo2k;V?5u)Pg)I9u@rxZwC~I8JnP0|BgK=xdeOvw1-*44t<;#4Gt zwkDsKga9I+iDaCiOn-f3-F<*mY|JpS>ICP*BZ;ULxvXVNS|}4a zO$z|Y+Z*qmuWMwEPdG16>n*KG_d%-qL(2{`sAM0?go=b|jO)7|6Npr$gX#R?n|jxUl7X9r{+c%YN!|(~hzDpsxLkk+8d=FWtoKE~O`!;QwQ_}ID_}+lVBC?&` zCK|?{Mw7I*^y}Bcc0e?VTA%1LO32hO7;K(O4e>sV#o7@iiHUWt1VSbL3uhqe4O~l1 z9(^sbi9yOFznlWZ`{!YSD-VuGgsjaS9gpg>n7JgvYdme=4LCy86;KG2J`m5VJpzcG ziCnHtR~hjDl4L(3uQx=_2ZP0Amd~EgRi}g_5zBI@8%-1Ln^f=DT3Fj*nV@) z#5cG~@MdmuWy1y2P+$R2ouPKhoM(nhenz4ou3v%bEIE&uRj;@b?p;J__DTL|PFKIE z_pj4(Cbzb6bX%0{qqQTiypS)@a`@~we_KE_@xf4A*Zc_Q#P1g!vL9J8gM!s-P zW8|}au=ONpMe0;a2NId3e8wQWjS(PuKO$zu9V^Akmj&9ChQL0$e&uO#ANpJSX8|fH zl$-vfQXULl%-MFc*uK0<9Vh_$2*}^aLDywI57k{09dlnC(IH(l@^O#_JR@9FxfKG( zjY3ZUh|3nGi*y#QdyBoE$?(S#6WogiT5&!&$7zzGYi$e66=Y&R<*DIEDQL1wHfBKP zQ?93|obE=8M-}mlLN(v3wfT!rbR*$}@%^n~DyUCoWtGV3LFDR`0(a4NK*0@{6`ZSk zzl+yv+3d^+rYj}U>F~Dt2jSiI6*L$ZVuQW;WAZf%>8L3W$~mj@rHHxgep7;Hl5*l! zS8+&ZO!UNSMGREMRyRX7EZ;u0cSt&qa88bf+YFT5yzI_q-+}y>#FR2|;dx)AHbSQp zgfvaM0+R^Hu|b3$%doyH zYS>4C%j!w-(A8A5enh%ejgXUiwR7IXKAG|S2?TmA)>D9Cw*TQU^6-yz?lfa7&57q$ zUxJ;3(0MU`jlpw*iv^w93Vs6>_v|wY4g-#DGbsx2ydg4Qs0~Fyt7pZ^n3~#p1&_k| zM|TL5cAkHHR*6F0f7v`vdd@vOEXv{EKEU|VwT8Qcv zsMwwv-Xn7K?a>+aezFpKS2rV7f4v>k6joe8mcIA$tIjQSh}+DJgbG>c#ZiS_|pmkxmdjl#tw+X84^j0nn8zgOQ(~O*6g7~{HSaCBE`>$REg-7hE{>s4V5in zCO~gfs-^u`)(%hUWj!j~eAK=k>{N4;$=(#@`^Dwxh4-m-S7+1R@wX7%1E>af_&7OT za9o9iG7?`LvZf_=sr;S&_Bzv}cX{9wHjt9^{@sl(HJfFNII8Y~DI*uNVo)OA-+scNmLk+*GL~{z>t|85h9SrzrMT)F$=% zWIN$XaI?_Rl>7OYC0mHoV!(FBS+#hu}Oh1asAT4{~uQ@wNc0 z6hZe641Amt>o>T{=hrN8>%2K1c0`v>X6qIW&MLmlq|J82@!l^Z{hZrH=n72#K{5W< zO(Cz4rVk8eoL)z?=hM@|l^nk&YfhtvOMqe{r*WhjPI7bQo0X#2(~l}Jv&n*R+rV8R z(B|FVos6NJU4sdgzlWRBN)%ukDgQ1GW$5H1`OEa??$?*xBN|n(AQ-biCyo{nW@gIS z!DF=Rh8(4w_&$dcyuSziUzAB=;Sb+uUe4J}OTET2&G?HrWZJ3| zf_%S;ML_oAJ=}b zlE$7~c*I|^mZp=gcF3SBt7>|;K$Ef+6xypfcT~*>bVJUVsJRy-=%f56HbgAN}hU-I+y@A?Mq0y>{UCDpN5s`FvJVRYHYATK0 zQ+(wPGx2IVQx*=hH)Hz@)|3%P1?S>#kv%M z1+yX;q4&4AS~~i&Q<9x?zlD@JAZz#ws2)WxMK*+F_T;jPiD^aVJr;5`ZsV?Mon z_8VDP&cdksVC0H*wko4<{S_!MbhaQ$Dyzn4U+~4|T^ZB?-kF%iYbP@Qid<2U$1|j z<=#PD?lI=KYu$ypoq6O?y67)&#pSJoXKP_c(Rv|Y3Yw`p%-H9@ydBrYE|Ln2SYF#)Q^)_5KK zcSvBy41zlBzHlU{)I$Vu+UD?O>QItxM((D=C*yC>Ws=0FVtBH;b&P%m6pH>VqWEjk zvgENH@{t!=O2;-uOX2eu}{QP3wJjsLd zMhf7*9y1o3m3igcU@y`OMHxwbLr?CMn7l@ala&tXSMb9YGogoXtRLW3Yoz?6AZ#f* ztc|j3el{COYo&Y(hk8zXgZ1Y?K#DG{_#VTEA`bFBaC=!ItX61`&mE8eb~do{N^`Vv zL(sK%rbSHpOL+`JfRn=1Ba24o@J9hYihVTpmg6`R=_ zM6FklVRqD{BCM4HV*E|#+eO_ixs5k*ScX~YY<#2#RU@hDw9YpVDzx(_9bPBtZx+8Z zM2)pRULxu;KP8p#S7>h7I|#%dOXDJy*<%v^Om0Wu2kV{)Poddu!pl^iftE5xMUuo- zN*2STnAcuqkHS6APoK$7FLKCp@W%pY=cqG!>DdyMl6@;yrfG-l=U`WNtae_m|0Ld! zfYX$Tj?9m^`S&=LwZ^bpaPl|VJ7eQLS6u z_HMYcfIV_V)_g{1MDTXqt@T4xOx{%W+4{dIOa$<3Qwp473^sS;`mS{)Qun*-w}``* zEOjuc!*jU8z5IAI5%Vb>V%q-#-QMV3gjunmvsRrqKRyC2dDp)qR;Rf&yf^sNd`v_% z*q5@7?6D?;dl>FGf&@gOV%;LlL+-^|?U#`XrHlWu%jYVqVJXw5TfRyah}yabg1*tY zMVYCv(q*H)Qjwq$(!%re6I1NAu?<^-i}UYkKhiu`1l~juOdW>~m;8!pSh_)n z^cSTYf`uqF2$&m}hNIdANu8AfJ#|nh6~;o7E+UBvjpSj1qQM<4Stp4|s#cLqn5_We z5<*3W0inWL+#g~SBL)+J6vqhe4@6ZgL6PKcrAkhrAy!0D3O0!V2!kmqgaGhT&_N2c zG8EBpeusd-(4@ksG6j%Ov0{bi)ZS^}wg!#Cz?DePNwQHIYgL{bbacQNBpsB6wt`H|Bt%#Xi?n7gNihkmRjQS$jp9Z<`G@35AG9TvSzX^G8{C`@onsIH zsjH$=+K3ECK_2`*7gQCNDId*~tKhfQda>1agn(urESqQphupk;8V-$0YV;}>qmB|Ug9VA(AJkRE`U5|dNV<%F9p*!5+24Hb255$y(1m63=45y3qW z5*dMrsjlkiHX%Sx%{^eF*-jMLG^G`&gOo}fBS;rJA}L&1o1NII5fdaN(=7=O76SL6 zRP-AxMps4_(RViKhbBAtKH+?%NU7PMzY(7pD!Si49(m8dPzHMEf1`*0&c)~Z%~=nS z@pEQp<6B&S<~R56ejBgb{{DPDTR^RcPaB}WzJGHY&^`PH_-p5S*VJ1KZ$YO-VykoK7Y%Kk2J`o`pu?`f#DiQ zMD7Gh#(T*foUno8NN6Q|jLHw5fXMw-_!2~}Wb8T}`7>gb$G92jU$C^0ChKmwT`QAK zHG!znB{^@$o5TfUL(PM?9KJ=}$Gt+)LA^_8NcVK8y-*Ln4p^0~Af|%Ydh6T%Ql3Uf z#*jOU@1m%eogDCaGrKHg*`080RQ7eRzblJHjo=2-JJ(F?78-Jp8quQno?>A<|z|uZt>G7O4y-!Q3eN92bHlZ6o-!f_PKCBGALQe&3OAQ1JRg#uM?;L5lqO z`K0a)JBmf14co{A;vMnFUIwna^XbMoxzThc6_(sngmwi9$K|S6QGPv^R><7Src1^oAAG zHSQ+_K_EMtLUj;7?yO~(yad3_o)?E<>}Xr$NwA{>$B(%kGlfdUuHKU$D8e+6(0^{e z+|=ZTGW@9DpY)_WcdD;qwmqWlocI?GFss+mndqEyjZ6xqivD^3!c_;4Zs*egJ7oZy zXliKzhA=yxF0EIy;xJr695Xl~Hf=xfbmU;{*0;@d*280H>FkIk&WD}RqV{LkmWn6# zXs&ax8g3kNi%ygWObRAZ!kUfV> zN`X(z%)6Pw3-q2|QQDer8?1q2`2!AW=jjXl)?UN%-S6QJJZvqVU)~`Chp5wzAirv9 zI)HDhFZXIf#-zJU1+BH*9)4&p6{f8GGB^(61zw^9^6y-GJ-3i;8wbWTOjXq3dFYgy-%hOS|5(q{!eT9cc%Ty z?CIETHARC&PQvD0^tqWRoz>woZdMd0qV+^Aek1SAACaVGb;_W(A*ajUgjR@PPf4u5 zwbVNyQvwO{!Bj>B*NE)08EJ$FNb+ zc{y{xkXLcKhST%kJlpg!*fEX(uh8pmynMyq!HHp^^cfSDGWcRVT<&nVLYCz3tY(ux zizM^7f}PUV5BCnc)I98$nGG%cYTK6CGxzX}TK62f6Oi??Ow%Xjiu#!e)FrY-kQ4G* z3JYvx$HIpkVJ8dN{CZu|0Kar=icjfHj`FhV%Jy3o4GAZIo=`LJv*jy3P)Q@07#SA( zCiwr7{;61>13#JYBQ`fMkpX7>jpm561L@s-a_1pYy}OXg8fO-fa*0-S9S^^TBxbh(Oc)zZcc-#f4-rg zV)8D|Zow=kV^Z8b2}{zw$JtDsi>9oFE{lcqo_jODXK$E4w9Pv#~?UG9z%H(M}Cl);Z^e-*2IV_y8{WFu>UxDrxI4t|X(b;N zLe~lwa5c!|?$Dl!q?v2G5G14-_x}V{{yv`(zxaPR>PPzRBRF63Lo{^XzQoP#b;zv{ zV#Q*xdfo4qyrs@LXt1CorF0}F_C3ab`o_X&x9m_h+~rE_&as4VzSm(dRvXkLpT2%3 z8o9OfnZYaE1r_cf44`~OPcp1T%8ofh5RDXUa6oZ@7<0rT*k;Tyu?-_u8-wmXk*v-j z28QLaLT4j}Atu}ovl=i58$S<3)S;utneg1pz6YFA$8IIlo} z&5lw*>rh!@t1MdlQW)H(PslpQOPdfKh+!Cn7LMCyh(lCrEL7q9!z%q)M;FA;y zJ3;r~4)jEBjQmflA!!V~>aMu^eVqdDNxHK$r8hueVIB#es(L(p1rWe$|s6={rRZx?wpaYb z=l+mX{%b6u#yC;iRV@Bcc#zMYF6P?2y2edbjKW0FyXZ-86A5PA*{?ZuxAN}cRvR_k zc0v=9AZI#$zgPWGC~2VfZA2QQ0EcS!Y?qm^Y)_mww7_?K+sMTSx2{t@e_D=BkEa`w zbbc%PHKn8+1gMb*yTcg3A(bKJz0&`^>yB!LE-iiMq^b<7@|S+c`RYsql z&f)_B0{nTrTAxHT;y;M5+x2z7@9&8?&NJP>3g^H)-nmFM%=dzT4s0QBrSLRUn5up4 zt1lWSqpn@956!#ROe=#j=up!?eHcN&6|pJO(=x3jl{zV8b&wUw3H*VaB2A zPFqY#gi=WOeqts*Ox{l6gz|HD!O@si&cSSq6WRT@PxH2`7}hUvF%L3BQinn&dsS`8tx4^5H=@tctdD7g0m`rbD^Cm(O`aPe?j3*HNq`3ZEU`JF0y_V4bd zf`Ltc0M4!bN~&$%tq3-bKR7dZMq5U2YO3A8Bhc9DOp`zd+6az%?|S`k!x&%sR}Kbg zj@F{!-V6N3K2*5P&7$q8oT>`V4WwEU@YbJ4jffy8@!U;|2oT7HNZ?OGQpoj6U||F$ z7~ol4{A*8;Y-Tl**i#FKgWP&sF^ffz<_!)>fKWn-6Jiy^1p(v+ZkTR=@2C;*>7fv= zA24$gU)v##tk;BJYvAn%Y5Nt@ zd@l?M1aN);9e94FriRQNm~5x-j&_Rpuwt^IY5{(|y-tN^^u&!I;EdFNj$r=>ZVOK; zvDRd0&=O)Cl<*dBUB*@hi+NDF!&k+cE3l~izgK*t;j~J&!WVL_OoQS4)Q5G`p8Cd$ z@*I8+R|nzfM~oog*@aBkdAU~6YDWWSF@~7S%L<<2Al8f#AMdB3F--u~qmYT|({eco z0FVi*$$|`zAvGPBhX{YPlC{R$?;g zhQ*7&H&GQUSB|a%t#>%-hR+cotKb}JneCohE0= zv=FqAep8VdrwxHz5Wd}wd{Ca(`Fk&2yOybXs>hN93+qx%z!MMNf_N7|xL24b*0um- zxPz6&#j4o8gcwpyi#+R*%hc7CsT_e&ck1ulM~>&#(W*Vn37=mdFlICf_Fr?eJb#CC6hz;Q#e1PPDoKXnvmdStzp?9ict#g6mY*x=Nh6 zEq1WKH0w%z|EU(SuM>)W5=zf}`9wT#*ETPD; z4YAr%Ns#@AH~+y{&BPkEcYX*^K^3v@V4SJksV_3&O0c?+L`J|jPIu{A$1u+2y-)u#R$Prf!+7>xaE{_q)Z0^`pEv)Vb=g-O*(4Yn(fI5 zA-^te(2F6JIRC*RsZ`untqnpE2>3;TQT*8sce81;GIIfqgMBS=zv(T5d#gS!Nd3fo`MRQi8nwx)swO)q63EKqz# zlp^im_cC!y6Khu?xrXZrJM}+fPV+SkUH}28+);#hiFu@O$Qm_Yh$CoM;S}# zo?clLfEVbQj8vx>o4*2fqOY-~=W0n8itu|@DvJ~z*BE`zgs?q@h})iV>+ z*94QN*I$v>RA7Zuq!zZCBY`z!?L3)<32{h8FG1qUPKf(})~|(WV(Pz&QX`5bil1rj-PeK~YDj9U-`wR@f zGe_73MM;z^F-_hd?YKfXuluUt_AJl7T^Kx{S}LsOE+(WSUYXJ1249}MX8V)u-Q$D< zxbj<4CdRtHth}SQn-okRR_p&-v(2FGuJS&!dnG` zD2&EB)bYcxXO~GT44N+L$WWb~F9gms2P8Cbau6J&AfRZ(*(#lqRE$9s3xl!`_3|o? zjY{Afp%X%qIWfiu+K^HR*dEip42tG_o?pnZRiCM_bu?s3WMKb2y6hx>0Jadl1v|Zq z&mcgdgu3ltmsS+PBb#v~*>W;Y)vA*tO_W<~7#=E-ic&^72pH%(O-etlUC((CPHJ|& z`n#eO_ny9_#{tUe>gbX)mPo(qpD|*`0VYSf_Fs?vTceFEGI7$`dDShWFxh6pawCR5wF{3W_#Y@Th*w- z<|%bg%U=m<^KfwT+Gw-3Q83|za#4TuGnJsYhA?W;(O~A53^+LE<(%#+)Qtge&!!C$ zq&XW(R}U53&3W~nuPD`_+Pj+oT~GV#WW%%i0S*M%7Y&Ydx4kX4>sX+UF*pBc(KWGl zaQ)1jHAYNcuqSR}2Lg6iRBbzoAAXU|EQ0iZ+Pa*hH+;22=2knYFHN>UE?MmUi4 zflNm$U@50YP^A-ZYQxA(xYlzm)7$@!x9_2RDtFluOu@7GrGVBV-2>QoX)KNVp8bSRM4kv zzajk2BFwPDc%R#V7oE%WRt(gvh(Refsu$?c`_kemR`nZ9;2L3O$R_I>MKf|T+{Dx^ zM(6x=qwiz&MOkY};+KkGFL8I+%Ypm6ecF{8K}^^j^1N5Y+bi4T#h!lfG$;T4<~W-A zk0lTHp$SDy%CInHswR@qn8cKnUWS!pm#GuP;`*s!Tgq}tKPn;P3^!1LoM|&xzoXR~ zt&qNJu$KQcss#JAs}$euH&{1%;vXGu^))5ulSmy!F5?wC?$=Sev>GAF-xcMj-_>H{ z;l-l0+^ES3)>4^zxIPjI%Hiq`q@CRd+dCEo&&I)@hmxzZ_I;ZeU?9JvsfAonVIq=kVv4JIcp6Ghl}hE*&CB# z<(w~c)sc(kyfuVj87StfSX^e8Py1au^wi%eH&)?g&2O{#uX&{H81&uOU6t5Q$b?OZ z;f>f%BDYSWOn;+f_qqJ7@)&-GymXc%EHz#gq8>_=Yw8XmJsv;?T``kwd}Xj#2KKa* z>+|H_EE(iK0vGVFloIE$iwWV%2=foo6}E!gXX`9g#k|rMb&G-SHzmR@LfAo~c=+~B z{$+0})o9yYJ2+o~cP*ZSTGut!x)P|N9LaxaCgDkUrH4VVE;fQXWHxmc<~%s1D4W^E z2&G+4t=MyY{yzZ%;PNNTq*`yctAbZQOgI(WEo8%+M7PI}zyvUBo=V2l(=ots2O$<@Fl4ZX6LKNCV z%*@x6g?!tOsO{@%I46}=sT5d69h)dmpEgNDc%8U+U?za!dEI!6IK1~~Ml_;CG}b zAXJSlmYO{Jz^tL!-+!_0vD)||I767*)U*?iV;EMvf!h;LeOG7rsg;r2Gol%ut<2rq zi?AYM$8EIC_Q2K4(!w0a{1*f)ZjE#T9}ZQ|lQZy|YjS|;m47#1MLR;!#SKZdM#aA+ zwnfdeuG#6s)*9g$J>-;#zdX~bEGyeM7-QxfyN0XK6fECRXKCs|aT^BRre^+EmNR5% z1W)=GRiYc3y@*3M6}?!1(CsphJChH4(UawG(Sm*@=WBbj2gWrbA9Qw>6Q^bucWu5V z7*XK9!gYmaU)Yd8KtL6gqOB!ibN?a{Ruy^^QgY}@dM+TPAAR_IU+ke9Vu}j_ z#%v*`;rd3i5|-Fw()eIIOmU*FO!Fj5FSK#FeB|@3KAB{K*L{I4BNpnW)?ae3$O=7P z4(Bk#_8WP4o&)y$3v1&#FEi z&V$NeL5iP_wMtJ_7ZGd{eUWNH%CXB^$*V6hhUEk;g}#?A{7C@os79)9g?wrEKA0rP zvcScAH@Kqw$kG~|M&JNM1qf84iWDUF!`2?-@?S!ObF&zV2pSi5x|Kwuv1#@}fNIGY z%po97Li*h!Z)O7OIW?~KEZ&IGYtY$j0;$;Bc0@oMoc?@d(PYppo%6^A+^ps<2&)ej zrT9+K*q0W+LPvLOK7#c^KrW^}9swhLp=tMTR#@Caf0b)_HGw??1Y8=}62GU^6Tz5~ z*LhPBD}=9)VjYyy4^mndaX86-JD(N-gW)7Uu_r%1=T$DXWem)gLqcSIoG&irI-V~g zY5!!fLSZmLfRs5!AMyB6vWC_VRBXb#X_<^bh*iIKZi{`qF9KG|I*cHI!mp2oAg zfpk9)>gAD-g4=W6mGZqvXQNt#btrQFjaO?%~#|+!z$j>Q!5!G+r4`?x{xsEh3d}0RJTAuj>euIGM zFx0M@>ZSCprZ%JV`4Ji@lL0{{BgD&jnop9GWnB)9AV5;#dWX-}&-mRCavr9gLS5&y z`-(jS_!!{**uMWom?0!z6O0c`&jc?wa&R!ea^oN-A{S@R23fOh*dVUd+T48N` zi*g`;t&P?qoS+HMhxGNE9Y4T5R8Zu)K<(VVPhZ_-L8RrFy- z9+ovk4v2ESU7c_IIR^o5lZ7eDqifpA8|i8pa~IS43Cs9k81WE%D#6m$|0HUuCb zVLwXdi!s(I9GtxD^HY?i3{s`!A4^r2Tf-)u96v9E z_tpGIBaIt_;vA(G?!XVSGA1++E~mx@QA`HI5-ZFddLr~l6Ktr0{z=|J6zz7TS*Owl zZrX9(Q0NtlM5!*r;mHEsA|#Q9*=ikWYH7ljNui zUi$zMpo87+X(D>yJ8JClb&6src6t@Vc7vt^F-~gI2YI%?C5Iah;eS8_{cBZ?(NKf& zw4byQD%t$2 zp)NE&!~3}+m_&{Y-dAE7q)Z2M%`2Ji;vbq-0+IXR`o%i0V!&3zj|W4Sc0|GJ)6x)q zzg-vVPf5LQ!+CMT$v!Uyl_Yy?Lcpx8uvf2F?T6=e@a)tIt6iN?Zo^{Stmt&3GLX47 zY-{T^-mDvx`!l;JleS<3IiZfx$~b>6=fI#bP2q)4U2 zFY)FDs%>lpGCkyg=9F1j%Wn`er{vt7qL-GxPQ7Xj(tH+7Q8vB`le ztJS0IRQHB?H{m6xj;QsLsl=ZQ9YLEdP_GE(20P-YBs!z-ou#>)$ZpKX*iNKn!*U^! z4$`|ao|@#1CUtF%1Onb!KZ%B$5Lbeb^#l0@^MjgOjNmi!<%?t#t9-$ghLA6q#e%X9#dpR3o{&;c`>s3g96Y;&CelHUIJ^j1mq;hK?H zM@|UIKA(o--N?d5iaXQ!$=%OER)CBMzPrH=5bLkCNpNivK`E`x0jWAI&Kt|%^d6A* z$2k7r1ez&Avu0%O4+nH=yR0eQ>QKFS`=PCF8=eFZUTkW5j9OIiaSW-p=m2>9Q)r7e z^O5Nf)ohx?aoEYCQ3Kwv&WhpFYLHAWqVZi{dhu{y!cFM4sH&rVvoDS#gnRN0PG}jP zjfe+lv#)R*3YzC|-QrKA-@y)f>Vu2b-iWhpxtXm*tnsxTj(PmxPi|SZYeZ5q4mqmf z3e;vHGBcc@hgwlYWUY3;CO}nFuiZ?NrEHjZT)3%w3vn8iGDPlP8u?|`LCk8Ub;>`4 zQu&^&0C^mf%i^3rk;P$l`IE+BH=eTKb_o}X&fJDvpN_+Rg4>I~iwwT)wipqyX2`Tg znVQzxL`Z_}orV@{w8w=73`2yYm#E;!!kC8tVqC*v_>$-E^FCMQH2B()iiXx$VP z^7yAF`}@H(iOdWaojaKt*B#B{rF7i&XM``@g!rl=s5-@`}Fv`?{WV z^`tFUK7Rfp8Tm>@d=Y_Nz`LFk;$C>%*6*Fg&`_+--J)rAX+YD|aMkS&iz9K4l=x?T z`fO0%H-f>%H%=P~%`bj5wm!6yQnQEoDI0Mvkeh5_il&nq@s0d5vqF!2zjI7(iol>E zEcQRtuy3diuEeorWr&0E!A&aS|IZQq2Qi!xC1kKL;=^x)XE&vZpvz>%(Cz{+F6m<_Y(B|R^JhWThwTMw|e(3G`ADWofDHQ~_K z1+P>*iRX0#n^Al%@ZvL=tfEy<07{ML61*4o78ULV8{p;&BK2D-Ci*t|H8uK zLo#P_V#0D_JlyL_*u$Y8zr&6u{=xlgjq!>`Ghn_wCP#6}fU?6r8(zWp{m?foIVHab_dGC9AF-Ea4K#a&#UiBduoU(Lr9m;jh`GIa z#=9eg__taxlQN+(Fh*|toyq#g+1cZ&79$8yo<99n$UOf;W36|TKQwsz!#izHVG9J@ zvyPc9KEb)l>ci9DCH$JoO-E?H)XaZ4#{7FsGbxD#z|Sp!i}5Vh!U{v(wA!N1pZ*$g zJ5JlS>D0L^b<>;=<^0IG;M)=>IfDaQzC^PEmDJ*J=7Q>-B0)$DTVXHeSgq= z2D}IjlYT>4lG)^9G=fxUj!JqjbDdyfa!pFi3fquLj86c@q6%0L?)Fcdk0u2vs6{ev z1V9Br8DEl|CXXAXxUy^x3BqDYPI|%)1gt%3XCB)~p*(~w6r)X16lWVVL@Jk-bXCJ$ zIvg(}51n~{^|EhdTFlg{{JStui$KGRO)oHid2m7l?(?32yvX;;B3ckof77%3%%NxQ zl+9K=-@OxWnc!wpDt(=UsU4EO*<)Y;SS+;(CxQ#f;i5+|mMqW3u~2muG^a4N3ne&GM34TK&a9bKIC`EePO| z{2}^{k+eHAT?2a%55bPxHeo$X!np8bApS0R*DF1VdY4LE8Q z$vLwUf{3q1h698l+mnkG0V^1=vW3tr?a8YsG#|dLMd74r{z}$7&X^u@5o=mK%)C(qW}U9I8REBM0gKj zw+|{cH^-Lw_mh=sJg*PFa@h^)w|GZlg#xRs z-`LL^B1LuZ;N351>BogIB-)UDg=Y_@%?)X%xywD&F*?r)flT-nf|ynOV05xSvjHaa=DZZ$5b4x{%%uBoaQ4(V}+}-2oj;&yb-DFev@b@jZE_eRZ5Gbt9;>ARa~nC za)(Ob>G*SzuLrWlxiZ9Uc>=t9V{xVBtdgu_t}v8!W};)PiD@AYM?t{8`H^lXwSdnw z%cUQZ0gVjvAxRnB)M0NzOE41i({$O8A6N@!NT1wql*+b+O~dHvi9Nc%Ps3r2uKv2| zM>o$o`b$Cy3kDja*)tubLk&~-6GJ}^f|6Wtu(c*4oN7!xy&qIvZAK@TWU3$hjA)+c zWJFM8QaysoFbr>-CRS}wHfnV+cO%jx-qLz~x6Z|6#@p^c{LfA$>9?}KlkA25y8|%V zN63G6#g8g9EaN9F7VrLUG zqGI`n-{>!%e4tf!y63a!vS$o@JL)R~^8J)jx#DrqAPLm8R{yZ5`4=zY5V8?4DjL>P z5hTG^OvqVqs(nNmQCe&kGQk*H`?4$3@H|t+>ThiR4nvEdT$olF9;pqXm#~RR@a{~; zBiwE*wAmnlBWrPHaYQegNKyGFID?4H%dPr$aR2?h%fX6>QOAK!+V5ad4BM$oybT}A zA~HH!T>Pv&yNol_*E-1spAcY@Jcx;QhoR}r?e87v3LL(oZiDHcDf*Hr8h(R1z=_GOp%^kLvUAjcs|y35ls2@O@@sV9GxztknERP*`n@Ij5ntqicBNqbgZ_^c(O z{>0c>_N=+U#_&=wr>fO0?_p$M8GECQH%&b#Og3W8a2f?@>Liib2z=UGEx4J`C4=up z4xhO0E|#@^JJdo9Uns@gnfB^mx_UF&Jdb5Os%NmTkZ}0*dLQG}8b}R2er!@p~P?UtPO-|HUq1)$j)uLF}g2lJ2a?TZhXUeMN?yg0-CBSk#^%YxCq;+R8ly- z=yl4s@Q8gDFL}&-!b<-60mM7nwCva9_4YM1N5+5PSE9)G<=^YgKC39F2ZojJ&!pl= zWCuD{ypL+w@aEZ%bg;Bg*9*0jI+en_OJ~z0g)wW8gLn!kf9Z7e$~bq7?1k!6Qb3ax z2(6!t8uakZ{|3rzqk`|*Ijj+a?60f#i@l>n7nVWDqc@A=>MfrytD2d9$Ixcni^8S# zS_hyc$>O($U@I)P{`NYj&x&t;UClWvnzv^s+O6@3Zt_AF0tcmhi zcYa1Pll4!`&RW7zPAf7WwhHSVKT>l=IKlqnyE9iqGYZzhnrl^}xXD@blNlF?z)G+= z%k{fEy$~x3v{D_V<4BOKDnp>FnaTmFtQ$y|(DxEVK?lzkj3i6DfAl$x z_7uMOY$qBW#YF?R82>C)x5wXv$fOU_-ya5I8( z&SmAWt+wJO)+?%{RJ}i!tS(ddEd1g>eY{LNsa>oW@m#;p>tE3rUOdA$4opRJKU*vM zvbqF}gz=o)=jWD6SdQERDrjY~`q( zMn3)|Oh~L+G^YR7|0PAhLongt4~)QK{J$n0F#Yjglg|H!$^TH1p#l)R!2fq ze5D*11}2;tEZV}fBb03v5pkFFAvXHU>VSc(#$D3vO9xfx};ZImlRbFyU>eX29QbH_yLcI^@>S7*nO)#88J#ZRnMI76g1c zp~;%j6N?Cz5&wZ(uSP*Qg3_@q;MzD5uWuqQH$WBbK@SdVZ04?vw3lnlH*x)dGZS{O z;w?YdSe=#a+xD0K$Vbje_k?Ef%8ijH#`J-J4}7hf3RGycB_(%75)TQC-T^q~6sv6_ zLS_))v>_9``kJDxF6VQdZ+R+c3*s2wL}31nD}YMDs;uqg0|H{zvZWt0S?BI(Vm>*s z8b?#ocQ0F4-5%$0hNTN@R|4=e(mvpP*516YwUP-FQ&KicCNRfJ%X90!!T`Z=deWga zM>C=GJ8;DaE_L%MEDi_Pyi0WcmD>nDtaJADg2wrlFD>;g!HMd`dI$;J@`3WE0jcy5 zKNERuUn~0aKC`*vUKQIG?&vDCbo5t3+6|b&frZ_IJ}OhDplsz)0GPA=3>5RY6mjO$ zXUp^*yN5ZaFnlnV-nK650XZt-8G~IuW;uS&Uum9gH+m5o&?+z;`4Sv)Ww$H>yY#u|2kf$M zHYW(U<~ruN*0{}ZUWTuCMj{aIUn@mfQ=*_2V;!1>>#P6LR-eHO0_4w^x4G-TTyd=k zd?7Mck@s9kUGu_#xa1ZrqVChW>zlNi>Hwa5-&5LbR7L zMT89vp5;`H)_rCi3?B_rD@?Bv)3D!)hk#Fhh>w=}5lTodye<7Wy(RWu_7F5+#%#dg zPl|1^mG(THE3x<3SRZ@&H;{e_)R0=3);(1SQaEx<-B11_7ODi8#{raOT>5-&-EVp)_aO+^ARu#7o*44!b#!fEb&OQzFG(z9(|6~JKoNQHgrMY6CweeZ3E01gk=Yd52g5<`F$T)+e!&@vUa|`gGG-z zd07R=_(99}l}#M`QnBs@|HMO^6bt=gdru4k+O3GoGa+@%}*Jzyk- z4&oo`|D@zLN&0I5S?}UU-ECu#ccH(&&Kydj-v3LCGy|__Rrc3lXXRhFMrONF(5IEl z0_Z>PjQUOMLWotweiYJ<%~XgWpnvE}ec`~OqInQQP*KSFHw2X&C6qXhwyK%AR1!nM z@^wl`WE+Q(_O1s6z&X=4}buRLB61OL`nx|dL?DJ z^_a-RTnc(DYX!LW9k0IX-{EyB^(7!cmb-xUD!&b3Nj|>emyeGDM0t>SZ16}`Vu*Xp=;k9Leu0$?t#yv`>suoIO0q> jzg{&CGeXGNj!8|uZuHaszTiB0w$RLX-$n34pnv}#pH&nd delta 5688 zcmV-87RTw=X!$Q7|4;u>051Rl0001h0001h00000000000001h0001hksulY1CcvO ze;0fJ0soWY)cqo;v%nG$Ca5h=Dg72k!0}z~-n~8m7ES4O9rUTZA|wjV&$p$P;X5?v ziK|bP8PL^&FM_nl!2qMNnz-Sl+JJZPsNc<<5lfnQ}V7j`kb8Hc}c1EyextRA{w*XBuhXRtTOn}DHKj}@TEW06B5ac zuErz`ODu4sR2HX=l5-EmvD|kBh|CosfYKq%HEi0?7f0hyXR#y-P z;8fq};%kH*dyCG*nXn-g;jBSQ+J+su}Et3H?-bwv|Zgb*~)|eZfm|d0;pl;XhYB_nQ?<5Tq6|BH>Ob|a) z2Po*l-uyO%84rn45eAsye>Ai5fd-|oV#5+p0M-ODQCmjS_<$Qm_&(v77O`9Qmb&kIHw`X@e*jbG6~vNk7m^4A zftiL+!3ojXCpC0jK76^h z#(s$;M3sQm@e0sHV#W?#F$1to5k80|P? zXch{5k(dGy=pgowt&_yLpli=+fa%P9VEZ$W4PJ!GyK(sP0n@-&#loi(!*VYIf>|Fr zf}NuN-ZZU@e~dRP+e{$>seXbc0%K(G8ZCd(p*41(bXq0JCvGNs^QEmB*o`i`y2Bv$ z-waA(6^_gv>olPr=T^6MQ?%4Btb z+fY9Rf2#E3KQC0HIYy+D?n?*cW=<9X^;H)CVFX~N1RuT#)nC${v@Ww4uXJ$?0bm)| z_Y-}ljMfU6qHhz-0Ag#{j*E!AFCjnzK}B;ky&na7bnJt2imQ$q%@A_%&j1;4HJ|dx z_4Ts$qYvd^)*yABzvhXqaDe{rfu6zj(K7Qg0AMH2}NGd zcRmA^s*pm21%%fCVtR*`gd!L_C1z9WB(;4Jg4{|l4jy|XElHTI9LiS0Lv@xDJt$NaAylT$B$vT$+t)BD!lxIGw6)GzKu`==s+(j% z=85|>^8i07Ec8Tb6}6^(^{&N^5Xt+je_3||Bc`I9xYhiBJXI}3LSB`PVJ%aL9t)n9*8*nZ(rs)V{33@KDJl$x5|~RjK(bi4PL*xdNXbXm~gB-e>}?d zEPn0eta8F{z|g6;kMnJtkhKMa*9zwu z?+d9iZDy^*EtSFY`tMf^F!qy)f7puW7`YmVfRcxB1^j+aH_MIdZ*&kT;_o>KQITmv z#Mg0)#UERJPeC+3cT+YueOR1O>X8vNRza$4YQMa0shXMuD4-8k&3$etT{iS-9g$_Q z@0-9k_VeU|8-^}Z!EXGo%U_f9@8)#w`a> zq6O1v$2@;s%qxzM8~_@&e z*P%_mG%4~A=ij>DCs@&fDfM~jz^=Px5F>Y4HYGLbhj zk6X;cu{#wU^c^*2>F8u*f4Cf6xt*muXK%?iwfln`T#b&R+!vL>mlT-3MxN20(^o*| zdref<{ieDP6g{WrkJ^iT0cowBVQiH_kn!Mw7k+gpv5zF+kY{&emnz6WzuhZdGx~4K zvtkrpmyu5&3G+E+NKhE9C67_{KI^1~v-v4?Hng^nc8Nk}a#GzbfA1gJ$-p5@Bm
}r~hL94%<+?L3vqH=En`W50633Pe zdT=^-uO(Vp+2G&^xp$yWe0N`E!c`U)(2wsHZK&Fa&uhtkVTRfJv zmDksS@`|S2HOw+(ux{^C45M3uN-q{jJ79@QqZjj9Aq_fQEIM1Rn91ao@xI-?E6V<9@7nA7Tel7 z-vz!aq+1rt+7jAY-qQy(RaLWUfRnHdBp5^iopQ`~rzAFPDAx-|8~lE^P>qiKNH&xD z4JI=vpS@O)Ce*{`RBF(qrGyW^H>&Kl85{vz{StcV{?de;__GnI;UK*!xV2FwEHsaK zOgWQ94nlvt9^M6L^M~#-ytQFIDz03z#0<|5N2o!3ZezYn#G)E%FFm4xbmTdQL*288 zbn;Aav+)6XvrlUBF~;7M627py_Q`|{xayIbxJx^F!bcee2TRF)M(mHs-BtdGN<*aY z58P&LbNwyBLe6Nbkdxp@l+3G)TxY!+Whc2J#zvFh4iu(4-cVC^l$XC}9K#ArS_GE;9m>f|QzwTr5d=sq0~W5zWr% zeJQe2FOzc>_4oQuheRjCv_sOh7v!_u6nNSGWdd_!h9YihLvya-GY@OQex%t{7z@Ps zJPu&RB{6^h2aZSx-7QQ?-lH)xiNr>S)a`Z*h8q$Pj_$wCAeFoc)8ar z<4)>K!`l*w2J|l+vXf!qA6_(kV0@NjW9>(W0H3<}Yr84BBSV}>6a$#WdVSra8cGF) zc`MO<-_w2;um3-`26PDX>}d~~`uoPvX;ySOl@fpJjmIuO6h#ht5HX7)Q?55=F^gRT z9{61Z2TF0L1)VJV9$p+u{a8H_x)XvCs*(#lMBibddk>TFa9?-VwcR{ZL*GKqY z7<&I&EA0sJU=eyTTLP*Av>MHQonX9#6o1=dz4EalnOJcFe~&FO)76M`fWVbf!vL4N z7G*(&4Lr^UM<4OBG`bJD&SEP{;0MT6C>K8mr`+}$Cl3VPT=eY+gd}VLmT4}i>i}l| zWaI&=x#PigdK!j`Q~I2vlNTEzf0AB^Lpy1`W?9o>28~%ot?lfGA)8Di)s_TFBb2aTETRd7%7kuAN8=ay%CTA*&l%L8~l1P8=}usBvCMXfZP z``UP_0TV<7#O^rxASc|+ae6LIl67HFE%MfTIEA1a5C_lnj; z7pxWe`$6mPU9Gd)THJon`$un9_p~0;El=7L+D~Y`qaMGv_d&}((t7&Twtb-Xg!YWL zE14R?OCTldw`?Fh_`93+AV*7`NR!|i z6o1PdlvaH!&~G#$Sa3vols50OBr(eI!oU3A9{e1M;{h)!`}e9Pi|&XCvhf3(?A!Y# z_!R^-+^z>-L{Whd6H2}789JbR1uKD6eRx)>#@+9eXs^Qk6W?qN0}RCmy=0%aEcxjg zQ73B|VF-(}DpHBlDhLcktQyDaF|YQeK7Xe-2nD>8=-9{CYTh2Cr%7^tFOa8S>tatf6dUJ`5F{<)4qRWDthmSxPf#_YvX!xbzM|2FG)G@rqA_tQ+MFh8_POq~R-Wyj>znvFgAz)#A(g(t&-p<_+hoc1K}W>5 zO$9I>UlgXaAb=wTVVp@s%NQZlA@0I5qb||6`1u4J1At)brKyT}&=WBSIx9%xBz&0v zB}(+Vxz33Pj`?F3tmH7_9KS+xwtt=EIxIVH2NE9Jrd)_UzAIWVlOaq`J?<)1Ib|iN zBp$ku*5(uWk8G{ax)Zbx<~g^f|3syU*ubD7=&7yK&1?~hHY3%X zZFsN9;p{&xhxpZ^t~AQ`mL~;?GB`HBZU;B+*?)xW1tjaw9Yf9L1QH;$zDeY)!>_K_ zrgHf1y-0mN>cOnQ|s&kr0}XR z$f<%>S1oN7i)f{Df#@8Q_#7X9`5gB_615;CM(=kLP41hncbZ@}h3nOhd*JNX?|22VAAaIgk_ z#em2qTA2?fQ!bD;^6@u-F$bK@UPBDS0@pUd17m?S%VfOyc)v92L6}KHj)-cVBKF3M zmJPxZDls;H0bP!0(2Puf1}DH2APBRqi1g0CDcs^txH&neH-l~?CRlxNFo91J`_KNV zvmTup@#<(k7~|pr_<2Z&t2%^Y`1jJ#_4o2qmjU)^TjE=FVTLHa1F zbp&C$Bo`v+zd)I%1lpEq-;>cN9+Ubf8h;_j^ao&yWXjl4y`u4Mf3+UaTfZdsi`rjl zYoYd-`Og-6Px({YF`L@2X>Fey?IGiN4nxEh9z2F`uVHq0_D^zX%FfqDuJ(IfB>B^D zPIuJieC}WBm7VLJgA*#txe*rdt{r7B7%EF#vmzLi2dm9UWHM+B-TQ^a6d-Q+(tmJP zIkWnT^+l}@e72rF9`LUset^Woi1dxEk!sVCQszyX#8CCRR{tpIfxYI-fSrqx zjC-{E|7_md3wD(J0)7NT>FUVc@RohcDhVWs5tN{5NvAKz}{Ll+z5wm%Te1wF;c~%l@KqS z)j4W*j-iXBBnO;UUyUI|4%2+YKJhU#^sw!a8GpTIzQ0x}ttAO!&z<4Fqko+TrPQm9 zmjoruOEyi!XoG+=Q1drd{*`7!0B)|&pMvcFFt-N*Cbk50Dc{%`dU&C!ji_J-95g95 zLmB;mxL`j{Y643RnRMaiiB>E%;AerE+r#PtINlV!X&Q1`sv2-Am&8~+r`(*|gRuRO%#FZPm8}LksfQ8+6=X6 zxBS}en3$;)1Aoo@uMRwuyeJuOh>lxW%NGSy;M2HIR26qD_tTZ))Z};e>}8QeLhB}1 zJ6chkT!eXPMc~U7X}z(tSvKqTyd@9c3w4K2#ED1i-7kykRF-N!r3#1f#Q}|G3=Z68 eJ(D&}-l-SP@U})*X$#~hjKsYLpQ<}UF8|p!N%ZRg diff --git a/crates/j2k-native/fixtures/multi_segment/bypass_rgb_97_64x64.j2k b/crates/j2k-native/fixtures/multi_segment/bypass_rgb_97_64x64.j2k index 10ff6cafc31c8b08b57440eb449e6158f6db1dbf..1c656db303668a7dfe007fc1bd80f7764366cf4c 100644 GIT binary patch literal 32840 zcmV)QK(xRAPybN>F8}}l004jh004jh0000000000004jh004jh00000000000|x;C z2LS;G0RjI~01*KK00#jC1Ox#9L{(;YcmG@f9YSv)ZvbxqZs2E7XHaKoPz6v0P)AqO zSJPKw|6~9q00BdCWnpw>WFTUBAWv{*Zc0!^M<8}(a&u{KZXhx)H7+#&kN^q*0001g zx&Q(HSOAd#1pxsI6bA+Y0RasR5EKCc0S^!c4gnDXA1@yy0TB)#ARsRX0RlYEOgff+f8M2I$!F^YMWTmga|ff9)ek6VySj#vMaz>F)< zl7N8mASri>+6Oglc?S6)RRl`d7GTldS7Eo~HAYotO4Hz=y>AU#kjhtJ&q2z7fPld1 zKsB2ueouYas_jbgKz640$s16~-0a--P%^<>HjZS-(M)=a83ENUG7P6>`$_QjdIU4K{jNdU$c zo*Rd{b;S?Spn$;Xz!s->8L|3Ac~=kvgLR&OperDB;25UZLzcj2W`O3TCLrk`yXzV= z-}RwTfL1!>^p8_UD*rP5s5Nml2F*YvJgcy6QYcAv1mkls8}{e^6QQ zV@CL^!X#Xb{~JMVa0`cj=J!=AqsGJu_744F60nCax62#g2~t?8kg0luw|Y+Vn(~goEq~Ltk3qujB;nES^u(FE>Ohw4 zP8AE;8V11~0O?{#u`#{EgjLuD2;6fV1x%;o#Ni%Dn%mdEo#1PK7kqS)A1r!2eCdiW z@Cl#6DS&7=YET>9PX~JV6;i(iy{gGDCQ^1BQvhZEe<>cc!I=d_6#zi~*vd8%xtg{* zDPthzn`pe=`;d(?yzN^J+G>F7@xbvE)&~Q3=O9+5N+6mw64xrk!*uDGpmltA5fM-t zMTIPIJrkCJvfOw|y$kSNnc&AE=1g_5=&pM@f9sEYNLa~u&C^YiIAKv;=xVrqI$aHT zv&0^_%1EcGjyTQoo0VLtd8KX6H9VD>t$RD}wGv!) ztt1jg2iy@p05xZH==wIzeqt7ni%ysKeqA_EdMQh9==bh}8<=-&f9f}EL5{Hhdms)t zN>V%6jXC@jl^k715pcBetTmsvl<_drv6V*KULVIr1M3OQiq;^{&m&b^ZSip*i-_G^ zQGnwv!_b6)CW6K*5jFH^qA~Cgee*&ps{emVlUzV+QrHuzo5DguDS)cKr|?>pAar>^ zc+Q5SVgG+^-^iS-EkQ!0aMjr67f-TOhBsL-K>aNXBDgKZ|9@{KR0&M}Vy z#tYpA_w^+-=QD)2)?UmAgXbXuL#BO{35}Tai_|U;06OUZFch*j62ZF4H<>aF=P(#5 zHXhPz=Vj9JQ+aPDw|MMm*YOj&yryM0Ji~rDRWhi}u|xN7zQw;;)_)lN!Uc_Zt;NVb zv$BAg;;!9!SbbaGXo@a91x0w5l!9Y}D+}IW#751P_RLLVG5EYQT5WkJO+SW)QY+AL zo|3hw3bj?1=<`!|EQhNPikocUXH*Is44urM=gRc{Y=k;ScJEPX!b|W(&JJ$`PF(u% zpC2ng$f=T!?)2w{d>a_g5__V*Uqj8fz7VFD)iMJ7A_gQg{>)>hO?G8$gwbg(z^$Ch zAvNTQ<2~I5d_7;@z@vLjGRH;{;|fOUIvK6dK@ImQ^uS2Ez@zD4k${T|I+VZ~5>e=! zD1xER5LrD07J8)N!GiyPFZ03k2#prY7c?i3Lu(WNc+LJ}UozL8QI$(f2Toc1BCIIJFf-BjB z9gF`g`VKw`f~19&HBSjZ$4nK%-$8+rFPONMsq5DN6Iz$@_^oG-ZO%XD%5~1G-uOqB zRi0K|X-s87RUShz2F?%sOTSjAjgwJ*6^#S>xYAu(;&y+Nd=J;}XxZ|!l{%aodQ5kf zRa4cBO-R3;sCK1@UdGLhfzQLNSKrGug1vPwP76;}irE35a--{UbA{!MWLrYF?Om)` zsQCKEF1D-Q`&CrCu~7?LD7aotmc^?Hln*H&!3#O-<*#g9Tpsw3v;ZJPGTeMiAt{v6 zpMO}UhWbKP0=~aee?H9hOEPa2xT|UqKF(1@>1cKxFK|Fy+VE))$Pd|VqzUqD5|f7A zxWhGs>up7^M?Z2vpnx5>9?L&g;e1t^$}LUj4`#jZ-BDnp2!c$4RJB#Be#l^=)noyc z$+j~hsQCL2o0s+aU67YA+pvT1Gq^S+m*+(p#2%12oY9of}Q_=OWWVO zh_b4|)2XD?%=UoPDJccx9g8y;P7q21z->)c=LkV&dtmZqs^ys@3jcrg0Uv{a68Gq8 zm&LVUvL_m>JUnxTIb9u9v1jDxP-_2wQL+I?mEMfUI<#|H9Nhpcf(6#tHVsT4BT65Y zJ5u@*QcJKWp(fRy=3dc+N{c`8gaRygR-ZjW_#IfNMVjbbGo=N=yaSDkkguZ&2@RYn z!%!MSnALg#A}vh$XdvVsymHTExfI*h4HGOb&Bvo4Ajn)$J@NKPtkOLfX+LOir~<-k zk_o#^5<4ejgcq3>4ulRc{7l8G>D|8jU$Mp+r0&fn)gd&jsf(CLz>dT`M!}cBM}oO# z-+!gZd&Ccj5>(D*p0s{+4fCpi)GJ@MV6J>fKS-~2*YEA0+#LLQ(1;{$DH&hWwPU|J zwc1#bcq2bZzDt(%_a^5+65dd?H%Wg3;rQ42NITabSwJt&R=SLZMla9a-j*Ftpvvnc zL6e$i`08Z?l6L^UQ9fPtoi=t1Er`(8n97E=Bg`+SIf}lp<6g~|DbSp?PkkIMX3&Gn zo+E01k>Y@%|8)ybGs-p-T@_)_XYZNwRI)hoLYQP#WibV7b6t};;6{3QYV+_T?d*65 z;iDNyU0F;Zm_M6tws(PB*Z*i;7l@)Zc2`HenF8~FYBEORE52Lg<^p;n*o%#`1?Q<# zCt|4%36%Z)s@6Eq811Ra3i|%zOdeah2x&D8fbl)l8;U@g{Z(-N3S4aH;`GaBB_JeP z1b$nI_m+v2PR?HBbd0{NQarv?3&_E+cuE?2Y(Og}2y$!)4)F?qqSC`J2dnQfBz9MS zy@eXM59XV@#(nN*rya$EFmW4Vm|igO)~&7e;dnqnhM643rI`N|HeM@t$n#F8C99)a zAi(R-K}C9kaQhS^Dr!^f;+meBWRJZ3v`GP(e|O=6L4P;a5_= zElmKL?NVn#S8z9^Tlb3mc8AdkN&5|L8>uuPFa2*qu2VOQAdvC0JMzVLD+g+mcu2IK z0NxiW{FCF7G=4$cH~^yzf@tw88Mz`z4$;jyLG-x>?^haMe`<{wYqgz1#yOsMCiWsk zh6_X|E&Z5b{uLLtH=uP@n&-fGT$ri(_(0~k4%K!i8c7~gwKJ3vZ?x;-g6^csrd{>S zcD5HBXLL}7vD6zqVO<}liiHBux0Yn%Ja^T42d&XdY(F%~&iHbGkqG0lABza=le4Q} z*8PLkf44ooF~+pt`^0un+}l{_@;U(4scp+(Vqgtelr(UwcYI=)REMFQb>|N4AmrE2 zvi^t6gBUuLetD(U5Bs_E5A6EcLg5jOmY$FXIiV2CDJvx`B>Q`%H*Wa5mPf=eZZp4V z9ZWCO z7e!}h7oS;7Az-Qrvv_FK=0CNMC<*xfaBjB3SemK;( z|5zHhX4m{Xn9HiQXsn?u|1%G0zQd#y6of&UV@CpjNJAT(mE*!r6CQZSS z{}9-AO8O{f-;I%57Jlmgc#R3osbLa^67`6bz6lBoSRUXDtweRQ+nZt)t#B_LS3ii>}zlqn%cDQIn_1a@_vSL9r>p8Vd+3X#B?&9&IOk9Qb;ad98G=@`P|W%x$kFB=VBKfnlU1o&SA536I+(Dd2 z-snUJ-?Y0eS^@Kd@pKX1FM=_~AVxqUgeWi$Wjtt4Di6)3D8Qh9Po3PbKPj{f6o>*s zuR`KCp$v|DEF@pb4L=$XV6@ml6^~V>08tW_(CJ1MKn1noir+&LVvGV*04Zo+AQd2W zPuvmmz(hd`;&NxI|9>A^*m zmrhU-=K-~pUanfV_^)tH7O~1qr$+lQTNTZh!rwUrfVM9{D9&5x3AWcBYIc}EYSYEk z<|8bFrW;O)_x-J^FkwUL#(8|q!C43f_#o#QbEY1~&XxavqJ)1Q5(78NFoxD^BZ1tO z(>^h{5oJSYk6Z=$B<2kXM8A z6rTTo3J^^|JM-IfL`Yoy3EUYSj=lDyl~S-w*vEIcw4=QmNk&U{=o%`Xdr5R2Ls#sYXynUhj>p4P`I|86T?O=BMk`KLMwSA(wZabk!_(% zV3=kv0)i6pUJp#O!#Z$M%LLUhb8afzkq>Kk?viHlygsZvU5s#|nMv3S}PAT93 z?~a!Fai*@1Z&0{aH5i3#yBLy^l#7jlX%m(ZOP{*im2830e2vMmW(vAe;XF)5nh%;| z$#(Zb`$D)o0D}p)H+d$j?HdP|+q!2a9W~zJTM8A$1(%{VcY=upXlXM>MVUVkD~g$u zvBuxNec8?5=^Rg5*(|V9`V1$r4|(J3P;K!O;KEqAG)Yrvl2Gtxyv)B5qZq?n`PM}^ zU@WdsJgA9ciH|=`3!TZ(cQ3V;5Q%v`eKDy&eT4 z6d=@o-1IBr9>p=wpj=BEC{(?yiN-Edcr|90M|YS~zMwZ=xBNAug%(1O<6T^Kmg)5$ zs+%<#K!K=AP-W%-Rm5Aw4L?8+Z4~urapSceqlE)Y)Cf0cVZIT~`5m~!#w7IR(et*P z8!AbwCif4g2vSy?OzHm{?Vaj8>aDdx$3Z;e`-cGgj$M`4xdSERNmONZ`UuBikB40L znm>(YH0J|hWsSITlehP2m#nDe;v~9*KVY=p0~f1~2G#cL@Os;X2AmL00CI#z#+%U7 zw5FJLytwg$s%F|+J&b&UzO5C4#;~t2IULy(g-^!TcKolbJG*bz^t<&gmgC2Wj7Ag% z==B+tkY9a@>*SFGIMo1r0*x$pw@RzTc|Oi%6V$wrHD#J2W)S92g|V!XO+dHb9-$s~+~ zLOHNT2>t-`3RfWXgcE^zUQv`AS-k5V8~}luNbJD|28Dzzn1b?2;wIURXhGG6+Iu#r zwfZ4X8yG{wJ=QfmRx4K|eY|afCRY;*VE8j)s{eml3vl@G!@_w@rEFRz_$QEZ_|9?$ zC+54Kn)#5~jmDTA{vBHTew7mam~&wD)`01dt-_wi}+g# z0L5+t`sMyfC`{jmZy93JRH#jYEBzH>@isasMT-bYlMk6>xONYUPXrZ9!@K|6mLX_V@#_5hK;ki(<1g&yE~8tE>uy&Hocp z(PkeLjprQs6PRXP>qsF@&MeyQnKF z`9IzySoDQ;KZZyenf1!d#XwezQ_MWD+F^rqMgJG3W0)*`L~4OC%PzFyd=knSkuowD znca|MhW4g)U0l=wKuckgxJT2S1H~oSME0~~EL{PEoBwp8DugcBL>+<5#oWy(SWlMb zhGQ7EzA%1y3bS?Bb3g)<`wAn@42^BwkC_#}8Fws*XN>uj71)rjpWzD;io)>+F(aA1 zghcOyND?6(=W^_T>OJI!kt;!dMl4AK-A(_0-dA;HUGZXmp4M|H$@=yQ$prQsZmi1u zKE2C*DUi~RZeV&PkNK5wKHtU%2s9lLN>Jqe0n<6uD7OK$R~UorQKT$r#%#k08ANO# zRE;gl;bBD+&AAe6w@rCj zO5sW{2SAtU6D2{#trnbH0B95e6L7$tBZ|Q(MzYnX20be4WyhyFk#TdH+~6Z*u8>8W zMQ?D}el_KlDeukm zcQZ(i=xif1t^_N0@O!U>I;EWp9Ebl6(3+u?SgI%$GN2oy)bihJHpJ>14xZpAt3ls7 z?STCmV#1M~4GHC#IzwiXSo8GT@2s&pd{XjdS1@OumCK!(oe5*}A=q_7Pbfpjf8 zq?kig;a1-)=>VFhL?3?Qs!~a=Gaol2jY}e;UF4A7tH%W0GV^fA`qmzwcsTkz#$L#< zae5SbA4J=uTeAoF?Jf9j0ZyM#K>wzogVO0TaBF+&3n^h9$l{=i< z1h-r-K+X?cSzErEQ>>UK_r66QL+EI+g8Ka(@}T0kieNr`Y3nHg^?^*+Pai_dE#PB&yMsg#7)uz_TzfM2z>_?iVDE|W$J^?gCbXcqS z;HmehX5Xc|%99oWT5~xmAuIc$rOcFr+rmNqmoZSY&fTwEOO)L)>rs;S5YmP&2I+$( zw%U2o)ND5!#JSW7-!rU#KaAyR!}eBXHb9^jzTH-kw5;q6Nj~TsfgMw>(9I_2RQze6 zL8{LBK5Jyo;&YcN*y@M1K*{A+RBhIL>T0GiT>)LGNo-X8d4HPH>wcS~I_mRlja$Y2L&e$+VlbaWO*r|t<=PZ0j-@|P_Zc^KJn4rQ$*S@3@@6) zX2Et!?ZUKS%GABi+iR0ZUV6SJ5-D>y;+lZN=`rv+O=o$G$2w|r%OQA<_C*N8sgEKD zwBYFduU+~tKt#O67m?S74K}SG0Wj+Bh(3s93k}X_#l}!>XlkmUO04b~A4q}FTuFiH1^mm!ij*f$SBX#6 zw@^pq;>eC?n91Y{>-9eGszzN;uf3TPhZfyOnbAjYWj$Qj3UdLma^OD2XQD@F{#@7> z0fpHFn~Eg+SuojNWM+actV}K^=~j|{7H)Blx?5W>4zoG88vT?#Hc6oH0d0L7 zflk=KNDPlH`?9_QV>M5{(lOAq`MPjCp=}&SM#rOi{N6pd_MLPa8kl}BX}McgC54}l>UML1?T52d3cd;T6fG18uG!V-22bMH=(7i z8-1ycCT5t$LtA%=)S;m9yp9MYq1n%J&ZLb3L`2@`Jk#=Xw!WadVVp>L6>6% zHTt}v%$vpy?;b@#1>lp-(p3cHl&h?yi_S50OMiSIgH`exq3j;-S8RK#mknaK` z2Ux)&n=h=IfBkW_pSMo!JB1#&qH?I|7apQ)@w7p39t+s-59gMSdoz~H=Qk0-z_D>5}g(2V)tVd#!MoN`JhV(f;DBkLu9dp_>F7q4cIjffAfd z8+Q(&8v0tq+e&NB1fihL46|3rT0rl6| z4I$7a#NNCaYNeIFb|YQm0v$!~aKG+2JE(}EmCEq0|qag7LRESE#%6FmC9ZP841%omglIS&=7y`jC z9fX}GAKDxMDpR^(JP*g{K}T3wwq6tE|9`!+hP1Pyl-%7{V7%)GpRHX!HH-JkZEn!V zeuLwksyXW*#fCCpK4tG{gfM1Rd(yANGLi#$p@ddKkh_ay;Y_$aA+yx+-C6lM!2%s6 z<}>)z|6?&>+%=~JbJd~7Vqw<+O$(T5iZCQ)LZ8WB3%m*KiFEa2SOc9G@UQo3Saez~3$_FGHSlZS0v-du$CFk9>*oD8 zr3Z0Teh}Jg2g{a%9&hXv&HcNjO%8R^2H_#YJaVXl@8kIE;M%?=TbjFh02I zU&0j3re7Q?vn=v~p<_Lr_1kDJ-cb)sL7u_}tCnK2ailKV2r&nJXh+AFNjE;BQ>4vO z#B0RI*h4*H9VFn#7#Z<^B3`Vd--LtL03f!O{q1CdSgM8RaIUM_PDi#_p6*K)2JZru>3!4V`D&t7*1HI-5Vx{)3uw;?M0F6gH^x)iC%&04vUIs6GfD3~uL(CW$r2?#6?>Sw)-xgHCy=pI=s z9~qITax4%DHF>Q?B|$ufES|#U)`O6Ij4l3fPnjUemSIniye3@U4HP3C<*ivan$FDNoH~1StP4 zoG89K?RyMdsp35QSSrkovd4?*QCGXU=@1Zfu*wE5f=g-XM0#ZZg4h}18y_NdgbnFBB!*JZDxgb71Mt&3JWhii$R|&c=lv0O%pfb8WiYp z7%(}1wqw9pp_N%{-Y?HhvbMeTHHTeq_QLO>awIHNhuUTTTFfNv!aPI%x9xg`0`=saM7sYE9a;admp&$5zYY{hHF*EyLF z!i*629CvCeA5yb0qA6)fj$Hlcb<}7$?)SfG=Ni6jjs2JnVf`(uuQwxyp%@yFfyl_) zvMz*mb)_7oV>qPQS%NPDhi9u=uvw80!Cfv_H?W3+r%Y3lV%~9v=~Oc-rK%&_rFz7$ z5TA*fm{nPRNj?2{{34hff*dDw3MYmjX3#iJa=V_Sd;?pA!7{)c05|7Dg#Ex#S&0LJ zrc;VGY(a4TEW-$;7J3oXi7^tg5bAHHLbWNeLtToVj%Es93!_=kcm$B+A=sS6}&+x(MBPMsHxmBO0Iap8cvZO20-VGd& zSWQkUJbbL4ujy>$l4XU@Cb9;;`{V7hh8F6}7X@C4i?lGs5`NdU)PR8Uptzv8p#ZQ{ zoyooeV}bv9{mhT+EMCUQnq402Tw+_PN;0CB%eq9JN;N^YGYAoGuA6!V2k00Xr1psp zgsyL~qE#{+VuBVrkGh~QbH1I{2xsN9q;+Amxj+?$;uQ^O>pAJvjftgx4b8;e&rdmZgHHLGE zBY;8=&TALTO^`T@lXTMev*_4JCR(1+4vhulq_GSbyrY8|M{7{cOJWQlXlL;43b#zW z$@YW>ij$a1tdxk}$$1H00ZGZPvHqsp3E%Gcmh$L~{J>I<;s}^PWf4BM;M`{ZJ=FDU zkI|ZB^D7TAQ_&X3YzcyF1F5&s1dvb*tu*=>R4P$)jJ#t;D2+qy(XdRZQ@J2Yw7W|q z&0k>vvSr{%k9iVCYShY6Z#Z?u*>XHr3JwZ^%>Y#{Dq2{;K89O7*iJPO*(4QJeUx_V zBswG6$__5JzNXhi(+^DMUaKB#>veqJu1^Ja8zBB&VZ^-c3uu^5 zi{`RALl6i6i+lR)P9cB?kU@59vplb~bk+@au>;!Mz$CfMBgMP+xiX`TREf&3t@ z)-WR!5jKpoqUGv4nI=*39UknmV+lPCdgd1K52J2rK{A-X`&LA`++R`~Q9 zO`>fbFsL-F9D5rI5pXJlCFMNV*yB}7B|*8HEM*jHUl6EZV1U@lNtiPCSyVEg`lsSn zcJ{2KVWrr_vG_vWBV1|73LU~2-=1Jc0vB@3zX*)HrXfj9WFo{>9`HG@#@7Yznnq{8ma^z6c3&>n3H*WbiMX)~}n_wu-z(_q4Y?tIprj?@e)X zx-THa{X)Ea*YUO;*}~yNY!jgNclETrktkFQoXucqmQFdriYwjp#&HuKj7o(H=f>nu z)wacK{QU93N<4VcI(GjA1av$tRQ}JiaV|+hiLo7DlG#bk94et#XrRC$?L9z=G$B=& zwb#De58#k`i&S7aS{y6C|nyKCjH)m1>@7l&#PcTtZPym|Xp$e1|$U10g^FUmkThud3k7YDV!LNkb=2dS| z9)d3@FqoJ`99+)0I?OzHUfU|fx(YV+hoEryAaXuuIrv6$j z?-4SzRrfc+D%Yge6zgQs^p0pLbEIKakL=AYHw34K7IE zYu}_s$hjoZ4d4QIf8+;1b@9(w`lIo9a-$5zLyo_Dq+gdLXmrjSd2KA=g+RX`U{4xM zj?kf)$}LHgPR;}_dlHQjT?*qU)W3^4m_58*Emv&K`0eCIDoC$kEP}@tOE0sRHNei(Iz1}_yoP!N*uWS!rnw7NYC*mSHV}9NThyM+2 zQlCOdQwksAEq|84;4))bFz>894ajB84Yv#dU!4zN$WgZ1sX%&m?1zXh2!UBGy<-dd zd!0Sdhy>WHUTkrD2{<1bTdtEVrX6?ZiG$*dW>Mi*p;`pPCYi{kkC2{HbS!W% z2sv|)eDag{lGME;dK`nq=k5-B=t$%^Rdi<@!5VykCW_NZ!})8EI9rMsTKs*pUg^}d z14oo{B30W8%XaV=lK%Mm`Pq)Z3qaav#~<8+JHIlvhdQEZ@zz zzzd)=SI$~VV=$MPS3Gb7yd%uS)CYij7;3sN(N!_j-yzq-0R zPfV!T)CL+S=?F!1v`3e27naN@nN~IB+U}nFNQfA_T>SCkka&$YYk0J0S^1`Xv)jHbNQnwZgoAI$c zH@*q;G^>mn1`tIb*FQES2ghyWi7UcEzGcMGg8(f)dt|*7K*^-QNwNTR*VG*J2}q2s z`)NF9=uf&H{;JUc*R>8w?d(~Y{hW?U;snpdZ_7b2fN$kb-4fS)0IueFcx!U6UDj5S1KmBXl0GOSaE z6+*R_#ZUd7jWiKNL-x-ZJG*~5*ZMHZ{CSEvMVM$!`W>h7MEB{N0;#J!f}5y7@@AxZ z)%V$N5KU#W=eW)kG;7AXa-y-7nux@RIp(u8m=#o(>Ng3;$^TAGg1!4eNot_clp$r` za5~~l1%Pk}RzKhJ*%XIcnWbp5)?-~woc@n>cF6?b^wb?me(?Mpq8&HumZsA32J)=L zlyK+{k-Zu%jUxI(!5#UgAOZ;ytmchm`{FwtILjEa+CaUKR`yey%4qbNpbu%Pa=nV> zkG;@mEr*pkEP$83(GVY|jnoc}cJ5W&DL4n@#;QQbcoriVjI3mXSBqL$CA5cGcXJKz zIML`RbrDdAuVD$*_%PmOm%`H&06YU{ga8$`QhUof~*d^<<$Jd7^|{gU@d7C!*u^Dfto{ej&!#aWb{OYF;<7ZYxz4Bv&COtI0dBd&OWTh-OnhZ%|Jj zW!!?JD5~A=Y{5FWTkp$2QLqZBAB{YdREO^~)Bx3B0-C+;Cjxq!l}Hf6!gZI{?lPi1}2J(>(R`#$znk4hCfg$ zD-q(rn&VwR9{6`vp?ngo5Rd1_SIIjy_)cF)Cg@iOfcw#>r@Xl0BZv(bQ+CgBZk@}# z=3CJ9y;e3zY7`u*^l#9M{Es{jmD{KiM~5WT~ih$@$l(>)Trl}oOwpMUg5w)X|^8#f60AZh3B*x z63dj~-{oi%-HC3>;o)lZ3QDupxO;6gy~;t4J*44ZO=uro%?1RH-&UDBEEKa0>+&6h zRp={EMiSw;obrgGU|OUyi&yD1z{-}MWPnNi-o6Amp%wRpyHIL4K0{A#C%^=VMgJnD z;U`UjC@O9-pWAkhX;~A8O|84dcreTf@qVrRm zo-TfA>6MyDC82GbH0=2_oG54Ux;naAn-sE=g4@=~B#D3;Ag=I6PN;|O=)^#u6XIn; zI6?!x>e<~o+!Z&2V(Q-8j zVvfx`KZwvydjz7M`?NO+=>>n~Tbag@UNB+5&6cxT)^J&Q{kH$s;hA5cOzH3`9yqeuo zWKzt-avfmOv!VN@+xWQ2Ump-Ezp2TI5^9_J=K1ROlX^dmUq<7#T(>f!oqOG* zjm$StS?LzYaxOHa?0+2MD!oTJ4{qKJ7)V7*+1Uo0mgLL3KBP}bA=ITV>zu4z2vpRD z%9V~1US1*RK-2o^n!oEH|5dCVh)_BXF&McXlck8}f=|Q$VCfq;)n;WG!&wEIN1Di( zmeE$>=a5;>6{_d)-BA5-=3l1-DLiciBGp`cvnL2Uh!JVjU1J4!>Cn$Xx`{atJybD- z%7O!6b^i%gAwsCe&zyZOj+xKB_1W0F>%FV9MSQ}4w4GYX(B0S>lL;NdDystqD&t5n zb(Q?HtQk(msTm;rSH-Ndt2m9}!Tehhh;+;vaP}cXD3@KT9NS0v+_YXUntM zE}`ZTPy;E?e@V?2$2poRAbBvgohK~UclJ2p*R(BSdo3=lM6DdpR3SJIgUhxu*m%F%?fQ^WAB>?kY$tEc+h3GiLGKJV4 z_0-I&h;EEIbD8mT0Gs%_fQ>v`z(;`cI2rJ8+)4x}HCY2roU_}gK-z)=2v=AXvnFMd z>n>?(koUau0y%Z1Av0AoW~cM~F@UYnTNy?V{!XgctM@5KGf3Tjf`{G3a~9xCMK@GQ z`wT|t2Lm8D-((26gK%5c9@nB_?GO3Fs5wX^qckE{Uo=G`JQ!$DzaOj!e7`c&3EmuPpP8+q9&_DnZT7+zAJ;f2lG(qC3=w$v!Au#ZdF*csy#`CW4$hKIP z>Jk};JeQ{KDj5ennVC!mr-$Yyh90xyxJHjTP@~b;$au+rIl2X>x7jp9xfm$y~ z!=9BWyyo??JA}mm5X~6_|5?t~$Tu3JVgtK!5Y%ltq)#iAn@4D1(L?=SiTp@qhFN8) zPR<9O;4sYKx(_}SM&`Mz68%TtmvZJR)AtNJ=5QV+J_eO_WO*O?fJ9wQ>ZS8l6I=Xr$e!Og<|g4d=el!fR}^{L<~X2iI7vMdC(~s@pq_~z+J5ih*--( zz^kTEq}Z@(E7qk@Tpkz=c@y1)Gmi9Eggd3aXJDfXG4lPtf<2|uCW}CEaGLPHL3DjF%Ngg^Tyx`_&hh0SaR9~@I_!-@ zPabTv7LgIrTnf|xzkP}eP@$Fa?Vt?=pG~(!`rw^hGM^ou)Mqk>z{-=kLb`I&zb+^T=`ad<(khrXcqR=66Ofd<*T7ZYl^^1Z^2H z3^~R30E#09T7^2KmaXa)GcJHWK?49GF~fuS1EdT4Rxz{R;YxyYe-K{4Ya-z1VbF<{ z`eR8TRXRI2M1TM#Fd)KQOG!)pM<=k@hGc{8}lYs8JvxC1FmWDcSfdM31x}gavG+N z;|RjlrvZc_CWvJyKsi|#hV)U@M?i+p&t07x>q~`)4V{b@0Ye^DT)$4Ip#>Tar^n;l zw1|KXj|}P93*3?4C-)=ue6Vm(KgTw}%>7NqsB~~`p^2*?OO*V0qM%gH(@rLm9FYJh zS_C*TiPV%#Jf23y@=Vn8$u2_{>rXh838>!9-I}`+jnt$%1u!XEVDW$3mJHqiVtMH7 z?ML=+_qlUIvg(RG+9v5blCW;kpv-BYLxvWID`z&E_9WSMFNqI4)B6K*a@ppuTcvjP zjt6bpo=!gZQw;%7y6s*pT-=|Go>&J?+`0Dj{3aDiegG5ny58m9Nu_LE5SX0d0LoSrwKdWPI`bmm_0$^>MY$?-FSyDG+=*TcY_m6kZsaZ}*uY^* z555QMM;+tK^l%m6ROyBD1@Sa6Ow5(OIe|FI^>$Lbd9>4JcWO3}+!eusMj{vMqV(c^IK zis8p-3!M0iSU>+efp5FxYN!-%6F8rER<)|XruoDzJ<9NB^V&oK_PRwahP4+$o4zuE zlneBWeEzk!Fb1LQ_X^|%Cwj;{^!c&w@kI1FMjL@y4~BuCuV#*$eeJji5tOe)^Epf| zD;T8Im?5iz&3BZ|-x8fMmS@B?zdziITr@8-fZp#RwVvph@!Dd56K|0j!%m{up!~Z| z@ikGb*<#L)qt11Jv`Al(qeTpY5G-Y4!5FG_Hb9z=ZCMj@6Ezm&N!4>YZE62L1(}H# ztok*;8fBSz^ML17KR{Pu)@C*cDc*^tAJ59LM(p>TegfKe@PGskHdk;eqij-CTBi6Xc4LJgtgoU02HMFIb`i5gzU#4|Bcp`i+2vR~wS|R`o6Q=~}P`G zHHJU3WuO@$eY?_7#WX`)1)WDnpw`uKwofkDc#lSatKwkeUH0wsZ+lh{sV_QMyfOEs z*Q-&-ipSM~aX(U~6BV!TdO*n=m!7{5!9m-y0i;+6p%NiB*`xkZRdcs}nWgEDi@(Dc z30;&hw9s-uuq@peTa9x?f_hycYc`tdWT;42Mtbj{DIGccNEkz|&R)WsmxFA`v7_n% zpFioNao@i=Qx;(pxe`j1C?{sxEtbb9B|@7EydI~kQOa^3yN$JOuP)W_;Uj-!;IAy~ z-l$O6C*JT}L4sflN$oT@AqMY|DtK3~_Q66lP05vE9OyCYhGfx^AA*U$TMJD{C{EO~ znLy11rY?ytXHJHQ9yqGA+LatW<6|ymYFs?Xsz7=fMv!`!R1i$!Tca08YhN0vbDy4z?a{#q#{%WyXO!9OnLlP6~z0hm%cQwM{` z5d8$M<(Zk#_1BTImOs|r4v&u)h_S$imRJn^l92+WJ7o=~em^lRf-Hg&f=xMe0I$_m z3Vt%sG#-OvFpPW5i@E<8LM&&0lBZ3&s_=EjINUDnZhn-%4T4;=O%M~6xi*RqG=Qg? zdu{L=`QMPdM}FM$^;dY={8r>bB) zfq)p+L1nvP4~1vfSnXCGFxKgP*@+~Z(TAD;g7ZXe8#imdUK|p0(dSGejNVT$TXwLw zQ}8^nTQpj+rC2Mlv(Ca7M5nyVpTu9`mCzYka+D@7gJltvieQU7tt;89uaZZ$8 zBW#Yr9eGO(&Cn^Ys8RO9*kcP2Ob< z8JN2Or}q$G>vSS$$Fz1C_jE%_2whi|phx|TQzs4Bi;-s^Tei|(BAWPA2|&`%+T zwoj?@GFCLB9zv8vK!a6&qwK8Y5QPU+c?;4pz!Dz7&)W0~k!kNk05*ca#HESBj3K5; z<}0lrDe1x>TC|%1Dw#@MY2wcg56Vy+Tnlg72^m*(BxjJa@PfL8JRk5G#4iS8(nvFy z3C2v4(tyt2V~sZ>yfuK8pp+nZYUZ*q{ct*=V>Gdrvkdoaph>BT)pM_U86Gvz9xO4grNC9Gnym`93sifUzym9du_c_SZK!(XdB{Qs5 zS4MhB6b7R5?_-gGLe!sp{lCI)Ovc|hV}Rqrr2zdUA98Er_dtD!>v1S%n+S-?O>AB# zD4Q?u?wj8w?zL}uM#y^!lvRZ zppGJfc{X4LY}TG6VhHU&4H;@a(3X+ACJ@BJgs}i>YwgFceWFxX(crDsKw-@|`vLjZ zO>nCjRg^E$JWSCV%FL&tq{N22SW5-Rs>`r zi6zjLw7pbTJijAia0D?w%_{8SVnFQ#d9mn;1S!QUru$owkiqM0q(PrwjpY3y_gs>7 zxpLP5u#-@-!eFqoaf<(UEi23iW~Gd3{Xk%@1rI5W5f^(n*+0t--oCw}Vsrmh9UosNYjgJE;=IcNOPHkVGNH2o9l#(_ z`E4t|0oW1QW~bA#))vGU&n^sW!SpRa%Nz-t+c7YRDT;&tLyf4^+~+t?2mg6PA+}jm zL9H#kr|kV2U!Xco65YrGw@#`ue7~olq^ERT@i@0%?2iogLo~uP`w1d#Y5_ri9M1g@M4JS+VR?KT)taj5SIP28%?Xcw=&jpavtk zL9S(2{~Bas=^ZkM^fW5Wu%1>8lM`U<#HlIk=%oJkYPKFv5h`Z?PuoI zQTJv;U;F4Py_LUrA_H7sEoDZeo`4A6uURG-0o%%Ul?o?i7=_YD6NQ`0?8u9l3EnvE zXb`N*qI$@NM=>&nIH=R6xgM@!72e~QQ)e6taFLkqBgLch}18CuvjS%88@_19B z*J@4u6nB@aOb<{ojPwD`O7ul46&HL=`(j?G7D(>9NX)@B3=ChG197GCi7#B5-@K>0 zruSJDYnGW3LU>WhJc_9*M|uvZdaEqVEi$hRJ0)*0*Vg!_#9F(nNGs*EDt7LPV9A$j^!3?ie1Dib~TC%VBA%`-QEzPbF3fBY^QeCq6DDaXnZ~FpdF-OKE}f; zqaTG5pTl}rVnfqPWZj;7ta^Rmn7V)vfwL|4n|ibjWLLfGFoFmP4K&2J1TuX7=>4vG zQ>=rVsKvnN0udw3bMzE75tCqyLsAD`2NH$Dr&dsyIbzM@O!19*leKat0J7;w=yGd~ zUKqmC+w<5jmj`WjAyi?PPL=H(u-g!G|Hxxcs&0T z)FLHiUr2^b$VVn|ndxZOluQcXuzD8TE%_KKu2AJT;#5dx2j--6Ujd3I(L|~B{3>cu z^=YgTk?w#k+HBaiD{Wf^X^3a%XiqY-qCkkAE|i=5)SMM61|?~ddMu-B>ngE_5o7%l z0oiZaUXlCpn!>mmrzD+S-xoFJ@u~Djdt;nYTm)%nmj)z5&Rg|SzvdQ%-|v7ni4Fvt zJ0{F#=HTKH@b}uVx`LTCf;xc+fpGJ1x7D004;9g${xcoXHvDlx147p=bZ$pnQMXXO zFK15C z4Ddfqo)5N91N7L&c9OaGzm)R1Wdt34uLD0vg5iaAB5W6NweTtOvCGfXJpMVSkL0W) zY#0;pA6Ii<>UPyMt(#QAhcGf8-IW~ht5X*kvP3HL;!wCAawV)rr+Sg9pJ>)Toq2J) z4-!VPo;+p4)3Reov8E$YZY%Lre)Q5K+{n?pQmis3U^!=w=PSU<80s)hDj`f`*rDnb zs5sLxg0X&}8CbYqVnnbDsOA<~dE`s`eVZt9He0ljw;f%sY6t5C7Z%_%`RO^0b3`d> z`QG*??(5p?gn?g_Kg5JCpY;hjN4$;0hui6fWhiS&YZ;)AgdE;4C3w zUzMo0Z@P?o>gYWMIFVn>|00XuIsR)+j?M$FB3EEzwC=LczVlNaPCn^T7;|viA?l-A zcbHhxY>9(7TB0J1{nPSm^kYe-KKEMKZ~uJe=3zp>ji%+7Wi`tjyu3}R0~jhAtia!+ zurv{@^5aW3f5TWRl0e<$Tci;Zy8;bPR*@_wcn>?N1-f;HNMEr&KaD!KM1p^)%Aj3g z_Y6DU-$I@3Ic4VaXMS(`Yvh-hGsrVdVT^id+DE}FCb$B+ zw{MO=Qeeer<%2KW2*OEPg~W0A{!CU{4I5Tu*d~;vGjE$iaDUrt{?{mvxU9R*1I+d;pA$OAkURX&r`YCp`WiA6^OmS0W#6N`3}yco z%Sc17dk3SIk&Fs9%&l&<(v!uJwu^$XiZXRr!a4U?6Lva}$w*=v3Rl%sT8~E>pvLsn zGuG5}5mnudkA#mdKS3;}NuYVG$8Nghf>7an9Kr(3^7ehJ5)N3u_E82AEXzX6q}!Fc z!W&2mPfILGNr>SHQ1wrJdj|MR$9t-&@S8#kKmZ1d=-8uV2dvI;>MVuIcFgluKI!%K zX%2*yVpW}sli~VD}!cQy8^X4{Z=pdU$rUeV( zhIUbQ05Paxz&{WJ%FJ@u;Od9UW6t=>+O5o0md?GvNdI@WAYA~HJcfR4%Fm1WrHS;o zfJl$4$OhrK#(#e-A}vv-mWPfcpHqgifYG1G5oI5NZwb zC{hv`H8^wV(R%VU`d`9yKY~S%3o>PCbB*_6&)wWlDB z*IR4Ya-dYsKWp&;C!jD0!55pRkg7gQkhI3Ka#Xa;FA8pOec{9g+7a0Q7q~0FOh-rd zO(X%!BJikA$O1+5nxr?w%Kv|q0?12wM!)w!6ABii3!!CjzDZ9R6X2-(;P|Tb?XOv7 z;}l(%IAO(Z{IqvgG!{F5fg>|TxSa{>p&akfH@q{-k{THnj@$cD%_R&1Eb~wE0kAC_ z9vA`-Eqo#?CI2W?a(?B!kh@z7C0N1Ih94_Nhnv}QB6I`ZH(6Svr;lNQz_33tej$S_ zptd43z`xq_VZ0J=9gw-nm8cZSz4~HAM-fV*SR+VhWLcnghnrI1kE%XWD^P)wO)5fQym&n85SRKBszbk;LX1vVz> zdmDIHrB|KV2QB{ya-TOE;Q|AW2*LWv^}|*gSB~VAM62%(o@si8MKlP5K?~*CZeeSI z1PuWcDi)2$cBYu%pG;h;hjLrUENzTdtt)S&oo0QgP6=O{&_?(g%s}B+=5D0K3H0Vu zN4TA;mze>BLZk#Waj44Wgh2+b_i)g+2jFy9$yPC%2BglwR$wxDd*Oaee#)>JjbTH& zIHVb5T#(K|mW3-Q`r*(DC9$hOjgP63P>bU}Qb86hB1$({35FB2Fo@N(!Dj-i1v9^f zfcuC^4;$h(in5Dj3E#vuldEfNn)hd$V2&_6UoMcN+D8FF!Xh+Y-9F%Nt&c5fr__6J zy{N>k6NW9MkVmYwvuYs`2+H`5qYH4gbqE@k_EDyYT^EfO)Y+j*FTsV$EWCn1UoEwS z5HawsrqiibyrVF(c!<--gt|1uMMQiL^p9JI8c-j>l`{*`LiT3r)hSGoU4YKv$Kd}8 z0ddAe4S3-$Q&)K)kTadpo|AxdNKKWctOGA(!zN1*0aKhN#ZQj~6uK~ib-dV``c`YA zVpsDltls$2A>{Ptd6?pZ^+;CCvRZ9kEL)#m7yWQ}y0(@?q3~;fGcglCgyggmh%&7q zh6RZj&M2z?e-+@B1^E5U@#Q|~rVPb3J}tQV91~q^l%sV(p=QKFNOr6i+SB8p;S=t6 z;Yu|nsS~GqvJ*7i-pkxS=LddI(fV%hikY#kK1*&2g1|DHGX=io2@Z9Xd`=o~31iFE z;>>82-Z*#nz{R+I;vg2V@t{I~G|_-oOqZE2c$QW3dqzLdLJ#9>1SX52kCIU91h5}w zNe7ut6mkC_(cRq=fKY2Dft1yiI$=cv2M#oh=vg88spCo9;*&@?brnzpCb~i{Yv4Vc zRo_-#b!m|3w)&2+qEzHKey&pzWEFj zRYfVDCB={PL_e z{J!D!o^Wi2xrQ5ZC9D^PQ*F80sDR$>-F>Id!fZWw<{L}rlvq97R=cp!jICrRJb3cRA2L@V|`?=M?_rj_Yi~?>9*r%fH zu{3q5n#1(7Sko8`<8(Yi4lp|aP+oeMwr>ktS^>bm0=(iBKA`T59EW4Bm){TXFdZyk zwq}EU;Ge9`P(E;g#V~6XxOM`lX1~)#Q(ALy>gRrk9P}hcl!2Sv|8EmpQ*s!rUPR(F0!O};7?}hB~h;~{8gntMZ z9I;7&Jnv8vYRY8pt#*|nd#3Q)Y`Lvp=A_|7|5C3a5Xa`MW-@0<{6hEyD2(yKTRL40 zTB+)rLNFYUbmif^r64j8S5i}oF$K;uFoJ|EJIZaeq_HMP2E>n>2x_TPz~k*!JS269V4jN6V}QKIt?fh zkPJGD>EjO`WLm6~<-wfo*|;J+7hD$-O_v02jf=Al#^fu_!OM?3^O#%2*0&6eQh*wb zcAS(7oQ-)f3`n@1LY*MAn!dbI9TiuHno-O49h5;=TXxR!*K?v%3>8I4W|Dw;XUKpy z_L!I};b&5*iH_ssz6pg;FTg=HTSOHGBN-fnTIol$KxzuXwF)>vfzTq@YX5cuUI$MO z7(xV1P5LgC$?456boI@Ug#rz8EgYRN5Rz*%6yVH zk$+-%X^n(k&rZ_dpSOGkQ6mG{qB7`Z=TR`PFzR^PE|~SY&R2v&g(u+hNRQ31xKJ3# z41IakB{%1|o&&oLTW!78=sn@tG@*dhKI(8-DQsoyGBcGWj zuc0Daef8$#I1tC^$-x6`l`HSWfWwg^=f82pr%@>r2C-oK^SkJ}og-$w8U=L_(;bCL zzt(~ry&Xv418N`)nU#RYJt&TP3?Y<1i1@XWQ=@bi>xlduA$0Hpijo*n zc0NxYD#ywh&h`vM&5ChX5=Lr^v7U^*&7UAO{zog)&O^yy-J;|Qy_`q;JCvgpz~y8W zFiA_GHd1tB4>r0QQ?ZgbgHeZF7)`qxfpSpNk`rv+Zcu{FewNc04W)mvtoYO!j_ojd~yH$A&)ia;ua zn3#KQPPNN+CZJla2Pt@JJ5*lBb>W>%r*)eKgxNyD0u4-VsJfqn?fc z!Vd?f``614D0k}>V~X}~DX4sMU3; zK(`+cDtfS8ekNF2;B$=v;A{bbaX1Zbd0ctwBOA}wo@4AlguvSaY;~@{t}|3mUTh;R z4gJEeQpV}!opqIUY=A2Q^;g`qvY!%|IY&@wcbZAHj8k}coT*phQAaAw@E@`XruJ7j zyu#>S5M7(^>R-GGC+)8x)3gY;@38G4t_G}VePmZmKT~6jhanE5Sj{Q<1 z%6_tp`(hiDJsmhm)i=3 z&qkwvbb2Yqfq!iB7VG}g_@*!dx;{HHmiRTbvM=CBZ;Qoq#N~3(xaA8_T=g_>-3PKN zVsnErHT)R@#79kuh@hj>|{I*N1{E@1#?p)NK7vyCGVXW70rG@sP^J+N*Bfd=F+=Ic9;=_xq*j^&I@cdrJL{ zL&e14By1SrYX2*p7>NaSdsMZKylG%!Ch9JH!aKM!l7*c99RQAFcd$ZnGrHjztj3RL zX7J$lq&1l5sc8cB)nfYW7hIQzsp0!5EaNNYhX8y(A4^VR6W}c1(2_wYWJyN_*kgB4 zlGd;OP}|NY->GkzWgc@z0xzO9)!~n$XFnMRw)+`9&ph|6g-xE|WE9ZS%qyBUEbu0%=e%a9{ldiZvj1l_7qaBE8?CeqpdC^2ev$e4* zb$JOTv1*LHZ9WL`*(a$PI`qI4q(cJ)G{L4qrH#v6h(g%YE+q!urHy*Li z55xArdDOcAHVv({TVq;=5R@C$$Ss5EAF*X&R{&H37xeCmNH$#ht!C#tWcw){LAnuu z3EO;8Ykx!Vz$)zi8y@l%wg{9b$&J2lq)?ETJBN~a+Nu)IXvv80=ha|mEAD0P=7M6B z$F{2bg5drf6((=eto6GKv#Rbn8Zs!6ihWQ5JR?-6f$=l4s#Jw8% zby}#q6q|OMHC_^S?ULez*#Ob446Ig451RB*dxECsTyfp&mVJjD-;ppIenD8uk|RU= zr&HTOXYo$)PH|0uugV!aN*vXz$$OnpBQ)HEO^WM6#|>z`N2cjt+j12kg+<|umJ5qN zd@D`BJ#ias-I**-m(I4&bL&7I@Ozg@`7a;t(3P#Nl>w16Ob?i^V!_6k3X4>c_;ijoUy~!Yqrqiv z?^}}75^kU{xI{D3!n`gjidkri){N)z1UW=-j-DVbfW(|>qH>DkA$*4fR?xSscLB!6*i@kZy0?Ua#cN@aQOHg9gqUDB^l$urc3 zz1Pf8Q3rbFklN*09CAj*tHzvN98Qh?KNs#8KhSAhFo=edT#Ay*o8sGV>PE#)Z8|uR zr<@P*tN}9)blutaQSfu~-jvja5%^I5RKLQw=;>T%CG-)2Qo|2~+4M*b4H%U&D0q_3yK?cMVPvF4030$pq-RHEH5=C9?JLZ88uQH2Xc{ zya8Rq{`C>;vpWSw!TN5{h#In#s<^w6vdlSP;4@?aHwhKaXTUc$O?P*|g~TYR+ET|J z(WY|MedlYN!yC8nz_+l$BjTKdF(&DJT2{Il7mUv82mp@G7(#MV6ylrV*MYRYd zqd_lojfN05-rWA`ssZw1B!@}=Ev}MS!nfrJ_qaY&5z-T_U8rg!P<(0FLK#~0qq$|f zf>U!C`|ICP(ETg3GpdJVEX_Yq&bnc3y4NJ=EY*iDB-{4rfaZn!r54<|HRQlLPyRRpT#y*Z6IzGX}{iB81ndg)8LDs}JicCO-br5N=7?Llm5s zXZPJQc@x1PCQm%DxcG?35J}M6$RSUE6&!KmI#o6%d9Dv(t5dSZ{VJKwjq6t-4cl*{mNGXuC&68ejX|rW5hkdcSCdX0t!luo2#s+rAr3CKXIbkXwfU$sh%f&Qiq zw3Ns^=9Usznr{qaA`#OY|7cCR{3zfD=wbM+!ke^f(p*!rSsAS&P$^xqSQUFD70zP- z7=byMlElf}V11=K)alnvR$k8*^n^oWli{B4Hc)mq$+%q zfwKtmo#rZT;9>;U% z8`w{{QNs7R((%AgOp3hTT?7~_nlOTqDS|v%C^}!QWkKk~ksEotVi+ZTO`*9@MnHQj8kKb38 zku^(GJTnJ`xv#ZHkUuTdZ^{<9oxdd4!&=81MVZAenuv9t2eyDkZK90Ft?m7bwIgo{ zAdR6)FR;rm+65@+7RFv+dhXAqq(mU!UDv4cP>Ll%^(3Dt&9kb3%rC7C#m-RsVY|Rj z6FI_y(l`{0fB6%3yb=szYGNr9cN31ihZuPn)r3h;IIY8iYa+o1GUvZ~o>ox#JA{H) zafu6BV(SQ%nbF6F|2#1%47)g_qCkTC<(mtgymn6JOfmu%AXBL5WrtwedRr9LmW7k`9sA@BmA(!30tGmMI12`D3CfkyG{wH~ zbgh$oFkV%TD{YiSp}1{8b4exaSt79rEJrlCVy{UMvi1|_QPfla1)crrW5?q;;Q~r6 zCd^DtPkEhC1}lAxT2wj9lsTUKM9#9Vfnw5zbvZ5&$`3GscaRM2W$i8fKnQsG8ag?M zImURkFxU$KL5Y3L=9d6$ZpnZ-FJY#Tfi2Yk=Kql!d zIoi;NF=(cdcjHb{R&M-oVdPdDj-Er6o1%!24dzSp7$&tYF0k~_p|D0H{Rwa~rA>Nq zCWq1ytYREs?r~p#<1WJzr!W?k;HY`YS!E!*n3o6>-gpNJ{Gi*d`5D;?u$>~=vGnP* zx}%Z8U-GTPfY~F>;mXv~9KkX>?^tEG1}oHz9T$Jqb>0ma+m?&euNt{jXshE;!DW!raJo=&sKlr zOfNG7`AG-YZ!5#Ay7&WWv7%&;;g@h!BCnq~`K~|U%G}LYGm0bT?r}H1(DqX z5$q7JyMWfVT3P=xE(hM5)dX-Hs@0NE^x*6i!}Gn!J<|(3A$r9fwH&6QNyeN#8dD15 zBP^A@QR1Ak@oq*4kWoiA$PFB&9hPgqw6d(>M)XWBZWN^wMZf%>z}r1nn5GIajk)uB zSIOX_3zSa{@->)I6%rQ#86Jd{GSjnmNhKcYqkM;&eDqIhou;V`lsqyFD>Mk*i7u1% zi8!@q*Tux2rbYvJg>I}^y)JLvUu+5!JE?A)f>RnV>ckDeL{KGhqX{k8{y#=*^|Cr9 zS;5mN&Hc6&YBVtVA5MM#mmB;dn+KIONlkYO2qdcsHo2t%fX_5#0BhB*y%jUo`>Dok0o zsE>^@ea*%x(BJQr*?lFleXpjF(qI15T}Ky9+;%~1Snop0xn0@dnL+rn>4d_Ihm*)Y z(g9OorU1b=W<~frMh|M7L=eCsC`NCjR=EFc(eRj|&lJn*O0KMw;46q#n~pbON)H6| ze)p8uOlg;5ejCIRUN4opBm{}^6+e}_eH4vKtoj$8#?;bHz;XSRo^E29GGUzEbXzBTuxt|0-uObi+&~5NClWb z-@#we7N9bJ)VhUyK~+{;wdD0jBw%m-7O@7Qu6UX5HzqG6VtOpbCcY||P5`$=x}6=- z>&|J+C_mcU-OecX)NBLak6Y{Th{L~zTg;8}#TnKY#@%k9uCOihy(kJi*{}(gPF3kT zp&JwhpOC#}uOYTJUMcwvH9fzny#9gA!NQf`pC3^%wvE$}IrKbuLtc{6rI1cgEbt$wjt60$ z{&>dyD6d9CvbmyY9tqbsL+nR|TWLP1C%a=x9!2ug^$mvS8v3Np0$m-hH_!=17dnSu z?NY?Y4~Wc0A)3NCe_n8F#!aoa1tJYh2h1lReD=Iw@%)s7RVu=EyL+}qjbX1Uop9n= zd>&Co6VfB|bF!g4nZcrB;YX`hCce(M%*x7?ISv5uCcE$tVkC;Z%^1P4-SwrDz-bBk zIph%4o5T3HXBuUYw!!eyL9B-Z>@peV4J2AFi6{j4647GYSZ_V9H#xRng<9_pX4bj}m-_ z#lU`F?F_Ea)urpN>BQtND!WroE8A&Ox>8T2hN~3dsY`#ZdqInL9fWYslfYB?=rU$b zxjB|`#`ExovSgT_e!!oIEgf;(dsVIn*C8-wL`rze< zmXA_2A#Vcw!G0Pn^@|ajVBa45s^ridNJgX#)RX+#NKPQ0ci<~Z_?QZnZg966QoHo; zoQiF0o)4kG*t8~>c?y0>N1s)l);F=96;8ZRsRGEaX;E$kUZsL|o8w&eoHM@EyPt;5E zcq$SV)6RPa6PjZ->_`T%{EVXt6_g04Sj`Cm8eozh8U-0&PqrZ;IK>I&Xe+7ITrF4!xdm3>A%_v27#j@r zzI~TMW|k?TzBns(wJwX||7oDtQ8+MhIl;WxMl@Kz@Kb%0Loj zi$F;vnSz~ju91b{Mbve zhLfD?7D7%J2wPy-k$Y*5oHKkbe5gT^Tv4}U8ZkuXkVcGaat9}jJzXWD6p``&fBbM| z>|(JJH$d5ivf0~?$dW32FQpo`^#+%}HEwOKkL?MO@H{3I3yIJ9JF5Mv>C+f0(;kRD z`rXQ!_A2@!K92aqb%**-vHL|ze%^~u=4$DSyCvdK&0x{Kfn>lbB|V|}9`_=Z>FW+< z&8yu$1Zxa~_7ArJp@Ym%qn?k5W;RL0ZV?$s6I>qUoZ_WGX5wK0mQ{vgx2fk%-3tZ* ztWxTS54)rS?0q0?*Lrun2P*)`N@We{7L|r*4h{=7i@Q0Z`l_|3N%R zz}GuCoL2Kj%+ULjFwpqn%_Dsjn|Ciwblqd$Oma? z1o1#1{RjcXHNBx1R3wr~G8XH;_ZQ<{3P1uX_L||7MPvSm1ac6>z@jh}Fw!z1Ddvq4 zMfQNBvK|>%MmF+ocu z8(|etZqqe2Zt_Rg^Vn;%vT=FGN~l>6*9C^Ah(6$cc%M(?!9&R;aOk=P(zw#d1gu#} zp@IZ?&u(N1_GHoXj*FF~=9a+PKkoNb%^Vs)P@_`E79B*FrQp5hVQXW944bhQNIwjI zp@0BYE;pjJ+zc_$!C{WGW$4Q%U>4R2|9{`^i9Y69bY~H^t`ndNvDP!TpyFv@kG;<$ zyI(9|8PY^U+LcSIQAKd8`fc!2e(SZKNTUu(fh^tn6~@p?rTY`hfhuO>i_7zMcyeio zd!9=w7;LGhujjEHN&+MERR2%DP%-e1AK`4b+D9kjA3dsS=6Vp|g1Vc7VW^h@0+W;MW(jLv=BE?YUC>SnT*vin77TZNB zriMQY0q7RP+5-V$;^CxNDCJp$1mr0XLL-JYgwu$gksg{b-o6_g_8o@-iyCr-X~}I+ zAToy8W_2@Ef}AKz0}R5-te$Af-WxW>U?E?W08R$L?c>cHlHj2{4R?1zCm@M}%hbik z#H0&8RuqD|`Z)R2DJdz4^;K-nvZ5a#a7BVtM*l9Z$q)fuS;Ux7#K591AEZ{h5GaU(1!Wt~JMg`tj52PR{5@$*U2~o@-Y8(j>%!Lt07*wIb zpnP$LAit2-HCuo@1fGr0xUM>xh9Nfymap@en#dtjOFBQ*9QuoCYN0siM6qM^gqIL_ zG{Pj*{JYMYnNImmg%(V`rb%qzDj>!)MF&AoXmaS;MII8V{OlxHgh)}mVP^E)iXnbN zb2+I5iYyXq3QL3~oIs}-@cx`7CuH6Ke=-$iV1{D=PsOVWCMasK+PZ6eV-XI@Tz)G3 zw0Q7ox@!WL#zHI|mqD+qQUvieFbC~%0w=Ac{)H6ymb7z&3WuoE^%tUN5>%bl5P-??#`uCN~BwM>F$Q5yFt1`NVNUCoj^rIi3dagpa79xyZ{2E zw5>hE!~YlpXe2zTJWijbH$hnzjDHFM{?P-(Jk$OUNdNWyZ;a~MiS|zb4aoa10RyP} zSD^9|B~}K z0q>B|VE>cZTM;lyMO4Nho<4xxPW!~~hH$k3tpx{Gb$};tHuou-p4ssXRD0n`6evFb zWcEVooAARtIJO|OR31f^P9jqIXj=AEQUO_#3+(v^g0W7hcx8@@Jqy(;Qu{wXoNVh` z>}hXB(;!;-e-*L_`|ne<01`%v+XPRICXl|5eU5z&0P&|8&hvdy<)i9!sdrGc18^xl zo4Y4`NTHgC8~C;0 zPta~zF#bO7z&|beBJTUgJ}Ut(xj5|=Zk~2w#1*fxsZ~jFK6CO@sI4;m!Q1Kc0|%tf z_Ps6S^Nva12Jf8B2 zHvYmIhy-Kvez-xq-egXP?rlxEkZ)?>b{Fwp_XPE(@W>+PhG(4#yiDRNVS*J(_+#vJ zJP`a)CW?)|zB3QfRgoOY~r>?2#2nT~*wT`cnBDZ4R{cFM&gNTTf2@E7raglA? zZ$vb#bW<89`xY0o`-q*!4@iz1a_iT|x#kRv?^NODw^==y<{Yq>nW?jM0jb)H?|5(- z4H!tm#WIB|uJxD81w&7$fw8bzgiyN{H}6i#?OOFqLy8{O;6w z#6I~4ujWMltRfm2Zz4}7j+o$k*mxlWB+Lva{LZyVYXs+H)^h=+K3@6^4ey}xW^ruv z=!%_AL+w310{om9WDt03QeuuacI)ssw}L%|SFXY}D)!{@2g%=V(s49A?f^IkGU zT?R-AU{NEUAZ&4L#okR4t#gY$(>me7jFKJ*mE70DH|IgBJP41>1*R9<$3KqfqYWD!hxa+XyfGFiWyJkf+H_~Y_sVOm`;OY-l}>z6!z=B4NjaXC>$ z(cWy#U$aV*+(Rc9 zT(fjC;c6H@C4>Jh^3h7`ofrX07-WO%2-T`!5Y5r+XlwDBBT?G&D!vKRf%+oV#-+MdtPd?VfJ`wy z-oKW{j6aaW6IwCU``FPAJj0{JwJ;>RPdp$0?f2o z)DtkVbkg_+E5V$=ve^MEG&=UzsUv>WIkZ}2JkhOp+Lf?hOJl#nivBKRRMJu=*A&9@ zg{F|r0EK(}Mj7e1VE(u9VU^+hkrlW$J=+IQ=2L6y7bWjklH7*(ppvg>yn0#K?)ab) zSm~x>wFggS=d;%MGsc{Wh$>T#+doCrdivkgem!tZiMi4Ff_mYaZ z&pND29-4&od;{zXX#>s{86zKgrU+>!_%Tt-Bih|L3OdVc%}7Mhn@jD?O2D*2;d;;i z@7|dYhYkMF!pz;wB)TvCT|^?1g_)1tQkMQ`if*&q#uz*Wi(OGPF$4M17+V1KxVCOR zZ=-RSeoKxVQ-!z7qs;hQ`0_0`QLlt96Uu02aQ5tbp#*2wvA9!QBqZjx*^J{)ci7RW zgAs*1^$s9fHMCzyH)F~tq~LW=ry=-MRNK<0z;=R;mX9-Y+TVCfyg9frRJ{FkJ(e*`}mz@nERk zk{63AdGv-xqP6v>3+MWquL>jc{PB|r5Sj+!wKPbw_n~f1PF5Mk1epBVtBRdXNEVL> z(J%t)Oy(pTH@3Z_G*dCs@qUbB?xJtLjCXbY5ThxO%ZHxmLovqLJ?F^JPb5V9o^6Wn+_Y}Pex@d0 zuQG!W%NU?A)8pVjG~6tM=uq;$mN}~n#T;cd5#>e#>$f;G6g+jy@|~Cj_L$342vGu~wkfG*m=x10y0@0a+x7qScoXzz3iqr{xWwmdh(MVLvv4O_pWm)07>Jt)ZmZ@p=zL{9e&kJ^x6{Qd^!ey2urT*-UCZg)M$jp zM;VbS$*(})VSe|rCx!fYPXK!_34wSbcCWm@P5lirik%Fn>g8`3oKo|WgW^`t$NK=C zU1ae#QOr(J^U;^^(Qns=0VaZb-RVr77LZE6WBa>JO2vma1e){O{nB8|27GDIn9dQD z?mZ=!;&*ay#IPHm9>Ps9>|xIWkIf&Ld9Q#vMxH238JkOg;orqLA-U{3G)p|$z@TbG zlM!H3L#6DBt~e^hp~^Mk@1}LShGxlE-+k@UME9keWz)4YvbdQq+C!_t>ILC4ek{JC ze%1MVlnRe{;{1X+0qLS(Gw?*mq^4hiBE63OI#YT_%EE!q{pqHD4TlrQZBjgW=6Q*Y zA<#%6*++x>E|u5>YcWzmQks!)3 zVzv+J8|3SDtRCv^=@9$?$Fa~;H|T0@OHvc<~ zHk?I!k0_Jx$2zWca%x>*p92P>z?snK*DTVI*PR%EY4!=fOvgl{Lw~%A{hxQ7l{NNn z7K4ohS6_98@@_|0Ie_I_%P-bJq&Lhz0{KYuY-Li@4Nh~@vC4YrE8Ae)Av-7wS8^vI znoS6QG((i>DqG@Ewv@0pZc~yu8XvD)Dy>6M@L}-}UwCwXq1Xb6DlENnSKlr;%o7?s zeOApIOSLpjF^YL{am_g>To)1dBQcE7FUR|e+bSphRjE*j1U1;bRCMq@wOmm7aPLE) zbc2ng*90f?n%qO;xQ4OSs~rcHE3~^y^!0<)PPJiAbGiKzjI;tS9CI@I`cZEy0Zd)J z*$)p@dR-*ReY*&n1G~bO$3bR2iZhBkcRd5dNjSyHnQB^=&6jZ5PoSWV7*t1z0uAz3 z)J4a>8Jry2cN}1sToto<_N~@?Wdg<75r0Mvy&Bg%xt5U(?}-MVSoLo!AF=upmzYT` zEaH%~gdkLi5zQj9vhn>TZ(*7VvB5r0>T4;+f7yuzh!Qu$0 zKI+Nvt(`0di9WdKhA!bX9@SGknuWpDeTzzgr>!N%zJuUIT!*iC4bJ0wJECz5~#Ngc>GJV-sCxYE@A{_ zV5yx9^geP;=tmHIb&`j>irNwA3J-jrmCB$(P`)41__n$02`ypvcGpP+E(aJ4vWv3%2nH%C|uEA-T6I$BfkBgrv}?iE5)uHJT8~$lJ#xuQTyk!z<^LomderO}1-v zQn2N^yJhJ96$J1sMUDZu?T{9on)_-3_F=>MhTKCTx}n9%mxNe-7TlN-8T(D%SABSs zk%%~&^=O=Fdh*>ghaIu@HfXoLO)yd7w(p6%o0V^}y4xQ)2)_I+IdW0%Jx}R=wqpVS z)i~?y5Q*pO@(j4Tpf%ZG!61#%%}R*%Y#Kb0`!2ERM(?3Q4# zTg8}66lWFf$+3B2sBo-O7fnyQu}5Yu*Ox&Js-74=5b%dhOWf-AuHhIw&_*Q5n2F<}|*a z?ulsAs40bu;+4@ckCwh@EX>BsLljaX$R@kr%hSPiEHTU4_I>{^O|{nuC{Tz zFu6;M#_35)T*@eHI`8uChz*QHad1y@tG0x2MpFkS01YhgqprP~E*m7aw^H)Ejk7Km zLQ}pE2hHsePiR+G>X-%bE2b5h+Vq(a}Fv`F@TRS<6?xYcy)j@bC@lj{}F6CL5T~zG&!g zS%n_om7&S$Xnaf~(^b4dEjaF7?+U^`>(cj1@ENa2C4Rm0prkNSjC`x{V`o~tdY5CY z^S)SIbpvZ$D)U^;q+&1>)^zJ8%>;7ejr$eqR87!zO*CS|Gy-R8E&`a7KuU^Z2%@Nn z;e-SOXrK<_BIO4 zO2&lph9WWPT+MpkUrpXae%qUwqic`fka``$;v6F++c`@N(X1kVx|}s2CJmA3uU|Di zu*w>no5bLIz=i+ zS(;6nndrD>saY#?>&lh}-?ZEo-bYVTDb=W30PH|~u++!7?dY|G(R6}1qk3g%7dTaB z%o2fqHtX$VA;IUcY}A(^L+5RPLZAEPx~7;AY^1P-@Vfg->*53J*4L%Ih>?s>CLiG; zzhN!B3K229+lp!S`aFz87Lzt}10oM}-lF+5Wn5#k#mtBu|1GYk_7c>MSt;3#_IG#ij6aM20V5sZR{#J2 diff --git a/crates/j2k-native/fixtures/multi_segment/bypass_termall_gray_97.j2k b/crates/j2k-native/fixtures/multi_segment/bypass_termall_gray_97.j2k index b3ad14d47ddfa889b5246e8e8cd3a7558f8ea8e6..492d1848bbeb25fc80fd4e68c02ad2f4ca1ac998 100644 GIT binary patch delta 4449 zcmXX|cR1Dm7q?yGy7qSM(Iu|Ex!EIoOJt8oL^9vVEGyS8n_QHY$_iJ)*GS@q5NS|$ z6f!=)+w=U+^PJ~-|MB|cbzbLv&g;yfmPG_03&@J(0?EI?{6GJ_{{`k>J`lJHnAS;1 zA?)jRl_O?^STn(q(3d=-P|%`LUC#GQ)M~gwbC&M-l*u;lncRKn_f0541*IE>&hV+T z^JO`!Tw5bs0sEg&$XCxZ#O;8yXcDqt`*c^li3y%0iKisN^x0-1m0FMtnfj4DUdv6# zlIQ7PFt-%waJ)PNcHs<$U;isXnvkHDbv95b?8x{L0M* zmpG$B6VZ>95O{RK@bUp;$qjtYj zO+OQ#|2E~BbbI>No$2>CL~f@QvyK*T%S$0A?hzm`+<{35b^ODq3wbUwV#WkQn*6J= zO=SE)%9g;@hJ{N<;}qX7ZmMX)5h)|qgIhEc->l|kv=Pf*mHWCfSj9p>(JpA+;$G38 z8J7`Q=H>6wmsY*UW=tvA*@ej>Xlb`qrE3~!_20WM-koyCPl$td{YTFu*i7-$_Uswf zm;4RokpR%WrzfNGTPPUXQ$tBRoZhB>^cid~M9aiiy}tdLwJ)(=V=~*ioT4h^)_y{1 zV{^G`+(3D3fv#-N2yMSSDdudtd39IF-7n@i_(0-%CEmCxg~qnucHmL=3Fj}juAgJF z(HyczA{wX3_+>t3YLmhJ4j&)n0GmNt{cwrIdc}*>SFr>e^HWstcnDqU@D1PCsclek zJ#7%nt!cg4Eu|s2(#5h_>d0kFSY}9Hiz?^0&-GGzrs>3ernwP16Z(#<4yoa%@a@9E z+Wot?bsuA4Sjvj<*!A7h74vICyfEbfDw5v2Z|Er1nPLP_-Bg5%VX0*B-NYDyz7mUj z{73A!e0{VKx}6PE`qZR`^vX=I_&~sUA=-n07yKA6a1JB(sqwGu?DQ5Vq0#^vT(hIsnezloRLFhyx$hQ-qO$1@Z3q)_}} z0NI1la0oaXRQ{PwV_~OsQor@aAU8uw`Z}d7+d*RmY3g}rI@L-E1l+|bfn4gUCVpQs zr4?TB1p;~EWR+#AzY-cxcq`Kbg`UHlFWXq#d*w$r3wV4{59P=+5c(v7LYQMx?(8f) zs6|nK$$DKoW!ZU6U#vl*L*9C!6eHpvqoX2fN37*%NI{S{*E9W}xP4}W$DUT~RNwV# z$ytz?!H?5iyF=_&qQp4>vrG9t+TTj!e$^hE-JCM6rb6yvT@Tj!MRT{C1Y_jswd_5kEr=VkuO%ga~|Pa*X3A$ zs?(5%v+}w|Uv~w~eH%yBN0&WD4M>YdXkuimS3Re7Kqs9)K}@yQ)>8imcHFns=Twt7 zVDXbKj2`x9iox7j)w^5@|* zaTFr~9Cs9dwH5P@mk*<2Z=>IC6xxvo^=bnOwU$x0?Pfs-)rHbk=|Zjyt-#5MayLhE z#T_^77A+r^n^^#qW;0I$`d<98DW%kyCuXCu___9R=*ygMQi!&-p&|MAK)J-1RikBI zSD%Ko-RRiYxY8k|(U`fFmBd|4weEZ6_JhbTUD|-8A@eAeyWeKUrfUH4?2553W7wTT zcvWDeHnX0ITMvxxtLcjv&kP6^-V}H;MDFv%hHJ@*c?Coy@|l4|zNwdD!GPY4tfwbm z1gnUbcTPI!({R+;EfrPQs%R42NgKL1SO4FGj``I~G`uUbm&Anb^@!)^A(D91&t3MG zf44pC4IBItwZ}9`>x$8=W))ZCRy%*hYH)g-`tpuMBa*|wPkppL57$;JqScFa;;8kwhG211 z?vs}T3QbKZ;$*F3C?{Ma>=Y%gGB3kzx>+)|Jfmu)QRB6C1+dU*zZV~_#Y{g)t+i#? z_zEnCsBY9;UK!*%kc6(_lyD2K&YK@Fo3PuX#;*4!=D~{(!gfM)>Uvs6r_H!mrq$%8 zH)5FnK%k=GRZh=`M8oKem)x8KUFV$V=;Hpb4VK|vI|$p}-8P;n8paM_YZD%5o<~zy``O+U0A7E8{#n8@gzeictvq;C|2@j5~CUbyLsGZ3oqAYS=bv@SMC`m zDLNWn!J-iC;BKl$Yic8Ox!UZ}=9F3Pq;!Zk+LBJ@-KVhvfHLE!zAk%N;jKe<#=|e+ zz{bC*Ie4uqO!>$HLI1ls_Kmx>%{Q~MathKbSsaA49~H<3WrL-Lw0e)1ZPqJ2V*5qB zhCVUV$svFwqk`UqLr-A!MP)0#G5n@rxrmwVBWwOoAYM1NHlo`peVQ>Plb`BGHqDbT z4ic=gPn^PJ-SUFF(Yuk6XwD6l_5wurZ}f|H4D-uvM116)rv8 zC|KhFx|!6^pJysiwPAebI;w`Vjd`Ew?cmBEty@#S(=In=?v(0^1Go_BQ1?_aSm@SA zeFb;HuheYLn$^oS)OHCF;9Xz8O6_PP!#ei5X!3^8>Qga_Oa7AlDsTCFD#_YP!D7>= zFaa8)zy7}Cu`v4^bQV5JS(ysvW~H;6f*(y2h(DS+L<(NxlhD8{| zU(-E|kvv?QmXRR!RPLee)-@-unc;zeF1*&C1|;q~U6ctOnoQ=5I{BDcNjOaBsP|^M zySRr;lFa?}9M}gku6Ic>0s?g3<%N_f++c1T@AQP{VZJ&v_6*iiR?CEd)#l0-ku4-q;?fWZY0Z zYSp?EiT-QM=fRag6H5E*gXq<~e-0H%%goJnsOOsocQeC^eg zqi*BJpp#CYb&UzfArfw#B_@7Qgj!*}Nz_yCqO@_e6aHv@A z3o0i?yXv>gIYGRIO9TQ|{^Yj4&Xu|KI+*xI3DICrY4@X*4zn#b@3Y7~1~;6ySdhZ& z3kN~x@bXfZt>_qkqP@B4YY`97BiGG7o;JjWBXbXh4+yji)< zYLzMRMI|Tg@Rvrxo8g5EYWBHo5HRN82oYWd9+-ecrLE zP{f1~It+o7{;Uyr!$sVNt`@%np<~%s<9TVB4>*+1&tMR~Qg1`(8V^Gu75IM+2kj9C4}1_;=q#FG;yH7Y1(Qci92|1nc8r7(Nm2;U6+IlyKJXCa;1>X~$YG6x!s?D#GS zH8b)}vS)&tUl4dQb#@6^#A=cwB*mq~fxqgd^fsj=QaAl3msps{2Mxra&8DnV2j6D6@L|-U4}9bm`62xd?MX^{aa0s} zqPs|$ey!5=Y^M5PINSJM1O%4EI)zrvF<#Rs9kVn%n!&iLy?{GxXou-Zrm>;dyO&XT zV%1AhY&;CoO&v=xwi@xvgF7e9rly}9hp$!Nb6WQ^+3f2gDhyx{w~Q7(b|n5S=9v*P zkoMWcyf8)Rm<#XmUjCg@xKhAB^|qhqe$CaBbCVlkDUoy=lSL&u>i66#b9g*xtEFWB zF@k1M?*}=t7vqT-X2*ZQp}x8bLg13gL<4ou znZ?n>x*Iq&t6{-GmJFkn?QMwRs|zzF$@2=?5a3Qey(N|GzdQD)a7PTae6j5b6K*XR z8QHd(|K^^E^0R9YF#6!};JO!O`g(8Nnq4|s6M=SZP(Mj3CYMb<=-bDEQgX@79{*WI z2yixZ9Iu;YaiZvakvL-!BO$vsMF)9>ivfsQ&`OS{Un5v^j1m{m^R$Hgva-#c-qs#l zg{K<~$5BXo?;M#Df5U!_kXuF=O~u4moev zqiEXC|I}j1Icm9AsCQc!o-aH!XOuV#fAX1M-BXy};W_3K=t!j1n*ynK z=Ph;W>dH-?1V3w0kqf&EkAu_3r)lcVUpGlX;B9xFev3izjFP=V<=*D&+Wx{fq}7*` z-se2tCPelXW=GXPU?IOA>2g8&()Ey}uAg6x`y6t8YppC#`7G$X7z*R*9_|wUk=isQ zai<@7&u_ny!Y0S`#8uLZm-G%E9F%WgI>fzrZNRK~)i=~lOM;!qam=fDH!3~F9X4R&&z1OhvJFx&s?QE^6IRl z0uG3(&uFpT*(jF|ak9lr^6T&y$wsFZ04V?f0001h0001h00000000000001h0001hksw`v6yX2? z|C8a=K5YEB5)US?)mYWPg~&9PusowCuZm>ZCnO5a&$p$PM%w$Se=X3ZeAUZ<`V2Zn zd>Db3hL3|7FzFZW1!zSSpB?fgQOJXqBdi|h(ZTkEA5+P}GzD8mh}t{5g(=^G&|v%D zd4ussFdXn||9`|~SD!^QjD)*?pHKpBXyGDj=f4;=ZLMNH0Yt$N&*~?wmGEJ|uV<$Q z$OmgPiIP$38B_#=qt=rAO(jR`8h8f-?{~(v?n0e4Sd)4~88S&L{BXB_7~*pW59zo` z8lNQMAPoP1v=sj@xKoyqedmGOFw^KL>hl{^*?_F^%-1U z9MF(8@^a4{Tk zVW7Z`fk2hr@U?+!qZDO;BLHe8(k^>Lw68zQ`uBf-3E^|y%w^dkQoR|c(@$QAxiMjX zEB}80tZ+EP=RmZ+-`mVH*GrkPBWj^-mM-|lD9?YYpLu_O3(^b%258%s#|1WSf-1E{ zsx$E}foa>5vY}efn+tbaqknt4g2yT6@{~~c$;NUD4e4Zep3o3YH)|* zK|=Qe^uh&@kIGTOEL7DQvXc!idnvW0G0wx}onlNoe9DMBuD6g1X4u+dZ&7aOzf{7` z@+P}Wx6z{+3p|XFo}MEzkkLy#twvlw#oII}+iwE!ZO1vZHYfHnXMf*#(9OjUe{&U> zmAy$p7BZ;#rodESS$Ynr0MQ@N810=tKtHEhxln$X_FnW;ayXtQ*o@W|kp|zSo4%HV zaerj8LfS`BQ?n=ObtCBWvOAo?jf-Jnww_1x@a=x-J+=QY+G-eCbv^_-4@-Y$A69JB zuOPlXV+JGqdR7))jenTvk!Y}M6Lv`BWT5D)9fvwqf`?Ov{vhYkb2q;X6YN}ecjkH# z(p{4b$R4**IR=jG{E#5#RRbhlbN0=t+r1@M6X#)S48S}--yNEMn=*>3niB@5s=~<$ zGm#Sj_~|y{GlRXoC(V97uC#UTUDAaDlh)CUL^N+dP2_*;On)Hme1L5v5&ZW-{}Q!% zXG*PwP%n2z`D08ML;@Fm&zZhO`z9dnGnTpNuUnlGOA49xR4fVK0#P!1A&3u(X zZw8^>-4(-qOwP6*Wdjrnfb!7MZ`zi~-psTMnsrECwYx%p02LT2MRdP<;b=nt1^`)q z)(t5XcmPdU!a0$stP+ z)}X(8kxlCsrAGpq3;9wh6KPxnk_D6o!}YL-E83W6!<3B?Xyo4&_Tw;j1-pP08>Wvx zpVw3S7O_BNTO@Ri)n=0r!5vI5e?hTd-hWB=mQjF{F9sWb%O%5CP`j5^18u%Pe*`+P z%3toUAy+U=7wo%pG49{XPdCGe+D9_G%TEuW3-2XiKJHaj#^E=-u}zPuTr6{yNlz5^FZr2!LI+)79j23l|pJARX%NsZ)V0bNe9t zG|a!8Pp^LjII{&fej8_3_t;@{h7O=m`ana`IKgcf%ysc+lXnFkerY9qcoL!*_mSOwp^t)n}w7ce>&Uvei&&XQI*>*qcR;b zA{$sO9&pdtbTBn=vLV4k&z?if-kY5*8h--_NnqrUJ9NE^HwbAc+}{)UpTVTgVs=M< zXL#&?mG9^Qe^tXpT4nCYO&1^H4|X8oq)BduLgnhORzR$OO*;pF2Q9jxJq7gsnv|V{ z3)J;N6jD+{XnY4KN0fcFeVk(J&YYOgR z57y1^3>yI(N7~fIOqBL+EWn2UQcr&wR!ACu8iW3{h81*cQjur0ElobQy6ifLJCeI&*&`9A=QKs&2LrxUC2pJ3F`JQEt3ONraDRUb0E*Q*|42E1 z_m160SsQ9rIXa(w6p0s8f*ph3g@1nq!&Bk|EO}1y(Boja(p$mTOHx!*_j6BzE&qRi z07jbCeY=vZO)y%r3mJnO)F@ExQKv=0&+dl@e}4temj@O0s0Pmz&!-mHd8Lbcv`pwT zd4Cb&0;#Rvtxtb{26NloQivJmSz051f(HgS)5Fb&wFKT?LQ)*{_VKctj+C6D7*0@( zKV2dB$^k^X%|IC(q8LALEk-C@pcHN$7=Aoca+px~@lnRgu2GB^D8&l)-$ssc2%~ZG zmxl!tkC=R;6g@&E-J6O0c&FpVKOQOhMmU_K5MI7hYm@^G$^k{|#Wwwejg;Oi;8~PG zxapC~5eDTbqVb}Whei{WV+ZUVbgJb5r27V+DfvV&xj>%5y~>c2q6a2_%!lxwEF%*) zi15kZJorjBzppM_0QKhrK`}*-kBwGQ7=Iy61J@t{(i#doo?Z4iEM5F`TM4UNZbwRY z{_6SNO=x-1{_89E!Bt&3|9^npcE43+aw;pp-p2#Zt<=$kZZEWPy!%`GsPY@B=g_>a zu0F3R%fdbXf5byta#qrR+oxL`C+F|UW&K=8v)*(i}RKD=|=g)I04Hx${J9rJI$2#)OVq0IFHlKe8a&2nV)S#W1qsMgg1e;U+BgrFJ z7HS_5KRE^!V-lZ#|85p@QBe?nx6k3C7%QCBOwvwXlpEIPaCjZae-Svc-rqZCoJh=1 zS@zHNvEszCWiP#o{P~R0V92~MCj^5Y9RkunMnM}-3d_%B_E&#=GP`fVzQ`A?E~F(9 zK?-b0{7oYMMa(==!K8S~JBO|ilIkZ-KmIsuT}VtIJrg$&5A2mJm}B~N>VmsItUF$PKWyHT zkz!xKS07ct%HI-D;p+wl|7upLf1ii#EAT(+ql(XioUJ)3mu;(a#4~?pu~}5Vg}(Eb zxtHSC_yOHg0?h*IOWV!O&2S2Dz!iylq*}4oC)C5F$}i&YIYzxirf~~~lO1QSrV>fI^V81_?H;0^O%WO9@c`4!}5wToKI%Roe`lQ*B_Iz2r85J1tx!gEGfrc z8S+U)unPthAPRP(P%2Fy3I=%@7Ye*ne>(RXfM-B~;ve7T3R6Ta?l>3>SJ<$0w)lbG zsnNGQ&H&Cza@~U`@GPtLmpDCq|57G{5`SX1hiOCqe_k#th0741p^L?z4*M`EL7%jt zREG=R&YLjaS;^>{uGgXq<}-hI+)ca9L!A$!e+YamNkh!0;$4^ev?P|5UoOKa$ zjZMQM7)h-+SU0wzvB7h6F5{<=kz)-|-TgbrZCm#Y!p|DSU~L_6_@m@S@QoxTA1_8- zQ%=9H@Z-;4dj-8fc~3%4#`!i#4bJsUgAa|_bG=M68!T-2wAY}hkw1Tc5u0?$&NM&9 zNb;#i51Tyt%!6C{u*?rh0_YD5(Ff5t;)CBgBCKyawwKQ z&B%Ev;3(_7J(+wM8`y$?WL=4iFyeoqhDnp;>{wqvrOz%AD@8|{X+33OMn>)Szc}i) zSb*;bl}$IDqUHS4uS9=p*TP3{9pr&ezYtM*lJVC@w*}3ZJmCG;uj?#zBxr(WsbLM- z-)nMd-QZPmw`l(@6c0{rt9Jxkq=fqdx1pQ+gU`?xvNi!^$_6*-dD^wN-={Fu(o@KE95!SitCeEA~At+GF7Q~thAF}M{1Ct|{en<~)3 zu)XqYMH=_s!ao|QdChqxlaU4>e}TAdZ%+vz>(MvDmWHG3*g9CEL0rj3K=d1>EK#3Z zTA~%MjtxkT*PQ8%qXf4I3Fxw8BsrRIRqX14XyvbG!sgh&{o;)1?HLldAHA)A2ghr< z$SJoQfi_99_$UDDNOZDOJLxzxZVZRRXTFTRcM-h(PnTg?S0}c@^S?eff3v0+|S?;!fj?H~l`0Vreiav_oE;?yh(~}AIfB^%VZw1;|2#!50ao+^OY}xAmXf`zh zMiN5s^>O&vQD^c8CV7^t*84E{!vM0(Wz#JKQ9Nlbu@OgqN- ze+~;6c;8*T#@c&4SYP1N(E0SdHQgW_dH)5{{K(sb!gc1hS4@0De_u31$195UU}67H z69p}!gCVXJSX*jam+MfYC2HTe266<|f23)kr=p-j!R9elOe_(tw^&E2j8_2ydMHB0lNj~3!@hzTJHH(*n$6lc25N6 z@ze(3UpDpy&1-q3e_St!&WXG{Lyxd_Cj z{J1CdF(6Y*y!&;DN>b5Bp+f9=tIFR?j7_yqUlM*&vO-8~^|S diff --git a/crates/j2k-native/fixtures/multi_segment/bypass_termall_rgb_53.j2k b/crates/j2k-native/fixtures/multi_segment/bypass_termall_rgb_53.j2k index 316fd342481f044a7bf7b8b7ba48628abde22cd1..48c359121dfd36a4e879774724e9372f4bfd2311 100644 GIT binary patch literal 15236 zcmd6ObzD?I-|w>2f^>&SBP`uWF5MDR(kap<9fy_%X^|GBBt=3g=@d{xxL-)nFX~#9T+d{*Ma(*eqH}MP=C{}YZwe2 z8x8Hx2h@e(!Qe1-G&mfNi3Xa&a8W1~6>aDP+QL|2X!4$RRuAlK8La&owA}4n)wC6r z8GP(Ky&T+J83cHQd4#|e81AnoB?w?>VA|WY7y%LyHxK|M&XS|b*;aHDC;N5{wR7aZ z0f%dKBttPwYf?BFyOpHw1ril&T#OrsdMgE)j>c(iIOCg!8P^6vnwG`;t{>oKQ(nAn z)ar^|Bkd;r{Nc(5@nd>H3a5xX=s*iOy8Y=9p}pbFeOn=mHI2~Sm5K%5cudKVWDuAp z`4aMGg?@he!HL%C9+FWd3&oB@Y9g8dUujN zaEbo1PsAV4f^Bx&#VA~{p?}uwp*HFhOD)n4NffcXH1vNvN9lW$=i6}z|;7X zq+TL$?M>)ZH!Z^4^f{T0YK*m@x`KmpFzjwABM5wxCX0AQuHd-{ZvqY0St2tZ;_{hF z8dQq}7)2$7-SQTA%me~QAHU1iqt0A=?yyK#v&ON+;W}Q@B_t-Z9|%mF2Ynrx&VU+X zrh9K19CH|IziD!`NqXhit3;XDQS)(p=2H%K4q?JD#GZb3Pqi!_@rRhnN+ykg0BcyX zP$W~0_d0?9Xe%5&ZZ7UwCcYgT2e6Inxm)?X>aa&>Ff%;&fwS`a7nyuDk;gYOdGd+( z{or>E2ruQnt_SzJBHD?r7{-fZ*)1wJ*E6*b$47!9m-*@5v#8t6sA!WOH7UF!NM?GF z!&8r`LxpLSGGb#_3j)rs4p!6FA1l)+gV*=_=S#(m4)Cwz(83KzVpX}Ov>7Ve@i-B1 z5TM|^KFWvLJKUiw#Mt{jvEU<5Lki3^;0_;e6jODHtPt5D2xu%Y@ob&RoN-uJJ`>r_ zn^BXJBg&Bv31dBos@5UE?s_f`0y`EuTiavYFLX>)w{x_fGR5+aVzkkpl|-nkjT&E2 zlQrmrz<2~gHad#@(dhF?|6s^w@L!ueCdtqlJs2EJqa#t>FuzCA6p_;DLhZYl;Tt`i zyvca_5Vw8GUv|Y3AS-%E(L>c$2y6@jl)$Wl%7i=E!q&wOiaO$thG@k%V`jh)!D-g z-e8FT@-yy4sB%ofQ~@>{PR@GhEW{d*xVNMd(o)$60-VpiwQ7nbURk2US=1|fQ3RQa z=1P7d6h<|LETDVpj%vdz5a3nwR~JhNioLN;C)1ThxL>)^^tSGX(#Evr5_M_2^TtRe z2=vFEvt|cKMv_`!(wjuqP2YLsV_JBfpGfrGvgbsZCY;fILPPq2*|QgsHZ}$Zm*?e- zud5A6`ZAlEeaX@(^;F&Gms&9yr#6_UxV?+onpkBO=s|!-XE71C6ttY>NgRHcpI?Yw zL+{bwi^br66?j@2DkD|yB@p=#1mt~tiYhsC6|PbR-)QON7ua}ENBR|A%ZWz!d52yt z^Y)$_q(9=7aq^|s4jt(t3B=C|_zi!mX-yu~IGw?dpB4|VUm(A}lbRaTXyv(mD zm6R5wVxH8SIx~+5m1gpIMUvl)2Z12!;OuV!10fmgDrThpa87Io@to}u%n$X6`GnS) zYo$fv45r#!Os9j+{)D$)6=@3D!xi~f`6en0NQJKvbjIb&S52AT&vo@OenF@r!RARQhI zhP>1O$Kug0iZoN$MKix5q{Eqqf&Za)~|X45FO#DY!2fX!(;VP9ybfZGQz;3r)(|Plg}0V zUfEji45O7Lp@Ip6z**uK-VsEkgNO!xrUSXEKKR3YKvOA7E$RMe* z7m5xWs}{-{N3*!(c8why>vsf!HK&eXCT>-%3p+S{9L%_!L$c|~2(6i>gdh8D3=*dV z%L@pExY|IA!c^Jv{hhLBd}C=R^>xdDed0nTc-t)cRAB?|{n5co*_#;2%{=T!E zT`t66-5l=FnVh*Cm$LXQROTBiUl-x6jeB7vl)~DG>-EVSPplzVS+HEP(yBOR(S$}4 zdCaY4aE5_HRZtXQsQ50@y=19w>+LgH5b$dFB1|Z8G7~uN^WfY9RjAg=|J8c*lHRev z=nl=*fZbgX=!=e3i*|zfO(N%y zx8Lo1F_>z$YofviGD1CdW*jb+0RkiD#&=$|kcLw0dG|ZraX%Ln+))lrYDu#! zjaF~$0mL6&^6czM&EWT&-+I0pY-b9HZ0I6Gu^Ro%Bsewis!=MN+(o-^qs#u(!e(}! ztRmXZDcrd7em*|`;$nwC6=~S=f@=^%zO^FEYRhcl_Ti(d1e~rn$GH>{4^?+Z1E%a+ z&F`KykZy;Ig z{shhW@gtROY?8wy=r7YCGc6*UfUNeHnFgmTM#lZIX{|`owcw)xjJP$tRmjBu8nOQr zY-oq$XJpdGbSKg>f2Rg{O>p4__LF|5s5K|;b9`j=w#oM6f^*w&hQ|I%vW&}iw? zBYy@3`Ks7RaxUdyUNjJ}uT9$%PCie@ea>*i$Ip_>USZjgi-8qT$ibZQ9Mb_V^g$m4 z_MYK(V@G&jdZrPwtH&85G}TjJ8XvKlMtiWGw7D(Xia`KL-)hr%7J5rmLHIKx$Qhx% zBJIBAQmub6X}2;}sC2sx1Rkt=Yy`>YA~;s1NwY=6chu{24MOkufi98R^$W9cI+})- ze!|fR(KMeoqnsdMGx80mO4g{LmTe^+k46Qrvp1(!(O9}0Nx+yE&(+2Y9caOKq@Y zjBkK9Sbc^Wr4w(cWZlMc8IFy52?AY4)U|n4a^&Ex6#vE-W}cG8^zYklkT=!~kv!UN zp3&O{fz}JO>zViIn3&T`ZhowxsaIV|-M0Cx7Z>@K5EpQT&A$S@FC4|buxurYwZmg{L-OTNL+ zJ))jPr>4xQc?OE?l5&oES)cvQ>(5VS+k`tc+aGScnQOq38GNT~Et;bQ50<~PH1co> z@b%@vFCc$m$`~{#eM_mpQBlP3Ew0hYoM9_sLgH(A6P3pC$1IHqE(-u7N3TP8;feM@ zCku5g0n?doNvU%i(8QlQ`OH+2Zis2|?#blkG(FvREx^s%85HcQCu5xxN^wD<==X`KsnCtmG^2XQovKsK?3yZhB^ePCI@nZ;! z9S(m;MkY2#+Sfb5WJ0617_gnQy#FyMijQ>nE5#iL0=$$s4*(j*!|4MQL%&UYCEWge z)WP{av7Hx_!phcJNajUMuYHmdZfOBP4=NOBRu&%6bl~=zI1lq{O zxG7r!uqrxZ{F@irWxeWVQTh^XF5$v%en8c6rGGQK7JW@T`5^jH1UAA!S!^ZK=5?fL z)mLI!5ni64Yd|0`B{LmgYotWpBWEU!HbzB4o0_pHgN}ICc$EdWA@?={%x&e94*V-} zRU5aaG!W40%yg$sK{$<@um$iXeyzAu{YB%45c6qALgST5K0B|W7FPan<-%v)?7`WG z+oG50Ofhq-`GfK;L~g%)XIc+p7>%aucaaP&I3^0GQI0<)>H)(Ff(m}lu7F>HCL9|5 zfMFR5q0c$6et!!UEunAEBDhjic!>5eNY^kwTE;jt-ar_JLzRbmWHnI_-Ju^*4u?asI1M&MbVW}BEQag%@+`=xzJY?ws#2hxfk z!~HRgz6ye=#b=p!yftrIxVNDX2{`0a$@ocJhxG2<*dkE&J{(>-fPXoH$7??AOE;2F8*;AB_+S%ufzoc0<13mD->Ecr+uo_Cxitl<^oS1 z-7$&C3cG3ZdUrPZ+WLo6N@9t)*#}KX)6~wo;DI-Mixm|)Znf_!giW3pTXx=!9w>`B zdVMXI-`BON_a5XBUG5wC_8k_%TDE?CR!pyCF~5du|I>!zL5%4`b`t6h%=^SUbWor4 zvmxrO$4~~6uh>5_UxbonTY|eYn@1hZ?8bLD#KjYjB}NqxP`Gq~6B?LE!u%K+hzf~h zblKK;?@K1C>TW>>{Ov3PE&sxAFzgK_L+qRlQsF<{-t9Km#Im#WKFxJ64vFhc1S!kcGpY2?Wwj$4pI4$BbH9 z<66DF$UQtf9oLILIcOSHnn@-Mk^0t3byIa`Gua*{I zg#972AbD%-e49#v#H;MBmQY+ziMR&t$MeHY44c%)3mBix+{CX>X6+b4LH0?x+zawk zzji^Mol^(Z!E(y%O;*eJAMglf;^^ecrT0d-ARxv`d^;YCB9i+)l^=|+D}|^_B{Z5M zgX+6*5n2g)6eT(|-!g`TV2ZU0d+0+DA(+-xqa);$f`-T%M;@-$@)jsetOzO?WQFG* zMp2Zn%pfjD51m1$nq)EQ`8p{pyt>^Pwde;po}ahMtbzRN&eMj`Rv(-k#(vZ+cMJ4{ zQL1uD-yz0+M>SJ`ZxWM)s!O4FRodsJ8ae_XiIy+C7Q|KeBXCdc@_)hQ==;`?oV__P zYNqX@Vw(Go!$*O0f(Kq8P}AZoUGsJ8?7EssKy1K`@w6_DWB0yskEp0G$u?M3o;^gL zK!Axz8->z_5}GXv28)Bkh{=dxm@o`9a$*b^IXMjK%5imMWl?{UA|@u9B0LTk_fLYy z{U#8g8jqKqlbs?zGdG%<<6R^5_t>$9*=uB$GiFr4Xao{QMNvtq-vcF{u>oM@KSw6)kc*eG~>-Un}d6VQ}3F?duqhH#j7+>gPStB-?9t*BY8j4@_ue ztI4nj8Mbcf2;ZB{scCypn&|h#-#@?H)Gm;p09VaJ1x2M1haW(kNr|GkU8Cx<0kIbh zHruhVSD3D}$~aY^aN@&8#XWK5Ijwtq8osKVO>BjN1H>!+SLDtNVCDe&jrlD=y49M?B`L+}{i zRb{%2^P4+^??mnSy9)MA$hm<)T}+L(d^{CN``C+&swVBK2z;QrVflu&4>6!-=N7EnIt-tph(-lQMC&Qqd8BUNP|&Lc zfs_4}{gqwc=HsUCCqF=A(B<+{YT?D}md~fRvRW)G!VhUhDn;XblblnkOHEuIdMOHCtr2u( zoXad0^9N9r4kb2gc2ZB>p5!PMcf(U~?3BgQ?Y7_EyDT9Bg0)5LjpHDJhE}lX6%&;g z@0os#j8##rTz{?etRUSt;%#op_)Du1{?)f05RXN=Fx3dwKH#EUV;;(Ct}nW@>GsZ4 zC(A9a_hmJb@ogianSD?S)IYm=`Leq6r3LU7R9Bz>exoBlPl{(JY5CG8;4IJ@1kNFt z#nGAnfim#9@o6Bi#xzjph-tPr_gzfLOrX_w`sgWrp-6oDc&QMIPg})8nl^dUM<%tKn+19QN@`sE<>?!9{S^=>QOiIh zOVCRQU&bNC7l8g6W~ z4i{CC1t%{xjqbN-NE<=g%d`1~faNpk+0N?Yvg6V#ne%CfZ!;^M=O#cK;VCo zV{vpM1Pvlw6WTun0X(C-(%EJl9_y#uTx(tZ{`qb}Iz5sYeqLE8K6p@P9vYB7xu3;7h@UZyb*2mg&~$}LHWC?% z#z(L8$uQMJ*`!63F=66~BVF1C8~)XJ1!)5a_`iTH^uVU0ceP;6D7VE-MyH3-soRXz z9yfX9mH6^-y`g~EHU;EXsTGMs^WfSDd{-eJB zQ+w-Q*f>Gt2mnU@1p|t5mfw+lhuw>>!`#+7iB#(vv?Qc zWB{Pqes8%t`^t*iIl1>#wB~BoG@gBuhhH%7E1tH2JbX6P3V~uRe9!Ko@0t@bmCc1& z;6QK}s|5&xeRRWOcs6O#D(j%1>YH`IZ$nv8<{c^ZQhw8GSSS_CEEXVhc{N2*ulz=% zufq83df6AeZ(}$5N!QmNMR|S0Ui4mDr>MQk7@OCSk&W{-FnO-$*{p01-=t49J=iEH zN4fyS>-#Hw?=_N-0Ejf~G|uQ&GAF!L+(pf2DRUl-qX<)@XFDEV zPHlXoe1xxjEpH<|DPU`zuB!Km^v@f7S{qG1Tu*O7K_($K;rDH3A`h0R5(=5y33nC6 zv^`j7$x*uOSpY2FJO4DQ9NmHdK3;s%CgGoiBfT`r@mIdvU!Yc**XWBIen` zau-kKgFDCm5-pcv@IgaPNbTER@{N;M@}#pLuYxt>L;*MB*)*P;EH)VTTvSs41-7iW z0xE0Po@tt|ZxK&f6s!osQA^V~Oi2kX``WzD39jd;B`b#Nu-MA%6NB z?!rhvhJs&l9Z|>6$zCfWP0QbL9sWUhByS7fE*r6Pq(m6jft)lEQPjzJ64PL~4I|zs zS?OPM?2N&m1~K*F{3%e>xbk})w&P3y?Jh&yeJB_dg<`|-P~H}l!!zRN7=ln6vK`R! z*GV3Ok}jej&R7@e2gQhD@bm2iO!9EXx+r%hQv?DI=j-@+6wX~3`>PFKIXDy}@^-kd z-qB`yjPQjsPJ~7>FkRb!!bhfuXo4Dvb^jHwhTkEGu#!j-b*Vx7cFAWlylXAhGm%@A zy2RK=iX(EQhMUo~GUJ;hEK^DGJdgrE>iYT^o&ind)tvSCoy-OEr}bto9DCB(7ifpc zkKfK22$L~=zUd2W_JzC{_7(63s9BHVi@2~|qH_7D1-_Jzte0NB!g>})p;Sox5(M(> zlvVCI>IaNJo}{zl$`F29+>z&9Uo_b0uduKh>U} z9Uuqo;){|f=at_7v`RZl>Eo~i0R~uXvtnG*E1DjI*(iNrn9MRUbBVD&unbW!P!v7I z3jw&0YSqV^U@m&``P{We6|MY;ep@FmFzE+t@*<^)pwb%pG~~$xescowcmE0T0-}ST z5Jw=M1W7|Yfv!&>hmZZUBTHTs9YnRqJeN75x%H^qG@VagOx}}?;J8Fno*9q}OQ`n= zDx8Ro*s*kSv7GV{hqhsyKFGdRhK89xRPHbEBy~lN@}BkRP)fyuEhZkOF~;IGkyi*| z5b#sjtiQI7;t#KdvrO$*e2Ll zVo9$x@J^6YZ~N`SuG#FMS0Ch{`0BhLbo9n^sweVc{+P^thZ+$?93l$iZS3kYOXwrgS3B%kxU{k`djaU`1@}P ze3QcciJKL=PCMIieZ~QmjBHu<>mYEgzo9ji_5PN6Ul+e+M~~7wB$=^)s|{bd>#Hz@ zE!T%o7k=fri;m3uU~vxkp@>KNBMcA5H%vH}&?in!XRsO~u2vw;nEuO%-$I6z|6lT7 zjJXQZ2Z7hw=eGChfrl0s ziCzLq+)w77roS2RTMT-%%yN&H{JgG#`3#@5f2hCe+EI76C#^0NbIaQgZ1vw^qB-8z z%m~x|IyFjWFk4vIXoJ^`Xt@|>NO&hCIdcpx9SgLz>xG?-=Ih851hcpHTl9y{UKbS##S4=w3f774*B%~QQ`9**8r85wC zj8PWJvvn=DDmXNiXP7B0t7S@w}RhL&rC;6AQ=Kp+2_lL zHj`5m%Q8)w&&=X{z7q??H$o876)G@T3F^O5*4yx~maS!%G|0K(W2M{NeHKNjW0!t# z;tqw}tQiD2KQRxU@pZeK5|>+HBg|EMa0S`lS#{0m*iq2God$se7kzjwvBt%x#=-bg z%Pux=+dgTS$RoG(QNjiIzPCyrq&&yJ83BRc1AU?M59quEx4Q-g;Ry~%zEy8k( zxJ0dN;`(|IcYDf5$O7Q-?Wj75XLxbkB^PwP#FB%rxOLO07!TCqxOBIDu*RUpko|@0 zN&EaC(0&92)oj(RiFbzut|>0w7`$b?F7eZ8e+sgI^W$H#hGZQ47aSz(UtoX9{0Ha* z`a(cINfGJxJ-U%r{>?UWzX{;ady`U(9p#rgYSCRxd<9VoFef0WHbJ+L_$&I=OLErnbjAgS1pz+~dqa!jVJpd`nmRVB&&=fsE$&9Ko6-BKdW` zj7{$7W{;P|kdb_Qp8kli6J5q3i28>Mwx)f+LUt&6lu!O%eX(oJajn?)ph z^A76kW*dH}3|}m?Vs{p)_{cY+NtMil-hfBaxuo9*Yi=(+q8{STBa#7u zvF$H&+5Xvh$lDUnRg$IH+6VNArai=HWmh@`_5D9*ximn%+zmb$?0B!hl_>GYEZP}| zg757)L2MoV%Tbq8X(aF4Rtretf9Q(bwtin14N*|!owSap(qK+D*>&3ojp|hcHv6J5 z2-MG|=G|V6>eNmm58C3Ac3YeZLPb1rt+g_^;h8fu-1(1opqTgXCFEuP(7)V3Tf&yB zz4y*yvhwJlV0WvP@d{*dzT1Q5_~W~la;1Js@oUpO!&Q$xNDsZnBtb_5)-{qtPf?Tb z-Ju`77P&~J;xYp|$@qoZNc;J#Q^5ipXp(P%L1FCLJ*#N@aErR~kYt$oUXP|H`{tG4 zn10P*$t!N}uZ)mVV8BH8myv}2?6~U}`nQi*ns%BFF6EGlKWJ>0U7as}@IZq4OTphG z2ae5%D_il-72_Em#u`_yV7R{}aTj=N80NPN0@}!FGhz=gAVDZEUR62JqYj%8n_G`r z)O>V#C%b&?SBLrU$@kCU2?7Q!v`={Oaa|<8sz^wCX6ZgY3O=`*dsFlvQU=R#LOJz@ zgK)TBeID=_O|1q+9vPiZ6kqrTLkTmyl;qn_1R-qHsMlC7y5;F`PR)fJw5vpzQNP8D zyynB8BCPQBHQQii6x4rQ{(bupwpn+S_`?qS*J1g)+n zQ+7@Smzegkgd||=M)uoFuU0hssfo;Yt2isOZr#v$kUo-l7 z7F`XH>Lwv??Lw_X3>vT{grb6yTV;d>2AQDM0vW2w;PxwnWRBck0Y?DP-{cRVV;H&h zvg;GkRnxUW7tLtAYfC1)RM(#vD6RByiPgUGp&josx&rQBt5M8;>sDw`Oo0y^w%`4N zJO=<;%YZhwPwd)~bLIZ6qLX+}UM>k}-%F%u9%S)1C(Rio+ zeF6TyO(x@a1cR$l(GM@1!=?3HH2-n7m!NAt`|&L%l4*x>*&xcWK)w%2E_u@cfexQY z(QTjhQxp07-7sw6XuP zWELj`|JC&}PdK~GSOfuqJS)}ij~e@~K!b?l=n=@TysB({@p$?W=d%Vj=Nj#!_%m1? z;Tl;=5;}7&iP}{2QrVoJcjLD&Q(1Q zLpu)PrCE7S)_swmH>HmiVJCZnK=8jUKc;$>ep@e&t(0iOor34_orWj!UC$XNZtVKf zD^RYgZomJDsp(pyuPo5keC!)XR+pgOzsX!jhd)2#~&w88|F7Yn|FBlmUq2 zX4MQPFhl06-11JHEKSs}#9B18hEdg)gGUmmrRT}ZJrCUr>!p79v%F+&U3s3}fv_-Qp+2rGG5QPAR-dPbqV}c%f%Tm&AdiR1qQKIDMdk_3Gyu zzIWB(DqDCRvdl%0hoEPAkFZ6yY(e2zbyp6w7Pv5)dw__`&R$+|Y3&RueFYeN`aDIC z7$dhljgIbpZ=`OQ?K92E-kM_4v^DNKKUFya^}*tkUXNT?*c+)tpCvSu+>zEK)n^tF zCm1WGjU7v9#Sop6$>`8`X3VVk?uZ_;PM;G)IG#L^w<+Ve|?-ZLdU_Clg6Dl^7<_yn-BxhzJ7%4RU?q#lJSpiV?X(JNwXJ7Ve>$8l=<+dpVrd zC_e^wARd6VK83L2G#Z3UPtj4e0rs31Z_r5=kB%I+b#!Z8F6U<^LC^9{H#&YmBW}H4=AL^hL z6;+Ukbp7D-mpQaCQR!~B@(-*K9gmAxNbgyX@avzK{&iacfAfFm-yQ#qK7Nfr{@wB4 zso`_ZQ19}Ep4pgtpaY+-6wRPxRK1x{_Lj3R9zP1hzFq{b?1YaKxLo8T?Cwbin}K6E zLHpWG#kowos_tiKVvVxB6RjarL~A1nu_M)vmg2NNmbnY{Oe(CyMRKYcgu&Mv znV!hn+j;B6xvas`UUS{E%&&F`Voq}V)qTl;z_08V1^&&UA>D6+2D-lq;zh<^kQje+ z#NV9gZ;ty3RyOV|XL#N%%yW+GsB>{edJC0<{o)gTpBEvu+{@dT0gW zxkH3khI?KyE4lFK&S-_#Buj+NK@M&-?&@UxH>X4a%;qQ1qFpE%xh}H!E~)7Rj$U<{ zsuN$?gL{`f4Jk`M|3&);^S?Voy!Z?7-)8@v8Z~S~r1m`IW)m~~qVGx_ zZsxlf$APx$KZAE!j>WDPwQb#z)Y_qUUJ3hNS+MYO?Cb~@unx0zXO@M$= zrW=NYGeMbXR~4&nM9Cq=dzlpwP<4NpSpFQd?)r3CvY2-_(m_j~+3jgDnZImQ)H(u) zaUy#SiAt}OK~*J{y<*s9Y1%dUCR{0wi7VdmZqB#V=>^rsgeaQqU%WW~%a9P6KNlC! zy??#${Xf-$;4d@&k^e(Yu>Q%5|D+E8cmmonCpH;`>lqLRt@o#vbJe*$Fd=*?L$i<= zmi)5B0eujl5T-yaaBA&%zX9OPl{Q|p&{#2@cI)YtL0C- z7sx&hy-$V8Q^>|FC|1%7MHVUQND8Dy`@NBTaU+|8MCeuHR(uGKLfr%t+V&tzk zl)R;SFq=R+!oL${#jJ%~`0#3F{oU7YqR~eF)expXsQ-`Z@o)9v?-$UT>dDvFCM&>9 zy<{oo>F4eJ4lG>VnWXMW@4y!OX#BgMqYg)v#JDOd@$N&rRUxLoiHs^ZZujv!2tp3y z!K_bg_Z=KjgR<0WpI=JYM=|ZFtzf%CwyO`1Vbseom(zWCi65dfgNGKG5{%E7#rtk~ zrwq%)cFjc!ssw|X%iaA1Ow4{8Fv>>9VK}3a=dmble}6mLC3Y# zxR+>tHG$_@qp@Rx&iV%Px=4>>?yVi}pRK?xibm0pp+yom7=Ijb>&TEm_Zf||QX8>+ z?Bnfz;&lQ(0M0eF#pCpd6NLlsS`|7FjL@ZrZ2Wq0J0{CYJx_W+{8whA>u2pqg-luG6;AfO(SaNTqW<&oBLc#=xrP65djIr_zZvs~J-^SvE1j=9gP~CP z-#P-~zXu3N#4j)%KO^oWC-3FH&rw3^{m9}awvWUupXR9-A=ot1J=qI^4BrYahmH8^ z`lcN|6+n*4Hn-GK^@&T-_r&4N^Uq{4C4+JLd2w!zvCLi~n_gkjFS#>hB ztngv-9`C#7*r%$jJiRICEGW4ACj8s#O3cG{f4v}7p^K`SPMTs!snq0JT6x{)pu0wg z6S<&ChQJgy#Iygs)I;|6_EtK-{U4!15wwJMobTa~!J`{ww#db9`+fNdVxf_#Tau@? z`sL8XghOUjRzRmTMUhNEzg>H*rwj;uXPK-$W!d@l27gTC_doXjOEZFIfc}4~(uA02 z_15U^oK;b(TU3ym4Bn9Ati1P}wQcJ)@Fe*TrK`XdsmN9R*a&CgI0zVjc?@6q!|DGL z+`rsJ{sZT~-1`rg|1YZuTn`MyPB-1Oj(E-%Hs+t&Dc?U147g%{pJf-2n%N5UH9(Eq za=d{Ah>lk3Ru4((@B3r^hnN1BBIK2Sm;OJgVyw-mGjV`hO@AY`%OV~FN#;v|bA_J3 v?6>`K$@bOj-ojM)8&3V*(_2LN&_4?!CI4T_5i-wzVcdUt`QOt9aPEHqN-<=S delta 7850 zcmXX~bx;)E_oaJjX;?zKK{}R{kPr}%Mv<15j<t3w1PAXND0y%veF^aDIehH zcjk9z&dmMm&V6&=JMWwu;hUtu{Gj`whbD;jzkvTg{+ItR;QvD^GaWiss*EClW}%@! zAhu=mDq`W};%*#G7Qv4R>4>3&g}=A&mP^yDSGNOg@oSIeo=hCEV_)N zz#f|XB!acpdz~2g|7gX)j9ZNR@E&5z!3|`^?#X-_vfpe@gmR^YApz+R3FUm-9#!nW zEHk~ytn%j@aQL>Mdl>z?t(nof`n)rme{j4T0BHC9$EmpnGIAZXplHE3N^uG+aM!mm z1xZ;PO~BBYCvbGTBW#ANK-km5QF<99%M=I|XGi-J10K|zg=QEdwXxCx?SR|D?*N9a zz2fkzGdy8l6{EszDMBN>e-nkYn`M({=>0ez8bUCVYpkKjZp05>28Epym=3DxAP3+` zn#7+oqJH8_>o$+nh#ubjSkyP92J?JZMN?$k5k(Ob`?E*K9hSTA-07p zh(fd)Cm9}xqb9Xg!-Uxq4?CZqTu<`_RDp>SNPXFH(LeDzmKj@gf;`Mq=a*X_=(2?0hK9m*^aZuUu^_HDp^*9MzyKn-*(>0p zX0oH0fIe}!tZ9?uy%YL9cBIyGwNkb5lCy25xy^9vPhElIE=8SM#8x13wW$_DDlh0@bj*3VNN(!EV_H&!SivV9yd4QJmZY^!<#?&#T(_d*kt7qIM*^ zvOw(sNr$KV@a7lCb(A?42{p?LQH^Cmw(+)uU#=sYw#>?n=30KW`41snr#LkCFv?zS zZtvAoPnT@9pW|U;v)-U5UMl}c@5az$Q^5o~sMO@>9VX)uO~I$uMcHi3)g~QqWQ}X>7D!Z!VC39{BjG$Cg zk;LIKwcoAa9;8b?%I0`Mog(SQoo1iQt?HAN%=?$nqA8X?#Dnh*KT9p7vdT! zX8$lka2O5lB8fEveDgfi*zvy%)No`e!3xJtCQHTnl_&=p>9#*n;lm}1((IHbdWo*X z^Y9X*OGTs|DnYkeGFGzEV#d*trI+LHIvbub&w+q7_Jk=fGC_1#bScHJ($kf3g_?X6 zjHx7I_zsRY5FN#zS@+EqZ%16}21KQY-cEks!MWhc$X3+Zl4|7e-?GHHe@VANvQR|;6Su>!s>sn3gnt0pb4da=cUMM5_ z#&*GQG z;*BU87M==pPUFZGa<>s7@a`DL#+K9RFe zH7X;@>g>V({1qO{cuN?>sm9cD4NJ5pXMVJz@iMBO_J#L=vw{1qsOX+ra8e7xlwVb< z%H2KtWYxZ5Mn{Kp-nCYGiM zb;!TD+J1#gUvDl!J$N?v&7X;*KGsp0$JXgZc2vteh}1be7#_@-g1iIwo)KL}I**@rtVO z3gGQFEJ&3kaD?h>%}Yy}GKJe=nKH8+pqV`ntMzBMUE#j^&ZIsDtBZz zPo%^z3-7q+Zy)`r1B!1m;@CKE@q*%z`+ZL-; znhvR#QBtGG!k7l=BjuO}r;AGX@9eLqUHLJg3HkKI;^ ze^K(<`jl(Ujr`n<(IPY=Ew+T0cWJ-4hiLh79n*Z8*y{PK6YIC%pM!}jANN_ge#&v_ z?Xk#@Di|KwI>z?XC1&qQ#1#K=c+~o52W-wP5BRqSnA4v^R#T6*s9GXzmF&x(*PXe! zUZs&qA1ZRKQj>+4S#)Qu($l;jY}EX)B8gHeriV$sR5bH&DYwuEL|4P|8TY3=KMV~0mf%*3WQDH)IZ@Ln8%moGh~$+{?x z+*A*2O{2L4l=; zj3+XzdTZSqdKO1y8)gn-hrBV)rmy;9tdWY9Vx8bCC4qp>qR8KS|B&7L-yO-RFNg1( zBUx$0e&o9c8wZ_P@tv7*boi}%)rO}vO-ON??X-L#G1uVuq0G?nfM z5W5%#u=)&8B-rh+a4(DN$(VPm+??lA(>uC(R%T4q=#j};5^2%vK#8IPh8QA- z0>uWaRX}BZ+)#B7uw4cPA75d4d@wZo936cqsUP)8Te!80lIOI9bv{!ooX@Iq(3Sca zZ^ihfGSXhdFy{QnT;QGJ+X@lYH$ zjPX0`ipkaF-7J)Grw$OYXviwi6+gIVKFe@M=l>2RUj@9kyvxA#j?-gbDilmy@KI$X z3jp-yP)myK0JN1eT*&n2RKWUXJWT{u@l9venb&3KEAfM*BFXQft8hG;OOoGSt9Z=T zwNY&xJ^zT5glU~fW7GM?E#PlrN@7PkI?M77UObae7h%-3QQ9f43Mh}m2~P{xEMi%k}whS zIfidDf1dIo{f*7wO=?8BilC>>{AmY6*|3W_`exqg7?WxC7N(hCLwLPB1R+9DC`z)5do+B1w7 zLx#3S3e`Yc>I&d=NaP-WHjQ0@C8C#h|;P5yi6%yqROx9T$sh$vk6GNiep!)wv+m6Js7mF<~Q*G z3Umqq{u69lwIcIu0x$CZSTeWef}y4)hI2>?l1!g8{6Z7hPh@ zxz7i?=Hdt*nXu!F5~}dv5~e0Y8f*Wm15&mDx1dSQ{4iO)22-@P#bgcnf#Qb54D|m3 zk$~W-%*M0mYNyGa46~sbDuoXL(^gCcqg7SkVK%Ab`R4!-G*0Dk4##S9dcYeUkL={v z2PrdRw+ZyWU0s8=R6;8qH(N1!fJy!fMwG#3RE1G&nDFC?fspdj&``nEcG51SV?II{ zovugd12H>sZPKyVoiF%3+lHH|$G8)1W$?mw`@&dmq27J?tLJBG-LDxc{m`!AoCOzF z2OD$ru7E=nwG)kSY|HUpvbwLalvKjK)tkz8JvAneToO=`uNC$|>dCAA&$HZZn>VBF^f4>XARF_01m(6J&?!^xLty zB0F40T$QAt`legT`^oDAHWi*{KR?+1o(_7~aNtl295*Kf>=SkMy8ZuZg-JUN)wr^m zhML(Pk}A7YHNoMp@D$1bjD^1K@DpIxpDfR)j)EGd`DPvq+fZ@}9}t5W>@N52V~nM~ zC-ElaM8u!)99$T@ZgxE{v>vk&>J|*+tVq`GKpF*}Lz{A}aNLPthK6PIsu|J87SYDB> z=o2js%W^Y{RniK8QJdNG=#1C;jofxw0X{_%m3EyU+>4845+}RR{aI1hX0Oag^F0Gc z#j{h`)T^n&%QaB>rY>>7*emp<^VoorHC91GZ&W&gTMA^-)Jree&k>lGtSy2#Hz{nu zw6aDhC-*f#u1loe8pJm!Q6@VS6-@#_MB^;VfEZ~i|9BEVaLp?1iQK>PV~Yuz?**h1?exIfok-am$JA1kGa+~bMB7YAi51`Z^QFp zvm4-?Lx&Q>POweN*B@OE02{%HO2sf#IL48VPms{kNWX_E+HV zTp|)M8aEkTf0$+YnDYK{vic*yDq}l?_bvAss>iQ5>Fn7u5^6#L(JH7DN+}xo$!wF> z7Z~5Vhkg#obpz)WX-rKzOtK`>iJL6{jMRdf=q{gu?QdqCBYr!jypvfY<1fUX<9C>y&OO z?p$?r4b5~$E|_?%*mm%0dX;gYOqYz&U@ZKL>C;oC;;xJxj-R^YfCQJkq=-SmhxAof zjirFITFg5SHiz}sa#|s~g@BQyTy!lw6xozvWpm^KVSwne+YEzn(s2G(fYbY-n+9<- zbEi*Y=@~44DyK!tggf@}U5I4NYLE4gQ9UgbLnEd3@X(sRewrYDN|O~YO+`v#(QD>z z78!sRvG3MuXqci>E=@1XuC}=IdCdpU5Qmw5Evqc4?MW}9lY#Z0vWWLCxQ=YV+iZbj zPZ*5K7m?lBSpjIm5NKyM9z1dR+0~}Uu`inSpSzZLGj$2(Gb>#;fW1C5QlS0OqJx_} zpB92sBq*}UDCsXOdFS>;kdTV@RXh@Kl?hJOi<6wF7&qQHdSU%u``*U%789)nb!fO;=CV^O5j4=_!+@|lKS8(G$ITKFd7cNZt%pO|H|D@&C?rh zNAK?~a=OXN2n-%aC|I2p0dtyvdL_Wjc?kGD&HvQ0q zRL!QiE@p)z&ula;pVeAAkUW_JjU`=LT4R}G&LRgl zoPx)O1j*p5nb58)#Up6xy6!Kz)ZJzD83`U;J$H8ftHL-DK`t^mjrDjA(l5S+gCSwg zGH8K{NtTWV+p(W;E5^Ad*^+LM+V<6cV@bLq)(k%%L)7J-{^_Xsu(WJgExo*<<=yzk zFz4g@JcxEwbtYpiK$G$*dvW5+XWOw~D7bdZb5b6NEEJ?+$TfP3>gSWJ>+zuxl z=njOns#)LokVQtNg>T)r6|S{pHIcwdLrw72bRdFRD#`QoLO^-l{q5EnS5No-I6D0< zvDzw1~BetGkW_J=7kZ-dO402f6V<$r58VsvZK* zdLWdX%Te_v?pdQ}LL3r51k%P(;fcWo8x1EWR%WN!zpSjDR~8JT6wKa zEb2S1({Y1|r!y<~&o7EPr=aATGtECM7X74K+Zn|uZt2|W0-ulzXy<^f9bApP1+hA# zm*u3~+_`+eUZnl_GrxMGPmezza^(7Nc1@nuMFD*l16gbA640^817ckCbd0~%_-hPHoVbGmru zzfvs2j*{ryz=V|FD~h<{i{DykzLGp)2#j%mppwm_YwZcAU-64d_RwLvjVn6TZ#W_l z66Ps8r~A4f4*C6NLBfkt!l(tI#K#H@6N{}$fM`~+qHe%c=RFFO|ME~_c^XsPI1_|Z zA>tzg0V>aC`)RR`qp%wQ@lF=>L@*Og&pc5O@O=*P$SObedN&IvyW9%k3o?JuoU}gI zC^TF9!{8`bQjV7SF6q`sZ&1lR$-GSrK=D%76RT5`>8<8-%L_ERvj|? z`h)7u^FqWZ-(ybGCkp^`^OynZC+jCaEll~AF)c(Wi9k>EQ(yax#?MEXt3t_`n^lMt zJ~lvJi4j@n`^rfa=Eqp~2}-?MMma(N0I}3Z2{fPmDRtT@!$gUf)vim-eG(@%;RBqv zx&SDTil(Z2NTzc-z8YqA1B4gmOa0Z;8X0|F%A@Db>ha%tS=Cd$3Ia}Hl+gl34#<*3 ze7p*)r?@VNkCmISOvm?E840JcRhhH@N+k(5!R$v6n(8os*&D`%QJ*7CPk;@?}gbriD9=CmJ{gOTVC!t4T>t56_vQ z1vMJk#m^$}v_(m+KK(a9ya}Gr^U~u+YVo-FDJUi$vMA4m7Z;%vH&$YB{t}~`G1rt@ z|3EXc;b%@)%@YZI77bUvez{FpQTf&IzK&X>jXD-<0Rtvt7)1 z)>a?iS&ci3rYP)-nKun^utm(4Yfj|?!o;#$-nhSEv}7+jU*}{t_*gS_kMh?M4-qqF zZWD~zg?ox6A8%n_Oj=v)%bcJ9PeWC*(?cLKet#afr`;gORk!T4H~VP!@;Xw`@;lz8 z+I$p$*)$^E!i0|C^f@B25=p4S$zSzv85rc;0A)s{xH=TX!trHj~U$& zRg=#9<8*1nlQsJJWm-mtboICag2SBn3=k@yi?xEWQvR>J^=@qR0mM91)Td;$&pRzu0WRTm+qO-=-8( source: PayloadSource<'_, '_>, (payload, segments): (&'p mut Vec, &'p mut Vec), ) -> Result> { - if plan.coding == CodeBlockCoding::Classic - && !band_style(plan, job.band).single_codeword_segment() - { - return collect_segments(plan, job, source, (payload, segments)); + if plan.coding == CodeBlockCoding::Classic { + let style = band_style(plan, job.band); + if !style.single_codeword_segment() { + return collect_segments(style, job, source, (payload, segments)); + } } let data = match job.payload { Payload::Borrowed(data) => data, @@ -1653,7 +1654,7 @@ fn job_payload<'p>( /// `segments` and, when its payload spans several fragments, gathers them /// into `payload`. The thread's scratch holds both for the plan's largest. fn collect_segments<'p>( - plan: &TilePlan<'_>, + style: CodeBlockStyle, job: &BlockJob<'p>, source: PayloadSource<'_, '_>, (payload, segments): (&'p mut Vec, &'p mut Vec), @@ -1667,7 +1668,7 @@ fn collect_segments<'p>( segments.clear(); let (data_bytes, _, coding_passes) = walk_classic_segments( code_block, - band_style(plan, job.band), + style, (source.layers, source.segments), |fragment| { if gathered { @@ -2114,11 +2115,13 @@ mod tests { use alloc::vec::Vec; use super::{ - gather_payload, route_fallback, split_runs, tile_route_fallback, BlockJob, BlockPlacement, - BlockScratch, CodeBlock, CodeBlockCoding, DecodeAllocationBudget, Layer, Payload, + collect_segments, gather_payload, plan_block, route_fallback, split_runs, + tile_route_fallback, BlockJob, BlockPlacement, BlockScratch, CodeBlock, CodeBlockCoding, + CodeBlockStyle, DecodeAllocationBudget, DecompositionStorage, Layer, Payload, PayloadSource, Segment, Strip, TaskCounts, TileRoute, }; use crate::j2c::rect::IntRect; + use crate::J2kCodeBlockSegment; /// A thread gathers payloads into scratch reserved for the plan's /// largest before it starts, as charged, and gathering a payload larger @@ -2198,6 +2201,92 @@ mod tests { .is_err()); } + /// A classic block of several codeword segments whose bytes are one + /// piece of the codestream, the rest of its segments empty, borrows its + /// payload and collects every segment's descriptor: ten arithmetic + /// passes, then a raw segment of two passes and no bytes. + #[test] + fn a_segmented_block_in_one_piece_borrows_its_payload() { + let data = [7_u8; 100]; + let mut storage = DecompositionStorage::default(); + storage.segments.extend([ + Segment { + idx: 0, + coding_pases: 10, + data_length: 100, + data: &data, + }, + Segment { + idx: 1, + coding_pases: 2, + data_length: 0, + data: &[], + }, + ]); + storage + .layers + .extend([0..1, 1..2].map(|range| Layer::with_segments(range).unwrap())); + let rect = IntRect::from_xywh(0, 0, 4, 4); + storage.code_blocks.push(CodeBlock { + rect, + x_idx: 0, + y_idx: 0, + layers: 0..2, + has_been_included: true, + missing_bit_planes: 0, + number_of_coding_passes: 12, + ht_total_coding_passes: 0, + ht_first_cleanup_pass: None, + ht_selected_set: None, + coding: Some(CodeBlockCoding::Classic), + l_block: 3, + non_empty_layer_count: 1, + }); + let style = CodeBlockStyle { + selective_arithmetic_coding_bypass: true, + ..CodeBlockStyle::default() + }; + let placement = BlockPlacement::new(rect, rect); + let (job, segments) = plan_block( + &storage.code_blocks[0], + 0, + placement, + 0, + &storage, + Some(style), + ) + .unwrap(); + assert!(matches!(job.payload, Payload::Borrowed(payload) if core::ptr::eq(payload, &data))); + assert_eq!(segments, 2); + let mut budget = DecodeAllocationBudget::from_live_bytes(0).unwrap(); + let mut scratches = BlockScratch::for_threads(1, (0, 2), &mut budget).unwrap(); + let BlockScratch { + payload, + segments: descriptors, + .. + } = &mut scratches[0]; + let (payload, descriptors) = collect_segments( + style, + &job, + PayloadSource::new(&storage), + (payload, descriptors), + ) + .unwrap(); + assert!(core::ptr::eq(payload, &data)); + let segment = + |data_offset, data_length, start_coding_pass, end_coding_pass| J2kCodeBlockSegment { + data_offset, + data_length, + start_coding_pass, + end_coding_pass, + use_arithmetic: start_coding_pass == 0, + }; + assert_eq!( + descriptors, + [segment(0, 100, 0, 10), segment(100, 0, 10, 12)] + ); + } + #[test] fn split_runs_cover_the_strip_in_balanced_parts() { let ramp: Vec = (1..=40).collect(); diff --git a/crates/j2k-native/src/tests/multi_segment.rs b/crates/j2k-native/src/tests/multi_segment.rs index d4763591..79d37757 100644 --- a/crates/j2k-native/src/tests/multi_segment.rs +++ b/crates/j2k-native/src/tests/multi_segment.rs @@ -8,8 +8,8 @@ //! whose rows then need no component planes. //! //! The fixtures in `fixtures/multi_segment/` were encoded with `OpenJPEG` -//! 2.5.4's `opj_compress` from two synthetic images, 128 x 128 and 256 x 256. -//! The unsigned 8-bit red, green and blue samples at `(x, y)` are +//! 2.5.4's `opj_compress` from synthetic images of 64 x 64 and 128 x 128 +//! samples. The unsigned 8-bit red, green and blue samples at `(x, y)` are //! //! ```text //! r = (7x² + 13y² + 5xy + 31x) mod 256 @@ -18,45 +18,65 @@ //! ``` //! //! and the gray sample is `(r + 2g + b) / 4`, rounded down. Written as a -//! binary PPM (`P6`) and PGM (`P5`) of maximum 255, `rgb128.ppm`, -//! `gray128.pgm` and `rgb256.ppm`, they encode with three resolution levels -//! below the full one, 16 x 16 code-blocks, RPCL order and three or four -//! quality layers: +//! binary PPM (`P6`) and PGM (`P5`) of maximum 255, `rgb64.ppm`, +//! `rgb128.ppm`, `gray64.pgm` and `gray128.pgm`, they encode with two +//! decomposition levels, 16 x 16 code-blocks, RPCL order and three or four +//! quality layers, the last of most at a low compression ratio, so that +//! blocks code bit-planes far below their first four: //! //! ```sh //! c="-n 3 -b 16,16 -p RPCL" -//! opj_compress -i gray128.pgm -o bypass_gray_53.j2k $c -M 1 -r 16,8,4,2 -//! opj_compress -i rgb128.ppm -o bypass_rgb_97.j2k $c -I -M 1 -r 32,16,8 +//! opj_compress -i gray128.pgm -o bypass_gray_53.j2k $c -M 1 -r 4,2,1 +//! opj_compress -i rgb64.ppm -o bypass_rgb_97.j2k $c -I -M 1 -r 8,4,2,1 //! opj_compress -i gray128.pgm -o termall_gray_97.j2k $c -I -M 4 -r 16,8,3 //! opj_compress -i rgb128.ppm -o termall_rgb_53.j2k $c -M 4 -r 32,16,8 //! opj_compress -i rgb128.ppm -o reset_rgb_97.j2k $c -I -M 2 -r 32,16,8 //! opj_compress -i gray128.pgm -o segmark_gray_53.j2k $c -M 32 -r 16,8,3 -//! opj_compress -i gray128.pgm -o bypass_termall_gray_97.j2k $c -I -M 5 -r 16,8,3 -//! opj_compress -i rgb128.ppm -o bypass_termall_rgb_53.j2k $c -M 5 -r 40,20,10,6 -//! opj_compress -i gray128.pgm -o all_modes_gray_97.j2k $c -I -M 63 -r 16,8,3 +//! opj_compress -i gray64.pgm -o bypass_termall_gray_97.j2k $c -I -M 5 -r 4,2,1 +//! opj_compress -i rgb64.ppm -o bypass_termall_rgb_53.j2k $c -M 5 -r 8,4,1 +//! opj_compress -i gray64.pgm -o all_modes_gray_97.j2k $c -I -M 63 -r 4,2,1 //! ``` //! -//! and, as archival images are often coded, with selective bypass, five +//! As archival images are often coded, one has selective bypass, four //! decomposition levels, 64 x 64 code-blocks, precincts, packet lengths and //! seven layers: //! //! ```sh -//! opj_compress -i rgb256.ppm -o bypass_rgb_97_64x64.j2k -n 6 -b 64,64 -p RPCL -PLT -I -M 1 \ -//! -c "[256,256],[256,256],[128,128]" -r 160,120,90,70,55,45,36 +//! opj_compress -i rgb128.ppm -o bypass_rgb_97_64x64.j2k -n 5 -b 64,64 -p RPCL -PLT -I -M 1 \ +//! -c "[128,128],[128,128],[64,64]" -r 24,16,10,6,3,2,1.5 //! ``` //! +//! And one codes its first component in 32 x 64 code-blocks and the others +//! in 64 x 64 ones. Two encodes differ only in their code-block size: +//! +//! ```sh +//! c="-n 2 -p RPCL -PLT -mct 0 -M 1 -r 6,3,1.6" +//! opj_compress -i rgb128.ppm -o first.j2k $c -b 32,64 +//! opj_compress -i rgb128.ppm -o rest.j2k $c -b 64,64 +//! ``` +//! +//! `bypass_rgb_53_blocks_per_component.j2k` is the main header of +//! `rest.j2k`, without its comment and with a COC marker after its COD that +//! gives component 0 the first encode's code-block size (`Scoc` 0, then +//! `SPcoc` as in its COD); then one tile-part without packet lengths, whose +//! packets, in their RPCL order, are the first encode's for component 0 +//! and the second's for the others, as their packet lengths split them. +//! `opj_decompress` decodes its first component as the first encode's and +//! the others as the second's. +//! //! `-M` adds 1 for selective bypass, 2 for context reset, 4 for termination //! on each pass, 8 for the vertically causal context, 16 for predictable //! termination and 32 for segmentation symbols. The samples contain no //! externally sourced image content. use alloc::vec::Vec; +use core::cell::Cell; use crate::{ ComponentPlane, ComponentRowSink, CpuDecodeParallelism, DecodeSettings, DecoderContext, Image, }; -const FIXTURES: [(&str, &[u8]); 10] = [ +const FIXTURES: [(&str, &[u8]); 11] = [ ( "bypass gray 5/3", include_bytes!("../../fixtures/multi_segment/bypass_gray_53.j2k"), @@ -97,6 +117,10 @@ const FIXTURES: [(&str, &[u8]); 10] = [ "bypass rgb 9/7, 64 x 64 blocks", include_bytes!("../../fixtures/multi_segment/bypass_rgb_97_64x64.j2k"), ), + ( + "bypass rgb 5/3, code-block size per component", + include_bytes!("../../fixtures/multi_segment/bypass_rgb_53_blocks_per_component.j2k"), + ), ]; /// Collects the sample bits a sink receives into region-sized planes, and @@ -367,3 +391,122 @@ fn rows_stream_from_serial_bands_when_the_planes_pass_the_budget() { }); } } + +/// What the codeword segments of an image's classic code-blocks hold, as +/// its packets give them. +#[derive(Clone, Copy, Debug, Default)] +struct Segments { + /// Blocks of several segments. + segmented: usize, + /// Raw segments, and raw segments an arithmetic one follows. + raw: usize, + raw_then_arithmetic: usize, + /// Segments that hold coding passes and no bytes. + empty: usize, +} + +std::thread_local! { + /// Whether the image [`segments`] reads codes with selective bypass, and + /// in one codeword segment per block, and what it found in its tile. + static STYLE: Cell<(bool, bool)> = const { Cell::new((false, true)) }; + static FOUND: Cell> = const { Cell::new(None) }; +} + +/// The codeword segments of `image`'s classic code-blocks, read from its +/// tile's records once the tile route on a pool has decoded them. +fn segments(image: &Image<'_>) -> Segments { + let style = image + .header + .global_coding_style + .component_parameters + .parameters + .code_block_style; + pool(4).install(|| { + STYLE.with(|cell| { + cell.set(( + style.selective_arithmetic_coding_bypass, + style.single_codeword_segment(), + )); + }); + let mut context = context(CpuDecodeParallelism::Auto); + context.tile_decode_context.tile_route_post_decode = Some(|storage| { + // Past the first ten passes, the passes but the cleanup ones + // are raw. A block of one segment numbers its layers' + // contributions instead. + let (bypass, single) = STYLE.with(Cell::get); + let raw = |pass: u8| bypass && pass >= 10 && !pass.is_multiple_of(3); + let mut found = Segments::default(); + for block in &storage.code_blocks { + // Each segment's first pass, passes and bytes. + let mut segments: Vec<(u8, u8, usize)> = Vec::new(); + let (mut index, mut passes) = (None, 0_u8); + for layer in &storage.layers[block.layers.clone()] { + let Some(range) = layer.segments() else { + continue; + }; + for record in &storage.segments[range] { + if index != Some(record.idx) && !(single && index.is_some()) { + index = Some(record.idx); + segments.push((passes, 0, 0)); + } + if let Some(segment) = segments.last_mut() { + segment.1 += record.coding_pases; + segment.2 += record.data.len(); + } + passes += record.coding_pases; + } + } + found.segmented += usize::from(segments.len() > 1); + for (at, &(first, passes, bytes)) in segments.iter().enumerate() { + if raw(first) { + found.raw += 1; + found.raw_then_arithmetic += usize::from( + segments.get(at + 1).is_some_and(|&(next, _, _)| !raw(next)), + ); + } + found.empty += usize::from(passes > 0 && bytes == 0); + } + } + FOUND.with(|cell| cell.set(Some(found))); + Ok(()) + }); + image + .decode_native_with_context(&mut context) + .expect("decode fixture"); + FOUND + .with(Cell::take) + .expect("the tile route decoded the tile") + }) +} + +/// The fixtures' blocks hold every kind of codeword segment: blocks of +/// several in every fixture whose style makes them, raw segments and an +/// arithmetic segment after them in every one with selective bypass, and +/// some segments with coding passes and no bytes. +#[test] +fn fixtures_hold_every_kind_of_codeword_segment() { + let mut empty = 0; + for (label, bytes) in FIXTURES { + let image = Image::new(bytes, &DecodeSettings::default()).expect("parse fixture"); + let style = image + .header + .global_coding_style + .component_parameters + .parameters + .code_block_style; + let found = segments(&image); + assert_eq!( + found.segmented > 0, + !style.single_codeword_segment(), + "{label}: {found:?}" + ); + if style.selective_arithmetic_coding_bypass { + assert!( + found.raw > 0 && found.raw_then_arithmetic > 0, + "{label}: {found:?}" + ); + } + empty += found.empty; + } + assert!(empty > 0, "some segments hold coding passes and no bytes"); +} From 6cd63f363ccc1885c05ab479bd5cbd8feb0fe97f Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 00:14:47 -0400 Subject: [PATCH 61/94] fix(j2k-native): hold the pool threads' scratch to its charge A tile streaming its rows from pool bands whose bands failed decoded them again over whole windows to find the error while its threads' scratch was still held, though that rerun's budget leaves it out. The scratch is now released first, with the plans and the executor's records. Each thread's block scratch, for a block decoded apart from its sub-band, was allocated and grown while blocks decoded: a part of a band grew it past the largest clipped block the thread decoded first, and a streamed tile's threads serve components whose blocks may differ in size, so a growth briefly held the old and the new buffer against a charge for one. It is now reserved before the thread starts, with its payload and segment scratch, for the largest block of every route it serves, any capacity beyond that charged, and decoding never grows it. A streamed tile charged each thread's entry in the scratch list twice, with the scratch and again for the list, and failed the decode instead of handing the tile to serial bands when its budget lacked room for the list of routes, the executor's records or the scratch list. It now charges the list once, and a budget that cannot hold any of them hands the tile to serial bands. Tests: corrupt packets of the multi-segment fixtures decode on a pool as on one thread, to the same samples or error, strict and lenient, as planes and as streamed rows, and the whole-window reruns of failed pool bands hold no thread's scratch; a streamed tile's threads, workers and readers, hold exactly the bytes charged for them, through components with blocks of different sizes; under the tightest budget its pool bands decode in they do, and under a byte less, and each smaller budget tried, serial bands stream the same rows instead of failing; and a thread's scratch fits a budget of exactly its bytes and not one byte less. Measured on an 18-core ARM64 machine against the commit before it, seven rounds: full-image decodes of 47- and 25-megapixel images take their time within the run-to-run spread, and decodes on 18 threads hold about 0.3 MiB more peak heap, for each thread's block scratch reserved before it starts. --- crates/j2k-native/src/j2c/decode.rs | 16 + .../j2k-native/src/j2c/decode/banded_tier1.rs | 6 + .../src/j2c/decode/parallel_bands.rs | 60 ++- crates/j2k-native/src/j2c/decode/state.rs | 7 + .../src/j2c/decode/subband/tile_parallel.rs | 344 +++++++++++------- crates/j2k-native/src/tests/multi_segment.rs | 185 ++++++++++ 6 files changed, 479 insertions(+), 139 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode.rs b/crates/j2k-native/src/j2c/decode.rs index 2a59b3dc..ddd128d0 100644 --- a/crates/j2k-native/src/j2c/decode.rs +++ b/crates/j2k-native/src/j2c/decode.rs @@ -775,6 +775,22 @@ pub(crate) struct DecodeDebugCounters { /// over components and tiles. #[cfg(all(test, feature = "parallel"))] pub(crate) pool_band_plan_bytes: usize, + /// The most bytes the budget of a tile streaming its rows from pool + /// bands held, every route, record and thread's scratch charged. + #[cfg(all(test, feature = "parallel"))] + pub(crate) pool_band_live_bytes: usize, + /// Bytes that budget charged for its threads' scratch, and the bytes + /// that scratch held once the bands decoded, summed over tiles. + #[cfg(all(test, feature = "parallel"))] + pub(crate) thread_scratch_charged: usize, + #[cfg(all(test, feature = "parallel"))] + pub(crate) thread_scratch_held: usize, + /// Tiles whose failed bands decoded again over whole windows, and the + /// pool threads' scratches still held when they did, summed. + #[cfg(all(test, feature = "parallel"))] + pub(crate) whole_window_reruns: usize, + #[cfg(all(test, feature = "parallel"))] + pub(crate) scratches_live_in_reruns: usize, #[cfg(all(test, feature = "parallel"))] pub(crate) parallel_coefficients: ParallelCoefficientStats, } diff --git a/crates/j2k-native/src/j2c/decode/banded_tier1.rs b/crates/j2k-native/src/j2c/decode/banded_tier1.rs index f579a15c..67e4eb26 100644 --- a/crates/j2k-native/src/j2c/decode/banded_tier1.rs +++ b/crates/j2k-native/src/j2c/decode/banded_tier1.rs @@ -966,6 +966,12 @@ pub(super) fn whole_window_tile_error<'a>( storage: &mut DecompositionStorage<'a>, banded_error: DecodeError, ) -> DecodeError { + #[cfg(all(test, feature = "parallel"))] + { + let counters = &mut tile_ctx.debug_counters; + counters.whole_window_reruns += 1; + counters.scratches_live_in_reruns += super::subband::BlockScratch::live(); + } storage.coefficient_layout = build::CoefficientLayout::Component; if build::lay_out_whole_region_coefficients(storage).is_err() { return banded_error; diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index b59b0231..dae3fca9 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -232,7 +232,7 @@ fn decode_route( let mut budget = route.budget(); budget.include_bytes(executor_bytes(&rings, &bands)?)?; let mut scratches = - BlockScratch::for_threads(route.workers() + 1, route.payload_scratch(), &mut budget)?; + BlockScratch::for_threads(route.workers() + 1, route.scratch_sizes()?, &mut budget)?; tile_ctx.idwt_output.placement = crate::j2c::idwt::IdwtPlacement::Coefficients; tile_ctx.idwt_output.coefficients = Vec::new(); tile_ctx.idwt_output.rect = IntRect::from_xywh(0, 0, 0, 0); @@ -333,9 +333,23 @@ pub(super) fn decode_tile_rows<'a>( // Every component's route stays live beside the others' while the // executor runs them all, so each is planned in the budget the last one // left, as one budget, which starts with the list of routes. + // A budget too small for the pool leaves the tile to serial bands. + let declines = |result: Result<()>| match result { + Err(error) if super::reuse::is_capacity_error(&error) => Ok(true), + result => result.map(|()| false), + }; let mut live = DecodeAllocationBudget::for_storage(storage)?; + #[cfg(test)] + if let Some(cap) = tile_ctx.pool_band_cap { + match DecodeAllocationBudget::from_live_bytes_with_cap(live.live_bytes(), cap) { + Ok(capped) => live = capped, + Err(_) => return Ok(false), + } + } let mut routes = Vec::new(); - live.reserve_new(&mut routes, components)?; + if declines(live.reserve_new(&mut routes, components))? { + return Ok(false); + } for component_idx in 0..components { match plan_band_route_beside( tile, @@ -366,7 +380,9 @@ pub(super) fn decode_tile_rows<'a>( let line_plans = plan_tile_lines(tile, header, tile_ctx, storage, &active_levels)?; // The executor's records live beside every route's plan and scratch // while the bands decode. - live.include_bytes(executor_bytes(&rings, &bands)?)?; + if declines(live.include_bytes(executor_bytes(&rings, &bands)?))? { + return Ok(false); + } let empty = IntRect::from_xywh(0, 0, 0, 0); let (copy, region) = line_plans .first() @@ -394,18 +410,25 @@ pub(super) fn decode_tile_rows<'a>( 1 }; let workers = route_workers.min(bank.saturating_sub(readers)); - // Each thread's scratch serves every component's bands, so it is - // charged once, for the largest of the routes'; the serial bands need - // none. - let (thread_scratch, payload_scratch) = - BandRoute::shared_scratch(routes.iter().map(|route| &**route))?; + // Each thread's scratch serves every component's bands, so it holds, + // and is charged once for, the largest of the routes'; the serial bands + // need none. let threads = workers + readers; - let scratch_bytes = threads - .checked_mul(thread_scratch) - .ok_or(ValidationError::ImageTooLarge)?; - match live.include_bytes(scratch_bytes) { - Err(error) if super::reuse::is_capacity_error(&error) => return Ok(false), - result => result?, + #[cfg(test)] + let unscratched = live.live_bytes(); + let Some(mut scratches) = BlockScratch::charged_for_threads( + threads, + BandRoute::shared_scratch(routes.iter().map(|route| &**route))?, + &mut live, + )? + else { + return Ok(false); + }; + #[cfg(test)] + { + let counters = &mut tile_ctx.debug_counters; + counters.pool_band_live_bytes = counters.pool_band_live_bytes.max(live.live_bytes()); + counters.thread_scratch_charged += live.live_bytes() - unscratched; } // Rows reconstruct in batches, charged by the region's width (see // `roi_idwt_peak_elements`): two of them with a reader per component of @@ -439,7 +462,6 @@ pub(super) fn decode_tile_rows<'a>( .max(held.saturating_sub(storage.roi_idwt_charge_elements)); } let mut output = RowOutput::new(tile, sink, tile_ctx, copy, region); - let mut scratches = BlockScratch::for_threads(threads, payload_scratch, &mut live)?; // One reader for every component of a classic tile leaves the pool's // threads to Tier-1, and decodes bands itself while it waits. let helps = readers == 1 && coding == CodeBlockCoding::Classic; @@ -519,6 +541,10 @@ pub(super) fn decode_tile_rows<'a>( // theirs written. storage.stale_coefficients = true; tile_ctx.idwt_line_rings = buffer; + #[cfg(test)] + { + tile_ctx.debug_counters.thread_scratch_held += BlockScratch::held_bytes(&scratches); + } let streamed = output.streamed; tile_ctx.streamed_samples = tile_ctx.streamed_samples.saturating_add(streamed); tile_ctx.debug_counters.idwt_output_samples = tile_ctx @@ -540,7 +566,8 @@ pub(super) fn decode_tile_rows<'a>( Ok(completed) }); // Neither the serial bands' nor the whole-window rerun's budget includes - // the plans or the executor's records, so they go first. + // the plans, the executor's records or the threads' scratch, so they go + // first. drop(( routes, rings, @@ -549,6 +576,7 @@ pub(super) fn decode_tile_rows<'a>( active_levels, line_plans, ring_lens, + scratches, )); match result { Ok(true) => { diff --git a/crates/j2k-native/src/j2c/decode/state.rs b/crates/j2k-native/src/j2c/decode/state.rs index e186c29c..8f87307b 100644 --- a/crates/j2k-native/src/j2c/decode/state.rs +++ b/crates/j2k-native/src/j2c/decode/state.rs @@ -278,6 +278,10 @@ pub(crate) struct TileDecodeContext { /// unit tests' tiny fixtures take the pool, unless a test sets it. #[cfg(all(test, feature = "parallel"))] pub(crate) band_work_cutoff: usize, + /// Test-only decode budget for a tile streaming its rows from pool + /// bands, in place of the default. + #[cfg(all(test, feature = "parallel"))] + pub(crate) pool_band_cap: Option, /// Test-only capacity failure when building the optional band row index. #[cfg(test)] pub(crate) reject_band_index: bool, @@ -357,6 +361,8 @@ impl TileDecodeContext { let force_per_sub_band = self.force_per_sub_band; #[cfg(all(test, feature = "parallel"))] let tile_route_post_decode = self.tile_route_post_decode; + #[cfg(all(test, feature = "parallel"))] + let pool_band_cap = self.pool_band_cap; #[cfg(test)] let store_whole_code_blocks = self.store_whole_code_blocks; #[cfg(test)] @@ -382,6 +388,7 @@ impl TileDecodeContext { { self.force_per_sub_band = force_per_sub_band; self.tile_route_post_decode = tile_route_post_decode; + self.pool_band_cap = pool_band_cap; } #[cfg(test)] { diff --git a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs index 10f5bcef..75db1f31 100644 --- a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs +++ b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs @@ -45,7 +45,7 @@ use crate::scalar::{ }; use crate::{ decode_ht_code_block_scalar_with_workspace, try_reserve_decode_elements, - try_resize_decode_elements, HtCodeBlockDecodeWorkspace, J2kCodeBlockSegment, + HtCodeBlockDecodeWorkspace, J2kCodeBlockSegment, }; /// Below this many coded bytes per tile (about 45 us of serial entropy decode @@ -161,42 +161,170 @@ impl<'s, 'a> PayloadSource<'s, 'a> { /// several, which other blocks leave empty. type BlockPayload<'p> = (&'p [u8], &'p [J2kCodeBlockSegment]); -/// A worker's scratch for the blocks it decodes: a block its sub-band stores -/// only part of, a payload gathered from several segments, and the codeword -/// segments of a classic block of several. -#[derive(Default)] +/// A worker's scratch for the blocks it decodes: a block it decodes apart +/// from its sub-band, a payload gathered from several segments, and the +/// codeword segments of a classic block of several. pub(in crate::j2c::decode) struct BlockScratch { clipped: Vec, payload: Vec, segments: Vec, } +/// What each thread's [`BlockScratch`] holds: the bytes of the largest +/// payload it gathers, the codeword segments of the classic block of the +/// most, and the samples of the largest block it decodes apart from its +/// sub-band. +#[derive(Clone, Copy)] +pub(in crate::j2c::decode) struct ScratchSizes { + payload_len: usize, + segments: usize, + samples: usize, +} + +impl ScratchSizes { + /// Bytes of a thread's scratch of these sizes, besides its entry in the + /// threads' list. + fn bytes(self) -> usize { + [ + self.payload_len, + self.segments + .saturating_mul(size_of::()), + self.samples.saturating_mul(size_of::()), + ] + .into_iter() + .fold(0, usize::saturating_add) + } + + /// Bytes of a thread's scratch of these sizes with its entry in the + /// threads' list, as the workspace peak records it. + fn thread_bytes(self) -> usize { + self.bytes().saturating_add(size_of::()) + } + + /// The larger of each size of `self` and `other`. + fn max(self, other: Self) -> Self { + Self { + payload_len: self.payload_len.max(other.payload_len), + segments: self.segments.max(other.segments), + samples: self.samples.max(other.samples), + } + } + + /// Charge `budget` for the scratch of `threads` threads, besides their + /// list, which [`BlockScratch::for_threads`] charges. + fn include(self, budget: &mut DecodeAllocationBudget, threads: usize) -> Result<()> { + budget.include_bytes( + threads + .checked_mul(self.bytes()) + .ok_or(ValidationError::ImageTooLarge)?, + ) + } +} + +#[cfg(test)] +std::thread_local! { + /// Test-only: the pool threads' scratches this thread holds. + static LIVE_SCRATCHES: core::cell::Cell = const { core::cell::Cell::new(0) }; +} + impl BlockScratch { + fn new() -> Self { + #[cfg(test)] + LIVE_SCRATCHES.with(|live| live.set(live.get() + 1)); + Self { + clipped: Vec::new(), + payload: Vec::new(), + segments: Vec::new(), + } + } + /// Scratch for each of `count` threads, reserved before they start for - /// the largest payload any of them gathers, `payload_len` bytes, and the - /// most codeword segments of a block, `segments`, which the route - /// charged per thread: decoding never grows it. Charges `budget` for the - /// list, and for capacity the allocator gave beyond those. + /// `sizes`, which the route charged per thread: decoding never grows + /// it. Charges `budget` for the list, and for capacity the allocator + /// gave beyond `sizes`. pub(in crate::j2c::decode) fn for_threads( count: usize, - (payload_len, segments): (usize, usize), + sizes: ScratchSizes, budget: &mut DecodeAllocationBudget, ) -> Result> { let mut scratches = Vec::new(); budget.reserve_new(&mut scratches, count)?; for _ in 0..count { - let mut scratch = Self::default(); - try_reserve_decode_elements(&mut scratch.payload, payload_len)?; - try_reserve_decode_elements(&mut scratch.segments, segments)?; - budget.include_capacity_overage::(payload_len, scratch.payload.capacity())?; + let mut scratch = Self::new(); + try_reserve_decode_elements(&mut scratch.clipped, sizes.samples)?; + try_reserve_decode_elements(&mut scratch.payload, sizes.payload_len)?; + try_reserve_decode_elements(&mut scratch.segments, sizes.segments)?; + budget.include_capacity_overage::(sizes.samples, scratch.clipped.capacity())?; + budget.include_capacity_overage::(sizes.payload_len, scratch.payload.capacity())?; budget.include_capacity_overage::( - segments, + sizes.segments, scratch.segments.capacity(), )?; scratches.push(scratch); } Ok(scratches) } + + /// Scratch for each of `count` threads, as [`Self::for_threads`] + /// reserves it, when nothing charged it yet: charges `budget` for all of + /// it. Returns `None` when it does not fit. + pub(in crate::j2c::decode) fn charged_for_threads( + count: usize, + sizes: ScratchSizes, + budget: &mut DecodeAllocationBudget, + ) -> Result>> { + match sizes + .include(budget, count) + .and_then(|()| Self::for_threads(count, sizes, budget)) + { + Ok(scratches) => Ok(Some(scratches)), + Err(error) if crate::j2c::decode::reuse::is_capacity_error(&error) => Ok(None), + Err(error) => Err(error), + } + } + + /// Test-only: how many pool threads' scratches this thread holds. + #[cfg(test)] + pub(in crate::j2c::decode) fn live() -> usize { + LIVE_SCRATCHES.with(core::cell::Cell::get) + } + + /// Test-only: the bytes `scratches` hold, their list included. + #[cfg(test)] + pub(in crate::j2c::decode) fn held_bytes(scratches: &[Self]) -> usize { + scratches + .iter() + .fold(size_of_val(scratches), |bytes, scratch| { + [ + scratch.clipped.capacity() * size_of::(), + scratch.payload.capacity(), + scratch.segments.capacity() * size_of::(), + ] + .into_iter() + .fold(bytes, usize::saturating_add) + }) + } +} + +#[cfg(test)] +impl Drop for BlockScratch { + fn drop(&mut self) { + LIVE_SCRATCHES.with(|live| live.set(live.get() - 1)); + } +} + +/// The first `samples` of a thread's block scratch, within the capacity +/// [`BlockScratch::for_threads`] reserved: it never grows. +fn block_scratch(scratch: &mut Vec, samples: usize) -> Result<&mut [f32]> { + if samples > scratch.capacity() { + return Err(DecodingError::CodeBlockDecodeFailure.into()); + } + if scratch.len() < samples { + scratch.resize(samples, 0.0); + } + scratch + .get_mut(..samples) + .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()) } struct BlockJob<'a> { @@ -250,10 +378,14 @@ struct TilePlan<'a> { } impl TilePlan<'_> { - /// What each thread's payload scratch holds for the plan: the largest - /// gathered payload's bytes and the most codeword segments of a block. - fn payload_scratch(&self) -> (usize, usize) { - (self.max_gathered_len, self.max_segments) + /// What each thread's scratch holds for the plan, with block scratch of + /// `samples`. + fn scratch_sizes(&self, samples: usize) -> ScratchSizes { + ScratchSizes { + payload_len: self.max_gathered_len, + segments: self.max_segments, + samples, + } } } @@ -339,17 +471,12 @@ pub(in crate::j2c::decode) fn plan_tile_route<'a>( let workers = rayon::current_num_threads().min(plan.strips.len()); let counts = TaskCounts::new(&plan, workers, storage.stale_coefficients)?; let route_budget = (|| -> Result<()> { - // Each worker gathers payloads into scratch of its own as it decodes. - include_payload_scratch(&mut budget, &plan, workers)?; + // Each worker holds scratch of its own, reserved before it starts. + plan.scratch_sizes(counts.scratch_samples) + .include(&mut budget, workers)?; // `plan_tile` charged one task per strip. budget.include_elements::>(counts.tasks - plan.strips.len())?; budget.include_elements::>(counts.shared)?; - // Each worker allocates its block scratch on first use. - budget.include_elements::( - workers - .checked_mul(counts.scratch_samples) - .ok_or(ValidationError::ImageTooLarge)?, - )?; if has_later_components { // Component-at-a-time decoding retains this serial context while // preparing the next component. Do not commit a parallel bank @@ -461,7 +588,7 @@ fn record_route_peak( ) -> Result<()> { let route_bytes = [ plan_bytes(plan), - workers.saturating_mul(thread_scratch_bytes(plan, counts.scratch_samples)), + workers.saturating_mul(plan.scratch_sizes(counts.scratch_samples).thread_bytes()), counts.tasks * size_of::>(), counts.shared * size_of::>(), ] @@ -485,16 +612,6 @@ fn plan_bytes(plan: &TilePlan<'_>) -> usize { .fold(0, usize::saturating_add) } -/// Bytes a thread's scratch holds at most for `plan`, as budgeted: the -/// largest payload it gathers and the segments of its classic block of the -/// most, block scratch of `scratch_samples`, and its entry in the threads' -/// list. -fn thread_scratch_bytes(plan: &TilePlan<'_>, scratch_samples: usize) -> usize { - payload_scratch_bytes(plan) - .saturating_add(scratch_samples.saturating_mul(size_of::())) - .saturating_add(size_of::()) -} - /// Prepare one workspace per worker in the retained `bank`, then decode the /// plan's strips into `storage`'s coefficients in place through /// `decode_block`, which leaves a stale buffer current. Returns the bank's @@ -519,8 +636,8 @@ fn decode_with_bank<'a, W: TaskScratch>( ) else { return Ok(None); }; - let Ok(mut scratches) = BlockScratch::for_threads(workers, plan.payload_scratch(), budget) - else { + let sizes = plan.scratch_sizes(counts.scratch_samples); + let Ok(mut scratches) = BlockScratch::for_threads(workers, sizes, budget) else { return Ok(None); }; let mut coefficients = core::mem::take(&mut storage.coefficients); @@ -1016,28 +1133,6 @@ fn strips_are_disjoint(strips: &mut [Strip], coefficient_len: usize) -> bool { previous_end <= coefficient_len } -/// Charge the payload scratch each of `threads` threads holds to gather the -/// plan's payloads and collect their codeword segments. -fn include_payload_scratch( - budget: &mut DecodeAllocationBudget, - plan: &TilePlan<'_>, - threads: usize, -) -> Result<()> { - budget.include_bytes( - threads - .checked_mul(payload_scratch_bytes(plan)) - .ok_or(ValidationError::ImageTooLarge)?, - ) -} - -/// Bytes of a thread's scratch for the plan's largest gathered payload and -/// its classic block of the most codeword segments. -fn payload_scratch_bytes(plan: &TilePlan<'_>) -> usize { - plan.max_segments - .saturating_mul(size_of::()) - .saturating_add(plan.max_gathered_len) -} - /// Gather the payload of `job`, a block whose payload spans several /// segments, into `scratch`: for a classic block every segment, for an HT /// block the selected set's cleanup and refinement segments. @@ -1417,7 +1512,6 @@ fn decode_strips<'plan, 'coefficients, W: Send>( jobs, output, stale, - counts.scratch_samples, workspace, (&mut *scratch, source), &decode_block, @@ -1445,7 +1539,6 @@ fn decode_part( jobs: Range, output: &Mutex<&mut [f32]>, stale: bool, - scratch_samples: usize, workspace: &mut W, (scratch, source): (&mut BlockScratch, PayloadSource<'_, '_>), decode_block: &impl Fn(&BlockJob<'_>, BlockPayload<'_>, &mut [f32], usize, &mut W) -> Result<()>, @@ -1481,9 +1574,6 @@ fn decode_part( .fill(0.0); } } - if scratch.len() < scratch_samples { - try_resize_decode_elements(scratch, scratch_samples, 0.0)?; - } for job in plan .jobs .get(jobs) @@ -1494,9 +1584,7 @@ fn decode_part( target_y: 0, ..job.placement }; - let block = scratch - .get_mut(..job.width as usize * job.height as usize) - .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let block = block_scratch(scratch, job.width as usize * job.height as usize)?; if !placement.is_whole(job.width, job.height) { block.fill(0.0); } @@ -1532,14 +1620,7 @@ fn decode_strip( let (block_output, output_stride) = if whole { (&mut output[placement.target_x as usize..], strip.stride) } else { - // One worker's scratch serves the strips of every component's - // plan, so it grows to the largest clipped block of each. - if clipped_scratch.len() < plan.max_clipped_samples { - try_resize_decode_elements(clipped_scratch, plan.max_clipped_samples, 0.0)?; - } - let block = clipped_scratch - .get_mut(..job.width as usize * job.height as usize) - .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let block = block_scratch(clipped_scratch, job.width as usize * job.height as usize)?; block.fill(0.0); (block, job.width as usize) }; @@ -1853,16 +1934,9 @@ pub(in crate::j2c::decode) fn plan_band_route_beside<'a>( } let workers = threads.min(plan.strips.len()); let prepared = (|| -> Result { - // Each thread gathers payloads into scratch of its own as it - // decodes, and allocates its block scratch on first use: a clipped - // block, or any block of a band it decodes a part of. + // Each thread holds scratch of its own, reserved before it starts. if !lockstep { - include_payload_scratch(&mut budget, &plan, workers + 1)?; - budget.include_elements::( - (workers + 1) - .checked_mul(band_scratch_samples(&plan)?) - .ok_or(ValidationError::ImageTooLarge)?, - )?; + band_scratch_sizes(&plan)?.include(&mut budget, workers + 1)?; } let (width, height) = (plan.max_width, plan.max_height); let structural = &mut storage.structural_workspace_bytes; @@ -1899,13 +1973,14 @@ pub(in crate::j2c::decode) fn plan_band_route_beside<'a>( } } -/// The block scratch each thread of a band route holds: as large as any -/// block, since a part decodes every block of its run through it. -fn band_scratch_samples(plan: &TilePlan<'_>) -> Result { +/// What each thread of a band route holds for `plan`: block scratch as +/// large as any block, since a part decodes every block of its run through +/// it. +fn band_scratch_sizes(plan: &TilePlan<'_>) -> Result { let block = (plan.max_width as usize) .checked_mul(plan.max_height as usize) .ok_or(ValidationError::ImageTooLarge)?; - Ok(block.max(plan.max_clipped_samples)) + Ok(plan.scratch_sizes(block.max(plan.max_clipped_samples))) } impl BandRoute<'_> { @@ -1924,36 +1999,28 @@ impl BandRoute<'_> { self.workers } - /// What each thread's payload scratch holds for this plan: the largest - /// gathered payload's bytes and the most codeword segments of a block. - pub(in crate::j2c::decode) fn payload_scratch(&self) -> (usize, usize) { - self.plan.payload_scratch() + /// What each thread's scratch holds for this route, as its planning + /// charged it. + pub(in crate::j2c::decode) fn scratch_sizes(&self) -> Result { + band_scratch_sizes(&self.plan) } - /// What each thread's scratch holds to serve all of `routes` at once, - /// the largest of each part over them: its bytes, as budgeted, and the - /// payload scratch to reserve (see [`BlockScratch::for_threads`]). + /// What each thread's scratch holds to serve all of `routes` at once: + /// the largest of each size over them. pub(in crate::j2c::decode) fn shared_scratch<'r>( - routes: impl Iterator, - ) -> Result<(usize, (usize, usize))> + mut routes: impl Iterator, + ) -> Result where Self: 'r, { - let (mut payload_len, mut segments, mut samples) = (0, 0, 0); - for route in routes { - payload_len = payload_len.max(route.plan.max_gathered_len); - segments = segments.max(route.plan.max_segments); - samples = samples.max(band_scratch_samples(&route.plan)?); - } - let bytes = [ - payload_len, - segments.saturating_mul(size_of::()), - samples.saturating_mul(size_of::()), - size_of::(), - ] - .into_iter() - .fold(0, usize::saturating_add); - Ok((bytes, (payload_len, segments))) + routes.try_fold( + ScratchSizes { + payload_len: 0, + segments: 0, + samples: 0, + }, + |sizes, route| Ok(sizes.max(band_scratch_sizes(&route.plan)?)), + ) } pub(in crate::j2c::decode) fn strips(&self) -> impl Iterator + '_ { @@ -2061,7 +2128,6 @@ impl BandRoute<'_> { jobs, slot, true, - band_scratch_samples(plan)?, workspace, scratch, decode_block, @@ -2095,7 +2161,7 @@ impl BandRoute<'_> { let plans = routes.clone().fold(0usize, |plans, route| { plans.saturating_add(plan_bytes(&route.plan)) }); - let (scratch, _) = Self::shared_scratch(routes.clone())?; + let scratch = Self::shared_scratch(routes.clone())?.thread_bytes(); let bytes = plans.saturating_add(threads.saturating_mul(scratch)); let sample = WorkspaceBytes::sample(tile_ctx, storage)?.with_tier1(bytes); tile_ctx.debug_counters.workspace_peak.record(sample); @@ -2118,7 +2184,7 @@ mod tests { collect_segments, gather_payload, plan_block, route_fallback, split_runs, tile_route_fallback, BlockJob, BlockPlacement, BlockScratch, CodeBlock, CodeBlockCoding, CodeBlockStyle, DecodeAllocationBudget, DecompositionStorage, Layer, Payload, - PayloadSource, Segment, Strip, TaskCounts, TileRoute, + PayloadSource, ScratchSizes, Segment, Strip, TaskCounts, TileRoute, }; use crate::j2c::rect::IntRect; use crate::J2kCodeBlockSegment; @@ -2174,8 +2240,10 @@ mod tests { band: 0, }; let mut budget = DecodeAllocationBudget::from_live_bytes(0).unwrap(); - let mut scratches = BlockScratch::for_threads(1, (1500, 0), &mut budget).unwrap(); - let charged = budget.live_bytes() - size_of::(); + let mut scratches = BlockScratch::charged_for_threads(1, sizes(1500, 0, 0), &mut budget) + .unwrap() + .expect("scratch within the budget"); + assert_eq!(budget.live_bytes(), 1500 + size_of::()); let scratch = &mut scratches[0].payload; for (code_block, len) in [(0, 1000), (1, 1500)] { let gathered = gather_payload( @@ -2189,9 +2257,8 @@ mod tests { assert_eq!(gathered, len as usize); assert_eq!(scratch.capacity(), 1500); } - assert_eq!(charged, 0, "the route charged the 1500 bytes; no overage"); // A payload past the reserved scratch is a planning error, not growth. - let mut short = BlockScratch::for_threads(1, (1000, 0), &mut budget).unwrap(); + let mut short = BlockScratch::for_threads(1, sizes(1000, 0, 0), &mut budget).unwrap(); assert!(gather_payload( CodeBlockCoding::Classic, &job(1, 1500), @@ -2201,6 +2268,37 @@ mod tests { .is_err()); } + fn sizes(payload_len: usize, segments: usize, samples: usize) -> ScratchSizes { + ScratchSizes { + payload_len, + segments, + samples, + } + } + + /// Scratch for three threads, each with a 1500-byte payload, seven + /// segment descriptors and a 64 x 64 block, fits a budget of exactly its + /// bytes and their list, which it then holds, and not one byte less. + #[test] + fn thread_scratch_fits_exactly_the_bytes_it_holds() { + let held = 3 + * (1500 + + 7 * size_of::() + + 64 * 64 * size_of::() + + size_of::()); + let at = |cap: usize| { + let mut budget = DecodeAllocationBudget::from_live_bytes_with_cap(100, cap).unwrap(); + let scratches = + BlockScratch::charged_for_threads(3, sizes(1500, 7, 64 * 64), &mut budget).unwrap(); + (scratches, budget.live_bytes()) + }; + let (scratches, live) = at(100 + held); + let scratches = scratches.expect("scratch within its exact budget"); + assert_eq!(live, 100 + held); + assert_eq!(BlockScratch::held_bytes(&scratches), held); + assert!(at(100 + held - 1).0.is_none(), "one byte short"); + } + /// A classic block of several codeword segments whose bytes are one /// piece of the codestream, the rest of its segments empty, borrows its /// payload and collects every segment's descriptor: ten arithmetic @@ -2259,7 +2357,7 @@ mod tests { assert!(matches!(job.payload, Payload::Borrowed(payload) if core::ptr::eq(payload, &data))); assert_eq!(segments, 2); let mut budget = DecodeAllocationBudget::from_live_bytes(0).unwrap(); - let mut scratches = BlockScratch::for_threads(1, (0, 2), &mut budget).unwrap(); + let mut scratches = BlockScratch::for_threads(1, sizes(0, 2, 0), &mut budget).unwrap(); let BlockScratch { payload, segments: descriptors, diff --git a/crates/j2k-native/src/tests/multi_segment.rs b/crates/j2k-native/src/tests/multi_segment.rs index 79d37757..61523f72 100644 --- a/crates/j2k-native/src/tests/multi_segment.rs +++ b/crates/j2k-native/src/tests/multi_segment.rs @@ -69,6 +69,7 @@ //! termination and 32 for segmentation symbols. The samples contain no //! externally sourced image content. +use alloc::string::String; use alloc::vec::Vec; use core::cell::Cell; @@ -162,6 +163,11 @@ struct Decoded { channel_bytes: usize, /// The bytes the decode's budget held after `reset`. baseline_bytes: usize, + /// The most bytes the budget of a tile streaming from pool bands held. + pool_band_bytes: usize, + /// The bytes that budget charged for its threads' scratch, and the + /// bytes the scratch held once the bands decoded. + thread_scratch: (usize, usize), } fn context(parallelism: CpuDecodeParallelism) -> DecoderContext<'static> { @@ -204,6 +210,11 @@ fn rows_with( uncharged_line_samples: counters.uncharged_line_samples, channel_bytes: counters.workspace_peak.bytes().channels, baseline_bytes: counters.decode_baseline_bytes, + pool_band_bytes: counters.pool_band_live_bytes, + thread_scratch: ( + counters.thread_scratch_charged, + counters.thread_scratch_held, + ), } } @@ -510,3 +521,177 @@ fn fixtures_hold_every_kind_of_codeword_segment() { } assert!(empty > 0, "some segments hold coding passes and no bytes"); } + +/// How a decode of an image ended. +struct Outcome { + /// The hash of the planes' and of the streamed rows' samples, or each + /// one's error. + text: String, + failed: bool, + /// Tiles whose failed pool bands decoded again over whole windows, and + /// the pool threads' scratches still held when they did. + reruns: usize, + live_scratches: usize, +} + +fn hash(samples: impl Iterator) -> u64 { + samples.fold(0xcbf2_9ce4_8422_2325_u64, |hash, sample| { + (hash ^ u64::from(sample)).wrapping_mul(0x0100_0000_01b3) + }) +} + +/// A strict or lenient Auto decode of `bytes` on the current pool, of the +/// planes and of the rows streamed from the whole image. +fn outcome(bytes: &[u8], strict: bool) -> Outcome { + let settings = DecodeSettings { + strict, + ..DecodeSettings::default() + }; + let image = match Image::new(bytes, &settings) { + Ok(image) => image, + Err(error) => { + return Outcome { + text: format!("parse {error:?}"), + failed: true, + reruns: 0, + live_scratches: 0, + } + } + }; + let planes = image.decode_native_with_context(&mut context(CpuDecodeParallelism::Auto)); + let roi = (0, 0, image.width(), image.height()); + let mut context = context(CpuDecodeParallelism::Auto); + let mut sink = Rows { + width: roi.2 as usize, + planes: Vec::new(), + arrivals: alloc::vec![0; (roi.2 * roi.3) as usize], + }; + let rows = + image.decode_region_rows_for_integer_output_with_context(roi, &mut context, &mut sink); + let counters = context.tile_decode_context.debug_counters; + let planes = match planes { + Ok(decoded) => format!( + "planes {:016x}", + hash(decoded.data.iter().map(|&b| b.into())) + ), + Err(error) => format!("planes {error:?}"), + }; + let failed = rows.is_err(); + let rows = match rows { + Ok(()) => format!("rows {:016x}", hash(sink.planes.iter().flatten().copied())), + Err(error) => format!("rows {error:?}"), + }; + Outcome { + text: format!("{planes}, {rows}"), + failed, + reruns: counters.whole_window_reruns, + live_scratches: counters.scratches_live_in_reruns, + } +} + +/// Corrupt packets of these blocks decode on a pool as on one thread, to +/// the same samples or the same error, strict or lenient, as planes and as +/// streamed rows. A tile whose pool bands fail decodes them again over +/// whole windows to find the error, without the pool threads' scratch. +#[test] +fn corrupt_multi_segment_blocks_decode_on_a_pool_as_on_one_thread() { + let mut state = 0x9e37_79b9_7f4a_7c15_u64; + let mut next = move || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + let (one, four) = (pool(1), pool(4)); + let (mut failures, mut reruns, mut live_scratches) = (0, 0, 0); + for (label, bytes) in FIXTURES { + // Corrupt only the tile's packets, after its start of data. + let data = bytes + .windows(2) + .position(|pair| pair == [0xff, 0x93]) + .expect("start of data") + + 2; + for iteration in 0..16 { + let mut mutant = Vec::new(); + mutant.extend_from_slice(bytes); + for _ in 0..=next() % 3 { + let span = u64::try_from(mutant.len() - data).expect("fixture length"); + let at = data + usize::try_from(next() % span).expect("offset in the fixture"); + mutant[at] ^= 1 << (next() % 8); + } + for strict in [false, true] { + let expected = one.install(|| outcome(&mutant, strict)); + let pooled = four.install(|| outcome(&mutant, strict)); + assert_eq!( + pooled.text, expected.text, + "{label} iteration {iteration} strict {strict}" + ); + failures += usize::from(expected.failed); + reruns += pooled.reruns; + live_scratches += pooled.live_scratches; + } + } + } + assert!( + failures > 20 && reruns > 0, + "{failures} failures, {reruns} whole-window reruns" + ); + assert_eq!( + live_scratches, 0, + "pool threads' scratches held while failed bands decoded again" + ); +} + +/// A tile streaming its rows from pool bands holds the scratch of every +/// thread, its workers' and its readers', as charged before they start, +/// though its components' blocks may differ in size. Under the tightest +/// budget its pool bands decode in, they do; a byte short of it, and under +/// each smaller budget tried on the way, its bands decode serially +/// instead, to the same rows, rather than fail. +#[test] +fn streamed_pool_band_threads_hold_the_scratch_charged_for_them() { + for threads in [4, 12] { + pool(threads).install(|| { + for (label, bytes) in FIXTURES { + let image = Image::new(bytes, &DecodeSettings::default()).expect("parse fixture"); + let roi = (0, 0, image.width(), image.height()); + let case = format!("{label} on {threads} threads"); + let decoded = rows(&image, roi); + let components = decoded.samples.len(); + let (charged, held) = decoded.thread_scratch; + assert!( + held > 0 && charged == held, + "{case}: scratch charged {charged}, held {held}" + ); + // The pool bands' budget held `pool_band_bytes` once it + // charged the threads' scratch, last; on the way, planning + // may need more, as for the workspaces it grows from one + // component's blocks to the next's larger ones. + let pooled = |cap| { + let capped = rows_with(&image, roi, |context| { + context.tile_decode_context.pool_band_cap = Some(cap); + }); + assert_eq!(capped.samples, decoded.samples, "{case} within {cap} bytes"); + assert!(capped.streamed, "{case} within {cap} bytes"); + capped.pool_components == components + }; + let (mut short, mut fits) = (0, decoded.pool_band_bytes); + assert!(!pooled(short), "{case}"); + while !pooled(fits) { + short = fits; + fits += 1024; + assert!(fits < 2 * decoded.pool_band_bytes, "{case}: fits {fits}"); + } + while short + 1 < fits { + let cap = short + (fits - short) / 2; + if pooled(cap) { + fits = cap; + } else { + short = cap; + } + } + assert!(fits >= decoded.pool_band_bytes, "{case}: fits {fits}"); + } + }); + } +} From eabe7129f40a16a9b397aaeda2e8c1bf49c50d3f Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 01:48:00 -0400 Subject: [PATCH 62/94] test(j2k-native): check each budget step of streamed pool bands and corrupt rows by sample The test of a streamed tile's budget capped the pool bands' planning alone, so the serial bands that took the tile under a smaller budget still had the default one, and the tightest budget it searched for came from the accounting it checked. It now keeps one limit for the whole decode: the bytes beyond the decode's baseline but a given headroom are taken as if retained, which the serial bands' budget sees too. On a tile of blocks of several codeword segments, the steps the pool's bands charge are measured from what they hold: the list of routes from its entries, the plans against their tables, the executor's records from the rings and bands, and the threads' scratch from its capacities. The pool's bands take the tile within exactly their sum; a byte short of the list, of the plans, of the records or of the scratch they decline it before any row reaches the sink, and serial bands then stream the same rows, each sample once, when they fit, and fail with a capacity error when they do not. Without the decline for the list the decode fails instead, and without the charge for the records the pool's bands take the tile a byte short. The test-only cap on the pool bands' budget is gone. The corruption test compared hashes of the planes' and the rows' samples; it now compares the samples themselves, and the errors apart from them, and checks that each sample of rows that stream arrives once. --- crates/j2k-native/src/j2c/decode.rs | 4 - .../src/j2c/decode/parallel_bands.rs | 267 +++++++++++++++++- crates/j2k-native/src/j2c/decode/state.rs | 7 - crates/j2k-native/src/tests/multi_segment.rs | 124 +++----- 4 files changed, 302 insertions(+), 100 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode.rs b/crates/j2k-native/src/j2c/decode.rs index ddd128d0..abc06b4e 100644 --- a/crates/j2k-native/src/j2c/decode.rs +++ b/crates/j2k-native/src/j2c/decode.rs @@ -775,10 +775,6 @@ pub(crate) struct DecodeDebugCounters { /// over components and tiles. #[cfg(all(test, feature = "parallel"))] pub(crate) pool_band_plan_bytes: usize, - /// The most bytes the budget of a tile streaming its rows from pool - /// bands held, every route, record and thread's scratch charged. - #[cfg(all(test, feature = "parallel"))] - pub(crate) pool_band_live_bytes: usize, /// Bytes that budget charged for its threads' scratch, and the bytes /// that scratch held once the bands decoded, summed over tiles. #[cfg(all(test, feature = "parallel"))] diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index dae3fca9..b0acb32a 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -339,13 +339,6 @@ pub(super) fn decode_tile_rows<'a>( result => result.map(|()| false), }; let mut live = DecodeAllocationBudget::for_storage(storage)?; - #[cfg(test)] - if let Some(cap) = tile_ctx.pool_band_cap { - match DecodeAllocationBudget::from_live_bytes_with_cap(live.live_bytes(), cap) { - Ok(capped) => live = capped, - Err(_) => return Ok(false), - } - } let mut routes = Vec::new(); if declines(live.reserve_new(&mut routes, components))? { return Ok(false); @@ -426,9 +419,7 @@ pub(super) fn decode_tile_rows<'a>( }; #[cfg(test)] { - let counters = &mut tile_ctx.debug_counters; - counters.pool_band_live_bytes = counters.pool_band_live_bytes.max(live.live_bytes()); - counters.thread_scratch_charged += live.live_bytes() - unscratched; + tile_ctx.debug_counters.thread_scratch_charged += live.live_bytes() - unscratched; } // Rows reconstruct in batches, charged by the region's width (see // `roi_idwt_peak_elements`): two of them with a reader per component of @@ -2106,4 +2097,260 @@ mod tests { }); } } + + /// Collects the sample bits a sink receives, by component, and how often + /// each sample arrives. + struct Collected { + width: usize, + planes: Vec>, + arrivals: Vec, + } + + impl Collected { + fn new(width: u32, height: u32) -> Self { + Self { + width: width as usize, + planes: Vec::new(), + arrivals: alloc::vec![0; width as usize * height as usize], + } + } + } + + impl ComponentRowSink for Collected { + fn write_rows(&mut self, x: u32, y: u32, planes: &[crate::ComponentPlane<'_>]) { + if self.planes.is_empty() { + self.planes = alloc::vec![alloc::vec![0; self.arrivals.len()]; planes.len()]; + } + let (width, height) = planes[0].dimensions(); + for (component, plane) in planes.iter().enumerate() { + for row in 0..height as usize { + for column in 0..width as usize { + let at = (y as usize + row) * self.width + x as usize + column; + self.planes[component][at] = + plane.samples()[row * width as usize + column].to_bits(); + if component == 0 { + self.arrivals[at] += 1; + } + } + } + } + } + } + + /// The bytes a streamed tile's pool bands charge beyond the decode's + /// baseline, in the order they charge them: the list of routes, every + /// route's plan, the executor's records and the threads' scratch. + #[derive(Clone, Copy, Debug)] + struct Steps { + routes: usize, + plans: usize, + executor: usize, + scratch: usize, + } + + /// How a streamed tile decoded: on the pool's bands or, declined there, + /// on serial bands, and to what rows or capacity error. + struct Streamed { + pooled: bool, + rows: Result, + } + + /// Decode the one tile of `image` on a fresh context, streaming its rows + /// to a sink: first with room to spare, which warms the workspace bank + /// and gives the rows and the [`Steps`], measured from what the routes, + /// records and scratch hold; then again as `decode_tile` does, on the + /// pool's bands or, when they decline the tile, on serial bands, with all + /// but `headroom` bytes beyond the baseline taken as if retained, which + /// every budget of the decode sees. + fn decode_streamed(image: &Image<'_>, headroom: usize) -> (Steps, Collected, Streamed) { + let (width, height) = (image.width(), image.height()); + let mut context = DecoderContext::default(); + context.set_output_region(Some((0, 0, width, height))); + context.tile_decode_context.line_region_idwt = true; + context.tile_decode_context.stream_rows = true; + let tiles = tile::parse(&mut BitReader::new(image.codestream), &image.header, 0) + .expect("parse tile"); + let baseline = context + .reset( + &image.header, + &tiles[0], + tiles.structural_workspace_bytes(), + 0, + false, + None, + ) + .expect("prepare output planes"); + let (tile_ctx, storage) = (&mut context.tile_decode_context, &mut context.storage); + let mut warm = Collected::new(width, height); + super::super::decode_tile( + &tiles[0], + &image.header, + tile_ctx, + storage, + &mut None, + CpuDecodeParallelism::Auto, + false, + &mut DecodeProfileTimings::default(), + baseline, + super::super::TileOutput::Sink(&mut warm), + ) + .expect("decode with room to spare"); + let components = tiles[0].component_infos.len(); + assert_eq!(tile_ctx.debug_counters.pool_band_components, components); + assert!(warm.arrivals.iter().all(|&count| count == 1)); + let start = DecodeAllocationBudget::for_storage(storage) + .expect("baseline") + .live_bytes(); + let routes_bytes = components * size_of::>>(); + let mut live = DecodeAllocationBudget::for_storage(storage).expect("baseline"); + live.include_bytes(routes_bytes).expect("routes list"); + let mut routes = Vec::new(); + for component in 0..components { + let BandRouting::Route(route) = plan_band_route_beside( + &tiles[0], + component, + tile_ctx, + storage, + &image.header, + Some(live), + true, + ) + .expect("plan route") else { + panic!("the tile takes the pool's bands"); + }; + live = route.budget(); + routes.push(route); + } + let plans = live.live_bytes() - start - routes_bytes; + // With the bank warm, planning charges no workspace; it charges the + // plans' tables, and a task per strip. + let recorded = tile_ctx.debug_counters.pool_band_plan_bytes; + BandRoute::record_live(routes.iter().map(|route| &**route), 0, tile_ctx, storage) + .expect("record plans"); + let tables = tile_ctx.debug_counters.pool_band_plan_bytes - recorded; + assert!( + plans >= tables, + "planning charged {plans} for {tables} of tables" + ); + let (rings, bands, _, _) = + plan_tile_rings(&routes, storage, &image.header).expect("plan rings"); + let steps = Steps { + routes: routes_bytes, + plans, + executor: executor_bytes(&rings, &bands).expect("executor bytes"), + scratch: tile_ctx.debug_counters.thread_scratch_held, + }; + drop((routes, rings, bands)); + storage.structural_workspace_bytes += DEFAULT_MAX_DECODE_BYTES - start - headroom; + let mut sink = Collected::new(width, height); + let pooled = decode_tile_rows( + &tiles[0], + &image.header, + tile_ctx, + storage, + &mut sink, + false, + &mut DecodeProfileTimings::default(), + ) + .unwrap_or_else(|error| panic!("the pool's bands decline instead: {error:?}")); + if pooled { + return ( + steps, + warm, + Streamed { + pooled, + rows: Ok(sink), + }, + ); + } + assert!( + sink.arrivals.iter().all(|&count| count == 0), + "no row before the pool declines" + ); + storage.parallel_bands = false; + storage.banded_tier1 = true; + let rows = banded_tier1::decode_tile_rows( + &tiles[0], + &image.header, + tile_ctx, + storage, + &mut sink, + false, + &mut DecodeProfileTimings::default(), + ) + .map(|()| sink); + (steps, warm, Streamed { pooled, rows }) + } + + /// A tile of blocks of several codeword segments streams its rows from + /// pool bands under exactly the bytes they charge, the list of routes, + /// the plans, the executor's records and the threads' scratch, as their + /// sizes give them. A byte short of the list, of the plans, of the + /// records or of the scratch, the pool declines the tile, before any row + /// reaches the sink, and under the same retained bytes serial bands + /// stream the same rows when they fit and fail with a capacity error + /// when they don't. + #[test] + fn streamed_pool_bands_take_a_segmented_tile_at_exactly_their_bytes() { + let bytes = include_bytes!("../../../fixtures/multi_segment/bypass_termall_rgb_53.j2k"); + let image = Image::new(bytes, &DecodeSettings::default()).expect("parse fixture"); + rayon::ThreadPoolBuilder::new() + .num_threads(4) + .build() + .expect("build pool") + .install(|| { + let (steps, _, _) = decode_streamed(&image, DEFAULT_MAX_DECODE_BYTES / 2); + let Steps { + routes, + plans, + executor, + scratch, + } = steps; + assert!( + routes > 0 && plans > 0 && executor > 0 && scratch > 0, + "{steps:?}" + ); + let all = routes + plans + executor + scratch; + let (_, warm, exact) = decode_streamed(&image, all); + assert!(exact.pooled, "pool bands within {all} bytes: {steps:?}"); + let rows = exact.rows.expect("pool bands decode"); + assert!(rows.arrivals.iter().all(|&count| count == 1)); + assert!(rows.planes == warm.planes, "pool bands' rows"); + let mut serial = Vec::new(); + for headroom in [ + routes - 1, + routes + plans - 1, + routes + plans + executor - 1, + all - 1, + ] { + let (_, _, short) = decode_streamed(&image, headroom); + assert!( + !short.pooled, + "pool bands within {headroom} bytes: {steps:?}" + ); + match short.rows { + Ok(rows) => { + assert!(rows.arrivals.iter().all(|&count| count == 1)); + assert!(rows.planes == warm.planes, "serial rows at {headroom}"); + serial.push(true); + } + Err(error) => { + assert!( + super::super::reuse::is_capacity_error(&error), + "serial bands at {headroom}: {error:?}" + ); + serial.push(false); + } + } + } + // Serial bands need less than the pool's bands, and more than + // their list of routes. + assert_eq!(serial.first(), Some(&false), "{steps:?}"); + assert_eq!(serial.last(), Some(&true), "{steps:?}"); + assert!( + serial.windows(2).all(|pair| pair[0] <= pair[1]), + "serial bands fit more as the headroom grows: {serial:?}" + ); + }); + } } diff --git a/crates/j2k-native/src/j2c/decode/state.rs b/crates/j2k-native/src/j2c/decode/state.rs index 8f87307b..e186c29c 100644 --- a/crates/j2k-native/src/j2c/decode/state.rs +++ b/crates/j2k-native/src/j2c/decode/state.rs @@ -278,10 +278,6 @@ pub(crate) struct TileDecodeContext { /// unit tests' tiny fixtures take the pool, unless a test sets it. #[cfg(all(test, feature = "parallel"))] pub(crate) band_work_cutoff: usize, - /// Test-only decode budget for a tile streaming its rows from pool - /// bands, in place of the default. - #[cfg(all(test, feature = "parallel"))] - pub(crate) pool_band_cap: Option, /// Test-only capacity failure when building the optional band row index. #[cfg(test)] pub(crate) reject_band_index: bool, @@ -361,8 +357,6 @@ impl TileDecodeContext { let force_per_sub_band = self.force_per_sub_band; #[cfg(all(test, feature = "parallel"))] let tile_route_post_decode = self.tile_route_post_decode; - #[cfg(all(test, feature = "parallel"))] - let pool_band_cap = self.pool_band_cap; #[cfg(test)] let store_whole_code_blocks = self.store_whole_code_blocks; #[cfg(test)] @@ -388,7 +382,6 @@ impl TileDecodeContext { { self.force_per_sub_band = force_per_sub_band; self.tile_route_post_decode = tile_route_post_decode; - self.pool_band_cap = pool_band_cap; } #[cfg(test)] { diff --git a/crates/j2k-native/src/tests/multi_segment.rs b/crates/j2k-native/src/tests/multi_segment.rs index 61523f72..17167263 100644 --- a/crates/j2k-native/src/tests/multi_segment.rs +++ b/crates/j2k-native/src/tests/multi_segment.rs @@ -163,8 +163,6 @@ struct Decoded { channel_bytes: usize, /// The bytes the decode's budget held after `reset`. baseline_bytes: usize, - /// The most bytes the budget of a tile streaming from pool bands held. - pool_band_bytes: usize, /// The bytes that budget charged for its threads' scratch, and the /// bytes the scratch held once the bands decoded. thread_scratch: (usize, usize), @@ -210,7 +208,6 @@ fn rows_with( uncharged_line_samples: counters.uncharged_line_samples, channel_bytes: counters.workspace_peak.bytes().channels, baseline_bytes: counters.decode_baseline_bytes, - pool_band_bytes: counters.pool_band_live_bytes, thread_scratch: ( counters.thread_scratch_charged, counters.thread_scratch_held, @@ -522,27 +519,21 @@ fn fixtures_hold_every_kind_of_codeword_segment() { assert!(empty > 0, "some segments hold coding passes and no bytes"); } -/// How a decode of an image ended. +/// How a decode of an image ended: the planes' samples and the streamed +/// rows' samples, by component, or each one's error. struct Outcome { - /// The hash of the planes' and of the streamed rows' samples, or each - /// one's error. - text: String, - failed: bool, + planes: Result, String>, + rows: Result>, String>, /// Tiles whose failed pool bands decoded again over whole windows, and /// the pool threads' scratches still held when they did. reruns: usize, live_scratches: usize, } -fn hash(samples: impl Iterator) -> u64 { - samples.fold(0xcbf2_9ce4_8422_2325_u64, |hash, sample| { - (hash ^ u64::from(sample)).wrapping_mul(0x0100_0000_01b3) - }) -} - /// A strict or lenient Auto decode of `bytes` on the current pool, of the -/// planes and of the rows streamed from the whole image. -fn outcome(bytes: &[u8], strict: bool) -> Outcome { +/// planes and of the rows streamed from the whole image; rows that stream +/// must reach the sink once each. +fn outcome(bytes: &[u8], strict: bool, case: &str) -> Outcome { let settings = DecodeSettings { strict, ..DecodeSettings::default() @@ -551,14 +542,17 @@ fn outcome(bytes: &[u8], strict: bool) -> Outcome { Ok(image) => image, Err(error) => { return Outcome { - text: format!("parse {error:?}"), - failed: true, + planes: Err(format!("parse {error:?}")), + rows: Err(format!("parse {error:?}")), reruns: 0, live_scratches: 0, } } }; - let planes = image.decode_native_with_context(&mut context(CpuDecodeParallelism::Auto)); + let planes = image + .decode_native_with_context(&mut context(CpuDecodeParallelism::Auto)) + .map(|decoded| decoded.data) + .map_err(|error| format!("{error:?}")); let roi = (0, 0, image.width(), image.height()); let mut context = context(CpuDecodeParallelism::Auto); let mut sink = Rows { @@ -566,24 +560,20 @@ fn outcome(bytes: &[u8], strict: bool) -> Outcome { planes: Vec::new(), arrivals: alloc::vec![0; (roi.2 * roi.3) as usize], }; - let rows = - image.decode_region_rows_for_integer_output_with_context(roi, &mut context, &mut sink); + let rows = image + .decode_region_rows_for_integer_output_with_context(roi, &mut context, &mut sink) + .map(|()| { + assert!( + sink.arrivals.iter().all(|&count| count == 1), + "{case}: every sample arrives once" + ); + sink.planes + }) + .map_err(|error| format!("{error:?}")); let counters = context.tile_decode_context.debug_counters; - let planes = match planes { - Ok(decoded) => format!( - "planes {:016x}", - hash(decoded.data.iter().map(|&b| b.into())) - ), - Err(error) => format!("planes {error:?}"), - }; - let failed = rows.is_err(); - let rows = match rows { - Ok(()) => format!("rows {:016x}", hash(sink.planes.iter().flatten().copied())), - Err(error) => format!("rows {error:?}"), - }; Outcome { - text: format!("{planes}, {rows}"), - failed, + planes, + rows, reruns: counters.whole_window_reruns, live_scratches: counters.scratches_live_in_reruns, } @@ -620,13 +610,23 @@ fn corrupt_multi_segment_blocks_decode_on_a_pool_as_on_one_thread() { mutant[at] ^= 1 << (next() % 8); } for strict in [false, true] { - let expected = one.install(|| outcome(&mutant, strict)); - let pooled = four.install(|| outcome(&mutant, strict)); - assert_eq!( - pooled.text, expected.text, - "{label} iteration {iteration} strict {strict}" + let case = format!("{label} iteration {iteration} strict {strict}"); + let expected = one.install(|| outcome(&mutant, strict, &case)); + let pooled = four.install(|| outcome(&mutant, strict, &case)); + // Whole sample vectors, compared without printing them. + assert!( + pooled.planes == expected.planes, + "{case}: planes {:?}, on one thread {:?}", + pooled.planes.as_ref().err(), + expected.planes.as_ref().err() + ); + assert!( + pooled.rows == expected.rows, + "{case}: rows {:?}, on one thread {:?}", + pooled.rows.as_ref().err(), + expected.rows.as_ref().err() ); - failures += usize::from(expected.failed); + failures += usize::from(expected.rows.is_err()); reruns += pooled.reruns; live_scratches += pooled.live_scratches; } @@ -644,53 +644,19 @@ fn corrupt_multi_segment_blocks_decode_on_a_pool_as_on_one_thread() { /// A tile streaming its rows from pool bands holds the scratch of every /// thread, its workers' and its readers', as charged before they start, -/// though its components' blocks may differ in size. Under the tightest -/// budget its pool bands decode in, they do; a byte short of it, and under -/// each smaller budget tried on the way, its bands decode serially -/// instead, to the same rows, rather than fail. +/// though its components' blocks may differ in size. #[test] fn streamed_pool_band_threads_hold_the_scratch_charged_for_them() { for threads in [4, 12] { pool(threads).install(|| { for (label, bytes) in FIXTURES { let image = Image::new(bytes, &DecodeSettings::default()).expect("parse fixture"); - let roi = (0, 0, image.width(), image.height()); - let case = format!("{label} on {threads} threads"); - let decoded = rows(&image, roi); - let components = decoded.samples.len(); + let decoded = rows(&image, (0, 0, image.width(), image.height())); let (charged, held) = decoded.thread_scratch; assert!( - held > 0 && charged == held, - "{case}: scratch charged {charged}, held {held}" + decoded.pool_components == decoded.samples.len() && held > 0 && charged == held, + "{label} on {threads} threads: scratch charged {charged}, held {held}" ); - // The pool bands' budget held `pool_band_bytes` once it - // charged the threads' scratch, last; on the way, planning - // may need more, as for the workspaces it grows from one - // component's blocks to the next's larger ones. - let pooled = |cap| { - let capped = rows_with(&image, roi, |context| { - context.tile_decode_context.pool_band_cap = Some(cap); - }); - assert_eq!(capped.samples, decoded.samples, "{case} within {cap} bytes"); - assert!(capped.streamed, "{case} within {cap} bytes"); - capped.pool_components == components - }; - let (mut short, mut fits) = (0, decoded.pool_band_bytes); - assert!(!pooled(short), "{case}"); - while !pooled(fits) { - short = fits; - fits += 1024; - assert!(fits < 2 * decoded.pool_band_bytes, "{case}: fits {fits}"); - } - while short + 1 < fits { - let cap = short + (fits - short) / 2; - if pooled(cap) { - fits = cap; - } else { - short = cap; - } - } - assert!(fits >= decoded.pool_band_bytes, "{case}: fits {fits}"); } }); } From ca9c14cc1cd56e984dffac0fda7b4ff7f3ec9efd Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 01:48:14 -0400 Subject: [PATCH 63/94] perf(j2k-native): reserve HTJ2K pool band threads no whole-block scratch Each thread of a band route had its block scratch reserved for the route's largest block, as a classic band decoded in parts copies every block of its run in through it. HTJ2K bands never split into parts, and decode a block straight into their slot unless the sub-band stores only part of it, so for an HTJ2K route each thread's block scratch is now reserved, and charged, for the largest such clipped block only, and for none when the region clips no block. A test streams the rows of a three-component image in 16 x 16 blocks from pool bands: its threads hold no block scratch for the whole HTJ2K image, at most a block's for a region of it, and a whole block's for the classic one; they held a whole block's for the HTJ2K image too. Measured on an 18-core ARM64 machine against the commit before it, one fresh process per decode, build and round, seven rounds: on 18 threads a quarter-scale overview of an 18000 x 12000 HTJ2K image holds 304 KiB less peak heap, back to before the reservation, and a full decode of a 4096 x 2048 HTJ2K image 642 KiB less; classic decodes keep theirs, and times are unchanged within the run-to-run spread, with the same bytes. --- crates/j2k-native/src/j2c/decode.rs | 3 ++ .../src/j2c/decode/parallel_bands.rs | 6 ++- .../src/j2c/decode/subband/tile_parallel.rs | 30 +++++++++--- crates/j2k-native/src/tests/parallel_bands.rs | 48 +++++++++++++++++++ 4 files changed, 79 insertions(+), 8 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode.rs b/crates/j2k-native/src/j2c/decode.rs index abc06b4e..2b4d4a70 100644 --- a/crates/j2k-native/src/j2c/decode.rs +++ b/crates/j2k-native/src/j2c/decode.rs @@ -781,6 +781,9 @@ pub(crate) struct DecodeDebugCounters { pub(crate) thread_scratch_charged: usize, #[cfg(all(test, feature = "parallel"))] pub(crate) thread_scratch_held: usize, + /// The most block-scratch samples a thread of such a tile held. + #[cfg(all(test, feature = "parallel"))] + pub(crate) thread_block_samples: usize, /// Tiles whose failed bands decoded again over whole windows, and the /// pool threads' scratches still held when they did, summed. #[cfg(all(test, feature = "parallel"))] diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index b0acb32a..b6f9a310 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -534,7 +534,11 @@ pub(super) fn decode_tile_rows<'a>( tile_ctx.idwt_line_rings = buffer; #[cfg(test)] { - tile_ctx.debug_counters.thread_scratch_held += BlockScratch::held_bytes(&scratches); + let counters = &mut tile_ctx.debug_counters; + counters.thread_scratch_held += BlockScratch::held_bytes(&scratches); + counters.thread_block_samples = counters + .thread_block_samples + .max(BlockScratch::block_samples(&scratches)); } let streamed = output.streamed; tile_ctx.streamed_samples = tile_ctx.streamed_samples.saturating_add(streamed); diff --git a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs index 75db1f31..e106a511 100644 --- a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs +++ b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs @@ -289,6 +289,16 @@ impl BlockScratch { LIVE_SCRATCHES.with(core::cell::Cell::get) } + /// Test-only: the most block-scratch samples one of `scratches` holds. + #[cfg(test)] + pub(in crate::j2c::decode) fn block_samples(scratches: &[Self]) -> usize { + scratches + .iter() + .map(|scratch| scratch.clipped.capacity()) + .max() + .unwrap_or(0) + } + /// Test-only: the bytes `scratches` hold, their list included. #[cfg(test)] pub(in crate::j2c::decode) fn held_bytes(scratches: &[Self]) -> usize { @@ -1973,14 +1983,20 @@ pub(in crate::j2c::decode) fn plan_band_route_beside<'a>( } } -/// What each thread of a band route holds for `plan`: block scratch as -/// large as any block, since a part decodes every block of its run through -/// it. +/// What each thread of a band route holds for `plan`: block scratch for +/// the largest block its sub-band stores only part of and, in a classic +/// plan, whose bands may decode in parts that copy every block of their run +/// in through it, for the largest block. HTJ2K bands decode whole, their +/// whole blocks straight into their slots. fn band_scratch_sizes(plan: &TilePlan<'_>) -> Result { - let block = (plan.max_width as usize) - .checked_mul(plan.max_height as usize) - .ok_or(ValidationError::ImageTooLarge)?; - Ok(plan.scratch_sizes(block.max(plan.max_clipped_samples))) + let samples = match plan.coding { + CodeBlockCoding::Classic => (plan.max_width as usize) + .checked_mul(plan.max_height as usize) + .ok_or(ValidationError::ImageTooLarge)? + .max(plan.max_clipped_samples), + CodeBlockCoding::HighThroughput => plan.max_clipped_samples, + }; + Ok(plan.scratch_sizes(samples)) } impl BandRoute<'_> { diff --git a/crates/j2k-native/src/tests/parallel_bands.rs b/crates/j2k-native/src/tests/parallel_bands.rs index 2e8a6cd8..bdfc3037 100644 --- a/crates/j2k-native/src/tests/parallel_bands.rs +++ b/crates/j2k-native/src/tests/parallel_bands.rs @@ -221,6 +221,11 @@ fn pool_band_parts_match_whole_window_tier1_bit_for_bit() { } fn rgb_fixture() -> Vec { + rgb_fixture_coded(false) +} + +/// The noisy RGB image in 16 x 16 code-blocks, HTJ2K ones when `ht`. +fn rgb_fixture_coded(use_ht_block_coding: bool) -> Vec { let pixels = noisy_pixels(3); encode( &pixels, @@ -234,6 +239,7 @@ fn rgb_fixture() -> Vec { num_decomposition_levels: 4, code_block_width_exp: 2, code_block_height_exp: 2, + use_ht_block_coding, ..EncodeOptions::default() }, ) @@ -290,6 +296,48 @@ fn streamed_pool_bands_record_every_live_plan() { }); } +/// A tile streaming its rows from pool bands reserves each thread block +/// scratch only for the blocks it decodes apart from their band's slot: an +/// HTJ2K tile, whose bands never split into parts, at most for the blocks a +/// region clips, and none for the whole image; a classic tile, whose bands +/// may decode in parts, for a whole 16 x 16 block. +#[test] +fn streamed_pool_band_threads_reserve_block_scratch_as_their_bands_need_it() { + const BLOCK: usize = 16 * 16; + pool(4).install(|| { + for ht in [true, false] { + let bytes = rgb_fixture_coded(ht); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("parse rgb fixture"); + let held = |roi| { + let mut context = DecoderContext::default(); + context.set_cpu_decode_parallelism(CpuDecodeParallelism::Auto); + image + .decode_region_rows_for_integer_output_with_context( + roi, + &mut context, + &mut DiscardRows, + ) + .expect("decode region rows"); + let counters = context.tile_decode_context.debug_counters; + assert_eq!(counters.pool_band_components, 3, "ht={ht} {roi:?}"); + assert_eq!( + counters.thread_scratch_held, counters.thread_scratch_charged, + "ht={ht} {roi:?}" + ); + counters.thread_block_samples + }; + let whole = held((0, 0, WIDTH, HEIGHT)); + let clipped = held((10, 12, 170, 130)); + if ht { + assert_eq!(whole, 0, "HTJ2K, whole image"); + assert!(clipped <= BLOCK, "HTJ2K region: {clipped}"); + } else { + assert_eq!((whole, clipped), (BLOCK, BLOCK), "classic"); + } + } + }); +} + /// With the pool's other thread held by unrelated work, workers can't run /// until the decode is over, so the IDWT must decode every band itself. #[test] From 48740184543011190f927c14afb772c8ce214ff6 Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 01:21:20 -0500 Subject: [PATCH 64/94] fix(j2k-native): charge the band executor's records of each reader and thread The budget of a tile streaming its rows from pool bands, and of one component's pool bands, charged the executor's records per band and per ring for one reader, and nothing for the lists that hand each thread its workspace and scratch and each reader its view. On 4 threads with one reader a three-component tile held 208 bytes more than charged, and a reader per component holds its own band being read and next band in every ring. The records are now charged for the executor's workers and readers, once the tile knows them, before the threads' scratch; decoding is unchanged. The budget-step test took the records' bytes from the decoder's own sum, so a record left out of both passed. It now counts them from their types and numbers, on 4 threads with one reader and on 12 with one per component: with the per-thread lists or a band's shared output left out of the charge, the pool's bands take the tile a byte short of the bytes it holds, as they did before this change. --- .../src/j2c/decode/parallel_bands.rs | 242 +++++++++++------- 1 file changed, 155 insertions(+), 87 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index b6f9a310..e60edb91 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -230,7 +230,7 @@ fn decode_route( // the bands decode, and so does each thread's scratch, the route's // workers' and this thread's. let mut budget = route.budget(); - budget.include_bytes(executor_bytes(&rings, &bands)?)?; + budget.include_bytes(executor_bytes(&rings, &bands, (route.workers(), 1))?)?; let mut scratches = BlockScratch::for_threads(route.workers() + 1, route.scratch_sizes()?, &mut budget)?; tile_ctx.idwt_output.placement = crate::j2c::idwt::IdwtPlacement::Coefficients; @@ -371,11 +371,6 @@ pub(super) fn decode_tile_rows<'a>( let (rings, mut bands, strip_starts, active_levels) = plan_tile_rings(&routes, storage, header)?; let line_plans = plan_tile_lines(tile, header, tile_ctx, storage, &active_levels)?; - // The executor's records live beside every route's plan and scratch - // while the bands decode. - if declines(live.include_bytes(executor_bytes(&rings, &bands)?))? { - return Ok(false); - } let empty = IntRect::from_xywh(0, 0, 0, 0); let (copy, region) = line_plans .first() @@ -403,6 +398,11 @@ pub(super) fn decode_tile_rows<'a>( 1 }; let workers = route_workers.min(bank.saturating_sub(readers)); + // The executor's records live beside every route's plan and scratch + // while the bands decode. + if declines(live.include_bytes(executor_bytes(&rings, &bands, (workers, readers))?))? { + return Ok(false); + } // Each thread's scratch serves every component's bands, so it holds, // and is charged once for, the largest of the routes'; the serial bands // need none. @@ -834,29 +834,47 @@ fn plan_tile_lines( Ok(line_plans) } -/// The bytes the executor holds for `rings` and `bands`, besides the slots: -/// the records, each band's status, slot, queue entry and decoded-list entry, -/// and each ring's cursors and slot list. -fn executor_bytes(rings: &[Ring], bands: &[Band]) -> Result { +/// The bytes the executor holds for `rings` and `bands` while `workers` +/// pool threads and `readers` readers decode them, besides the slots' +/// samples: each band's record, status, held and owned slot, queue entry, +/// part counters, decoded-list entry and shared output; each ring's record, +/// cursors, slot list and slots; each reader's view, with its band being +/// read and next band in every ring, and its entry in the readers' list; +/// and each thread's workspace and scratch pair. +fn executor_bytes( + rings: &[Ring], + bands: &[Band], + (workers, readers): (usize, usize), +) -> Result { let per_band = size_of::() + size_of::() - + size_of::() + + 3 * size_of::() + 2 * size_of::>() + size_of::, usize)>>() - + 2 * size_of::() + size_of::>(); - let per_ring = size_of::() - + 3 * size_of::() - + size_of::>() - + size_of::>(); - let slots: usize = rings.iter().map(|ring| ring.slots).sum(); - bands + let per_ring = size_of::() + 2 * size_of::() + size_of::>(); + // A view holds references to its workspace, scratch and decode, + // whatever their types. + let per_reader = rings .len() - .checked_mul(per_band) - .zip(rings.len().checked_mul(per_ring)) - .and_then(|(bands, rings)| bands.checked_add(rings)) - .and_then(|bytes| bytes.checked_add(slots.checked_mul(size_of::<&mut [f32]>())?)) - .ok_or(ValidationError::ImageTooLarge.into()) + .checked_mul(size_of::() + size_of::>()) + .ok_or(ValidationError::ImageTooLarge)? + + size_of::>() + + size_of::<&mut (dyn SubBandRows + Send)>(); + let per_thread = size_of::<(&mut (), &mut BlockScratch)>(); + let slots: usize = rings.iter().map(|ring| ring.slots).sum(); + [ + bands.len().checked_mul(per_band), + rings.len().checked_mul(per_ring), + slots.checked_mul(size_of::<&mut [f32]>()), + readers.checked_mul(per_reader), + workers + .checked_add(readers) + .and_then(|threads| threads.checked_mul(per_thread)), + ] + .into_iter() + .try_fold(0usize, |bytes, part| bytes.checked_add(part?)) + .ok_or(ValidationError::ImageTooLarge.into()) } /// A sub-band's ring of band slots and its bands, in row order. @@ -2038,7 +2056,8 @@ mod tests { .expect("plan control rings"); ( route_bytes, - executor_bytes(&rings, &bands).expect("executor bytes"), + executor_bytes(&rings, &bands, (threads - 1, 1)) + .expect("executor bytes"), ) }; let headroom = route_bytes + executor / 2; @@ -2150,6 +2169,8 @@ mod tests { plans: usize, executor: usize, scratch: usize, + /// The tile's readers, beside its workers. + readers: usize, } /// How a streamed tile decoded: on the pool's bands or, declined there, @@ -2238,11 +2259,21 @@ mod tests { ); let (rings, bands, _, _) = plan_tile_rings(&routes, storage, &image.header).expect("plan rings"); + // The threads the tile streamed on: the most workers its routes + // planned that the bank holds beside its readers. + let readers = tile_ctx.debug_counters.pool_band_readers; + let route_workers = routes + .iter() + .map(|route| route.workers()) + .max() + .unwrap_or(0); + let workers = route_workers.min(tile_ctx.classic_task_workspaces.len() - readers); let steps = Steps { routes: routes_bytes, plans, - executor: executor_bytes(&rings, &bands).expect("executor bytes"), + executor: executor_records(&rings, &bands, workers, readers), scratch: tile_ctx.debug_counters.thread_scratch_held, + readers, }; drop((routes, rings, bands)); storage.structural_workspace_bytes += DEFAULT_MAX_DECODE_BYTES - start - headroom; @@ -2287,74 +2318,111 @@ mod tests { } /// A tile of blocks of several codeword segments streams its rows from - /// pool bands under exactly the bytes they charge, the list of routes, - /// the plans, the executor's records and the threads' scratch, as their - /// sizes give them. A byte short of the list, of the plans, of the - /// records or of the scratch, the pool declines the tile, before any row - /// reaches the sink, and under the same retained bytes serial bands - /// stream the same rows when they fit and fail with a capacity error - /// when they don't. + /// pool bands, with one reader or one per component, under exactly the + /// bytes they charge: the list of routes, the plans, the executor's + /// records and the threads' scratch, as their sizes give them, the + /// records' from their types and counts. A byte short of the list, of the + /// plans, of the records or of the scratch, the pool declines the tile, + /// before any row reaches the sink, and under the same retained bytes + /// serial bands stream the same rows when they fit and fail with a + /// capacity error when they don't. #[test] fn streamed_pool_bands_take_a_segmented_tile_at_exactly_their_bytes() { let bytes = include_bytes!("../../../fixtures/multi_segment/bypass_termall_rgb_53.j2k"); let image = Image::new(bytes, &DecodeSettings::default()).expect("parse fixture"); - rayon::ThreadPoolBuilder::new() - .num_threads(4) - .build() - .expect("build pool") - .install(|| { - let (steps, _, _) = decode_streamed(&image, DEFAULT_MAX_DECODE_BYTES / 2); - let Steps { - routes, - plans, - executor, - scratch, - } = steps; - assert!( - routes > 0 && plans > 0 && executor > 0 && scratch > 0, - "{steps:?}" - ); - let all = routes + plans + executor + scratch; - let (_, warm, exact) = decode_streamed(&image, all); - assert!(exact.pooled, "pool bands within {all} bytes: {steps:?}"); - let rows = exact.rows.expect("pool bands decode"); - assert!(rows.arrivals.iter().all(|&count| count == 1)); - assert!(rows.planes == warm.planes, "pool bands' rows"); - let mut serial = Vec::new(); - for headroom in [ - routes - 1, - routes + plans - 1, - routes + plans + executor - 1, - all - 1, - ] { - let (_, _, short) = decode_streamed(&image, headroom); + // One reader for the tile on 4 threads, one per component on 12. + for (threads, expected_readers) in [(4, 1), (12, 3)] { + rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("build pool") + .install(|| take_at_exactly_their_bytes(&image, expected_readers)); + } + } + + fn take_at_exactly_their_bytes(image: &Image<'_>, expected_readers: usize) { + let (steps, _, _) = decode_streamed(image, DEFAULT_MAX_DECODE_BYTES / 2); + let Steps { + routes, + plans, + executor, + scratch, + readers, + } = steps; + assert_eq!(readers, expected_readers, "{steps:?}"); + assert!( + routes > 0 && plans > 0 && executor > 0 && scratch > 0, + "{steps:?}" + ); + let all = routes + plans + executor + scratch; + let (_, warm, exact) = decode_streamed(image, all); + assert!(exact.pooled, "pool bands within {all} bytes: {steps:?}"); + let rows = exact.rows.expect("pool bands decode"); + assert!(rows.arrivals.iter().all(|&count| count == 1)); + assert!(rows.planes == warm.planes, "pool bands' rows"); + let mut serial = Vec::new(); + for headroom in [ + routes - 1, + routes + plans - 1, + routes + plans + executor - 1, + all - 1, + ] { + let (_, _, short) = decode_streamed(image, headroom); + assert!( + !short.pooled, + "pool bands within {headroom} bytes: {steps:?}" + ); + match short.rows { + Ok(rows) => { + assert!(rows.arrivals.iter().all(|&count| count == 1)); + assert!(rows.planes == warm.planes, "serial rows at {headroom}"); + serial.push(true); + } + Err(error) => { assert!( - !short.pooled, - "pool bands within {headroom} bytes: {steps:?}" + super::super::reuse::is_capacity_error(&error), + "serial bands at {headroom}: {error:?}" ); - match short.rows { - Ok(rows) => { - assert!(rows.arrivals.iter().all(|&count| count == 1)); - assert!(rows.planes == warm.planes, "serial rows at {headroom}"); - serial.push(true); - } - Err(error) => { - assert!( - super::super::reuse::is_capacity_error(&error), - "serial bands at {headroom}: {error:?}" - ); - serial.push(false); - } - } + serial.push(false); } - // Serial bands need less than the pool's bands, and more than - // their list of routes. - assert_eq!(serial.first(), Some(&false), "{steps:?}"); - assert_eq!(serial.last(), Some(&true), "{steps:?}"); - assert!( - serial.windows(2).all(|pair| pair[0] <= pair[1]), - "serial bands fit more as the headroom grows: {serial:?}" - ); - }); + } + } + // Serial bands need less than the pool's bands, and more than + // their list of routes. + assert_eq!(serial.first(), Some(&false), "{steps:?}"); + assert_eq!(serial.last(), Some(&true), "{steps:?}"); + assert!( + serial.windows(2).all(|pair| pair[0] <= pair[1]), + "serial bands fit more as the headroom grows: {serial:?}" + ); + } + + /// The bytes of the records the band executor allocates while `workers` + /// and `readers` threads decode `bands` of `rings`, from their types: per + /// band its record, status, held and owned slot, readiness entry, part + /// counters, decoded entry and shared output; per ring its record, + /// readiness and freed counters, slot list and slots, and each reader's + /// band being read and next band; per thread its workspace pair; per + /// reader its view and its entry in the readers' list. + fn executor_records(rings: &[Ring], bands: &[Band], workers: usize, readers: usize) -> usize { + let per_band = size_of::() + + size_of::() + + 2 * size_of::>() + + size_of::, usize)>>() + + 3 * size_of::() + + size_of::>(); + let per_ring = size_of::() + + 2 * size_of::() + + size_of::>() + + readers * (size_of::>() + size_of::()); + let slots: usize = rings.iter().map(|ring| ring.slots).sum(); + let per_thread = size_of::<(&mut (), &mut BlockScratch)>(); + let per_reader = size_of::>() + + size_of::<&mut (dyn crate::j2c::idwt::SubBandRows + Send)>(); + bands.len() * per_band + + rings.len() * per_ring + + slots * size_of::<&mut [f32]>() + + (workers + readers) * per_thread + + readers * per_reader } } From b4a5d2c8d99f5e5d90c739139389ed54f7b9de1f Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 18:28:01 -0400 Subject: [PATCH 65/94] test(j2k-alloc-probe): track the live heap bytes and their peak The counting allocator now also keeps the bytes live at any moment and their high water, always on and independent of measurements, behind three safe functions: `live_bytes`, `reset_peak` and `peak_bytes`. Probes that bound working memory (rather than allocation calls) read the peak a decode reaches above the bytes live before it, pool threads included. The bookkeeping is one atomic add or subtract per allocator call plus a `fetch_max` on growth; no new unsafe code. A self-test checks allocation, growth by reallocation, frees and window resets. --- crates/j2k-alloc-probe/src/lib.rs | 54 +++++++++++++++++++++++++++- crates/j2k-alloc-probe/tests/main.rs | 44 ++++++++++++++++++++++- 2 files changed, 96 insertions(+), 2 deletions(-) diff --git a/crates/j2k-alloc-probe/src/lib.rs b/crates/j2k-alloc-probe/src/lib.rs index a71c17de..451e065e 100644 --- a/crates/j2k-alloc-probe/src/lib.rs +++ b/crates/j2k-alloc-probe/src/lib.rs @@ -8,9 +8,13 @@ //! delayed allocator call from reporting into a later measurement, and each //! measurement waits for already-admitted reporters before taking its //! snapshot. +//! +//! The allocator also keeps the bytes live at any moment and their high +//! water, always on and independent of measurements, for probes that bound +//! working memory rather than allocation calls (see [`reset_peak`]). use std::alloc::{GlobalAlloc, Layout, System}; -use std::sync::atomic::{AtomicU64, AtomicU8, Ordering}; +use std::sync::atomic::{AtomicU64, AtomicU8, AtomicUsize, Ordering}; #[cfg(test)] use std::sync::atomic::AtomicBool; @@ -33,6 +37,8 @@ static ALLOCATIONS: AtomicU64 = AtomicU64::new(0); static REALLOCATIONS: AtomicU64 = AtomicU64::new(0); static DEALLOCATIONS: AtomicU64 = AtomicU64::new(0); static REQUESTED_BYTES: AtomicU64 = AtomicU64::new(0); +static LIVE_BYTES: AtomicUsize = AtomicUsize::new(0); +static PEAK_BYTES: AtomicUsize = AtomicUsize::new(0); #[cfg(test)] static TEST_HOOK: AtomicU8 = AtomicU8::new(TEST_HOOK_IDLE); @@ -60,6 +66,7 @@ unsafe impl GlobalAlloc for CountingAllocator { let pointer = unsafe { System.alloc(layout) }; if !pointer.is_null() { reporter.record_allocation(layout.size()); + grow_live(layout.size()); } pointer } @@ -70,6 +77,7 @@ unsafe impl GlobalAlloc for CountingAllocator { let pointer = unsafe { System.alloc_zeroed(layout) }; if !pointer.is_null() { reporter.record_allocation(layout.size()); + grow_live(layout.size()); } pointer } @@ -77,6 +85,7 @@ unsafe impl GlobalAlloc for CountingAllocator { unsafe fn dealloc(&self, pointer: *mut u8, layout: Layout) { let reporter = Reporter::begin(); reporter.record_deallocation(); + shrink_live(layout.size()); // SAFETY: forwards the pointer and its original layout unchanged. unsafe { System.dealloc(pointer, layout) }; } @@ -88,6 +97,11 @@ unsafe impl GlobalAlloc for CountingAllocator { let replacement = unsafe { System.realloc(pointer, layout, new_size) }; if !replacement.is_null() { reporter.record_reallocation(new_size); + if new_size >= layout.size() { + grow_live(new_size - layout.size()); + } else { + shrink_live(layout.size() - new_size); + } } replacement } @@ -191,6 +205,44 @@ pub fn assert_allocations(label: &str, budget: Budget, operation: impl FnOnce result } +/// Bytes held through the global allocator right now, by every thread. +#[must_use] +pub fn live_bytes() -> usize { + LIVE_BYTES.load(Ordering::Relaxed) +} + +/// Start a new high-water window at the bytes live now and return them. +/// +/// The window is process-wide like a measurement: allocations on every +/// thread raise [`peak_bytes`], so unrelated threads must stay idle while it +/// is open. Windows need no guard and never block the allocator. +pub fn reset_peak() -> usize { + let live = LIVE_BYTES.load(Ordering::Relaxed); + PEAK_BYTES.store(live, Ordering::Relaxed); + // An allocation between the load and the store raised the live bytes + // past `live`; keep it in the new window. + PEAK_BYTES.fetch_max(LIVE_BYTES.load(Ordering::Relaxed), Ordering::Relaxed); + live +} + +/// The most bytes live at once since the last [`reset_peak`] (or since the +/// process started). +#[must_use] +pub fn peak_bytes() -> usize { + PEAK_BYTES.load(Ordering::Relaxed) +} + +fn grow_live(bytes: usize) { + let live = LIVE_BYTES + .fetch_add(bytes, Ordering::Relaxed) + .wrapping_add(bytes); + PEAK_BYTES.fetch_max(live, Ordering::Relaxed); +} + +fn shrink_live(bytes: usize) { + LIVE_BYTES.fetch_sub(bytes, Ordering::Relaxed); +} + struct MeasurementGuard { finished: bool, } diff --git a/crates/j2k-alloc-probe/tests/main.rs b/crates/j2k-alloc-probe/tests/main.rs index 2886631e..6e2ffeac 100644 --- a/crates/j2k-alloc-probe/tests/main.rs +++ b/crates/j2k-alloc-probe/tests/main.rs @@ -4,7 +4,7 @@ use std::hint::black_box; use std::panic::{catch_unwind, AssertUnwindSafe}; use std::sync::{mpsc, OnceLock}; -use j2k_alloc_probe::{assert_allocations, measure, Budget}; +use j2k_alloc_probe::{assert_allocations, live_bytes, measure, peak_bytes, reset_peak, Budget}; use j2k_native::{ decode_ht_code_block_scalar_with_workspace, encode, encode_ht_code_block_scalar_with_passes_and_workspace, @@ -32,6 +32,10 @@ fn main() { harness_counts_allocation_and_deallocation, ), ("harness_counts_reallocation", harness_counts_reallocation), + ( + "harness_tracks_live_bytes_and_their_peak", + harness_tracks_live_bytes_and_their_peak, + ), ( "preexisting_deallocation_cannot_credit_budget", preexisting_deallocation_cannot_credit_budget, @@ -131,6 +135,44 @@ fn harness_counts_reallocation() { assert!(stats.requested_bytes >= 4096); } +fn harness_tracks_live_bytes_and_their_peak() { + const BLOCK: usize = 64 * 1024; + let base = reset_peak(); + assert_eq!(peak_bytes(), base, "a new window starts at the live bytes"); + + let first = black_box(Box::<[u8]>::new_uninit_slice(BLOCK)); + let second = black_box(Box::<[u8]>::new_uninit_slice(BLOCK / 2)); + assert_eq!(live_bytes(), base + BLOCK + BLOCK / 2); + drop(first); + assert_eq!( + live_bytes(), + base + BLOCK / 2, + "a free lowers the live bytes" + ); + assert_eq!( + peak_bytes(), + base + BLOCK + BLOCK / 2, + "a free never lowers the peak" + ); + + let mut grown: Vec = Vec::new(); + grown + .try_reserve_exact(BLOCK / 4) + .expect("small allocation must succeed"); + grown + .try_reserve_exact(2 * BLOCK) + .expect("growth allocation must succeed"); + assert_eq!(live_bytes(), base + BLOCK / 2 + grown.capacity()); + assert_eq!(peak_bytes(), base + BLOCK / 2 + grown.capacity()); + // Nothing was pushed: shrinking the empty vector frees its buffer. + grown.shrink_to_fit(); + drop(second); + assert_eq!(live_bytes(), base, "every byte allocated here was freed"); + + assert_eq!(reset_peak(), base); + assert_eq!(peak_bytes(), base, "a reset forgets the earlier peak"); +} + fn preexisting_deallocation_cannot_credit_budget() { let preexisting = black_box(Box::new([0_u8; 8192])); let violation = without_panic_output(|| { From 47ddf9fdca3080f740be74642dc40554146ea0ca Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 02:00:28 -0500 Subject: [PATCH 66/94] fix(j2k-native): charge the band executor's every allocation before it makes it Measured under the counting allocator, the band executor of a three-component tile held 9,574-12,598 bytes against a charge of 8,414-10,142 on 2 and 12 threads, and planned its rings and bands, 4,808 bytes at most, before anything was charged for them: - Merging the components' rings and bands grew the tile's lists geometrically, to room for four components of three. They are now reserved once, exactly, for every component. - Rings, bands and the rows' scales were charged, if at all, only after planning had made them. They are now charged before it, with the most planning one component holds beside them, and a budget that cannot hold them hands the tile to serial bands, as for the executor's other records. - The queue of bands to start held one entry per band, but a band puts an entry in it for each part it hands out, and a reader may start a band without taking its entry; it now holds, and is charged for, an entry per part, counted once the bands are split. - Each band decoding in parts locks its shared output, and the shared state its lock and condition variable; where the platform's locks are not futexes, each allocates its lock when first used. The pool also boxes a job for each worker the executor sends. Both are now charged. A test runs the executor under the counting allocator, in a process of its own, for classic and HTJ2K tiles, whole and in a region, with bands whole and in parts, on 2, 4 and 12 threads, with one reader and with one per component, with shallow and deep rings: what planning keeps and the most it holds stay within their charge, and so does the most the executor holds while its bands decode. The budget-step test charges planning as its own step, from the rings' and bands' capacities. --- Cargo.lock | 1 + crates/j2k-native/Cargo.toml | 2 + .../src/j2c/decode/parallel_bands.rs | 580 ++++++++++++++++-- 3 files changed, 527 insertions(+), 56 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7d8d12ad..a2b893b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2775,6 +2775,7 @@ dependencies = [ "bytemuck", "criterion", "fearless_simd", + "j2k-alloc-probe", "j2k-codec-math", "j2k-profile", "j2k-types", diff --git a/crates/j2k-native/Cargo.toml b/crates/j2k-native/Cargo.toml index 55e57c52..941beb81 100644 --- a/crates/j2k-native/Cargo.toml +++ b/crates/j2k-native/Cargo.toml @@ -37,6 +37,8 @@ j2k-types = { path = "../j2k-types", version = "=0.13.2" } [dev-dependencies] criterion = { workspace = true } +# Counts the bytes the band executor allocates, against what it charges. +j2k-alloc-probe = { path = "../j2k-alloc-probe" } # Lets SIMD kernel tests also run on the portable fallback level. fearless_simd = { workspace = true, features = ["force_support_fallback"] } diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index e60edb91..554e7d8d 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -216,6 +216,13 @@ fn decode_route( profile_enabled: bool, tier1_us: &mut u128, ) -> Result { + // The rings and bands live beside the route's plan and scratch while + // the bands decode; planning them holds their row scales beside them. + let mut budget = route.budget(); + let (planned, scales) = ring_plan_bytes(route, storage, header, component_idx)?; + let mut planning = budget; + planning.include_bytes(planned + scales)?; + budget.include_bytes(planned)?; let (rings, mut bands, active_levels) = plan_rings(route, storage, header, component_idx)?; if route.coding() == CodeBlockCoding::Classic { split_bands( @@ -226,10 +233,8 @@ fn decode_route( |band, parts| route.strip_parts(band.strip, parts), )?; } - // The executor's records live beside the route's plan and scratch while - // the bands decode, and so does each thread's scratch, the route's + // So do the executor's records and each thread's scratch, the route's // workers' and this thread's. - let mut budget = route.budget(); budget.include_bytes(executor_bytes(&rings, &bands, (route.workers(), 1))?)?; let mut scratches = BlockScratch::for_threads(route.workers() + 1, route.scratch_sizes()?, &mut budget)?; @@ -368,6 +373,14 @@ pub(super) fn decode_tile_rows<'a>( } } } + // Every component's rings and bands live beside the plans while the + // bands decode; planning each holds its own rings, bands and row scales + // beside them. + let (planned, planning) = tile_ring_plan_bytes(&routes, storage, header)?; + let mut peak = live; + if declines(peak.include_bytes(planned + planning))? || declines(live.include_bytes(planned))? { + return Ok(false); + } let (rings, mut bands, strip_starts, active_levels) = plan_tile_rings(&routes, storage, header)?; let line_plans = plan_tile_lines(tile, header, tile_ctx, storage, &active_levels)?; @@ -398,6 +411,18 @@ pub(super) fn decode_tile_rows<'a>( 1 }; let workers = route_workers.min(bank.saturating_sub(readers)); + if coding == CodeBlockCoding::Classic { + split_bands( + tile_ctx, + &rings, + &mut bands, + workers + readers, + |band, parts| { + let (route, strip) = route_strip(&routes, &strip_starts, band.strip)?; + route.strip_parts(strip, parts) + }, + )?; + } // The executor's records live beside every route's plan and scratch // while the bands decode. if declines(live.include_bytes(executor_bytes(&rings, &bands, (workers, readers))?))? { @@ -456,18 +481,6 @@ pub(super) fn decode_tile_rows<'a>( // One reader for every component of a classic tile leaves the pool's // threads to Tier-1, and decodes bands itself while it waits. let helps = readers == 1 && coding == CodeBlockCoding::Classic; - if coding == CodeBlockCoding::Classic { - split_bands( - tile_ctx, - &rings, - &mut bands, - workers + readers, - |band, parts| { - let (route, strip) = route_strip(&routes, &strip_starts, band.strip)?; - route.strip_parts(strip, parts) - }, - )?; - } #[cfg(test)] let strand_once = tile_ctx.strand_pool_band; #[cfg(not(test))] @@ -758,6 +771,19 @@ fn plan_tile_rings( header: &Header<'_>, ) -> Result { let (mut rings, mut bands) = (Vec::new(), Vec::new()); + let (mut ring_count, mut band_count) = (0usize, 0usize); + for (component_idx, route) in routes.iter().enumerate() { + let (component_rings, component_bands) = + ring_counts(route, storage, header, component_idx)?; + ring_count = ring_count.saturating_add(component_rings); + band_count = band_count.saturating_add(component_bands); + } + rings + .try_reserve_exact(ring_count) + .map_err(|_| DecodingError::HostAllocationFailed)?; + bands + .try_reserve_exact(band_count) + .map_err(|_| DecodingError::HostAllocationFailed)?; let mut strip_starts = Vec::new(); strip_starts .try_reserve_exact(routes.len()) @@ -771,12 +797,11 @@ fn plan_tile_rings( let (component_rings, component_bands, levels) = plan_rings(route, storage, header, component_idx)?; let (ring_offset, band_offset) = (rings.len(), bands.len()); - rings - .try_reserve(component_rings.len()) - .map_err(|_| DecodingError::HostAllocationFailed)?; - bands - .try_reserve(component_bands.len()) - .map_err(|_| DecodingError::HostAllocationFailed)?; + if component_rings.len() > rings.capacity() - ring_offset + || component_bands.len() > bands.capacity() - band_offset + { + return Err(DecodingError::CodeBlockDecodeFailure.into()); + } rings.extend(component_rings.into_iter().map(|ring| Ring { bands: ring.bands.start + band_offset..ring.bands.end + band_offset, ..ring @@ -835,24 +860,26 @@ fn plan_tile_lines( } /// The bytes the executor holds for `rings` and `bands` while `workers` -/// pool threads and `readers` readers decode them, besides the slots' -/// samples: each band's record, status, held and owned slot, queue entry, -/// part counters, decoded-list entry and shared output; each ring's record, -/// cursors, slot list and slots; each reader's view, with its band being -/// read and next band in every ring, and its entry in the readers' list; -/// and each thread's workspace and scratch pair. +/// pool threads and `readers` readers decode them, besides the rings and +/// bands themselves (see [`ring_plan_bytes`]) and the slots' samples: each +/// band's status, held and owned slot, part counters, decoded-list entry +/// and shared output; a queue entry for each part of each band, as many as +/// its parts put in it; each ring's cursors, slot list and slots; each +/// reader's view, with its band being read and next band in every ring, and +/// its entry in the readers' list; each thread's workspace and scratch +/// pair; the pool's job for each worker it sends; and what the shared +/// output of each band decoding in parts, the shared state's lock and its +/// condition variable allocate when first used. fn executor_bytes( rings: &[Ring], bands: &[Band], (workers, readers): (usize, usize), ) -> Result { - let per_band = size_of::() - + size_of::() + let per_band = size_of::() + 3 * size_of::() + 2 * size_of::>() - + size_of::, usize)>>() + size_of::>(); - let per_ring = size_of::() + 2 * size_of::() + size_of::>(); + let per_ring = 2 * size_of::() + size_of::>(); // A view holds references to its workspace, scratch and decode, // whatever their types. let per_reader = rings @@ -865,18 +892,128 @@ fn executor_bytes( let slots: usize = rings.iter().map(|ring| ring.slots).sum(); [ bands.len().checked_mul(per_band), + queue_entries(bands).checked_mul(size_of::()), rings.len().checked_mul(per_ring), slots.checked_mul(size_of::<&mut [f32]>()), readers.checked_mul(per_reader), workers .checked_add(readers) .and_then(|threads| threads.checked_mul(per_thread)), + workers.checked_mul(SENT_WORKER_BYTES), + bands + .iter() + .filter(|band| band.parts > 1) + .count() + .checked_add(2) + .and_then(|locks| locks.checked_mul(LOCK_BYTES)), ] .into_iter() .try_fold(0usize, |bytes, part| bytes.checked_add(part?)) .ok_or(ValidationError::ImageTooLarge.into()) } +/// An entry of the executor's queue of bands that can start or hand out a +/// part: soonest read first, then largest first, and the band. +type QueueEntry = Reverse<(i64, Reverse, usize)>; + +/// The most entries the executor's queue holds for `bands`: a band enters +/// it once when its ring has a slot for it and again for each part after +/// one it hands out, and a reader may start a band without taking its +/// entry, so a band puts as many entries in it as it has parts. +fn queue_entries(bands: &[Band]) -> usize { + bands + .iter() + .fold(0usize, |entries, band| entries.saturating_add(band.parts)) +} + +/// Bytes of the job the pool boxes for each worker the executor sends: the +/// worker's closure, its two references, with the scope it runs in and the +/// job's context. +const SENT_WORKER_BYTES: usize = 4 * size_of::(); + +/// Bytes a lock or a condition variable may allocate when first used: none +/// where the platform's locks are futexes, a boxed `pthread` mutex of 64 +/// bytes or condition variable of 48 where they are not. +const LOCK_BYTES: usize = 64; + +/// The bytes the rings and bands of the component of `route` hold once +/// planned, and the rows' scales planning them holds beside them. +fn ring_plan_bytes( + route: &BandRoute<'_>, + storage: &DecompositionStorage<'_>, + header: &Header<'_>, + component_idx: usize, +) -> Result<(usize, usize)> { + let (rings, bands) = ring_counts(route, storage, header, component_idx)?; + let planned = rings + .checked_mul(size_of::()) + .zip(bands.checked_mul(size_of::())) + .and_then(|(rings, bands)| rings.checked_add(bands)) + .ok_or(ValidationError::ImageTooLarge)?; + let scales = rings + .checked_mul(size_of::()) + .ok_or(ValidationError::ImageTooLarge)?; + Ok((planned, scales)) +} + +/// The bytes every route's rings and bands hold once [`plan_tile_rings`] +/// planned them, with each component's first strip and levels, and the +/// most it holds beside them while it plans one component: that one's own +/// rings, bands and row scales. +fn tile_ring_plan_bytes( + routes: &[Box>], + storage: &DecompositionStorage<'_>, + header: &Header<'_>, +) -> Result<(usize, usize)> { + let mut planned = routes + .len() + .checked_mul(2 * size_of::()) + .ok_or(ValidationError::ImageTooLarge)?; + let mut planning = 0usize; + for (component_idx, route) in routes.iter().enumerate() { + let (own, scales) = ring_plan_bytes(route, storage, header, component_idx)?; + planned = planned + .checked_add(own) + .ok_or(ValidationError::ImageTooLarge)?; + planning = planning.max(own.saturating_add(scales)); + } + Ok((planned, planning)) +} + +/// The rings and bands [`plan_rings`] plans for the component of `route`: +/// one ring for each sub-band it reconstructs, one band for each strip. +fn ring_counts( + route: &BandRoute<'_>, + storage: &DecompositionStorage<'_>, + header: &Header<'_>, + component_idx: usize, +) -> Result<(usize, usize)> { + let levels = active_levels(storage, header, component_idx)?; + let rings = levels + .checked_mul(3) + .and_then(|rings| rings.checked_add(1)) + .ok_or(ValidationError::ImageTooLarge)?; + Ok((rings, route.strips().count())) +} + +/// The levels the component reconstructs: its decompositions, but the +/// resolution levels the decode skips. +fn active_levels( + storage: &DecompositionStorage<'_>, + header: &Header<'_>, + component_idx: usize, +) -> Result { + storage + .tile_decompositions + .get(component_idx) + .ok_or(DecodingError::CodeBlockDecodeFailure)? + .decompositions + .len() + .checked_sub(usize::from(header.skipped_resolution_levels)) + .filter(|levels| *levels > 0) + .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()) +} + /// A sub-band's ring of band slots and its bands, in row order. struct Ring { sub_band: usize, @@ -916,12 +1053,7 @@ fn plan_rings( component_idx: usize, ) -> Result<(Vec, Vec, usize)> { let tile_decompositions = &storage.tile_decompositions[component_idx]; - let active_levels = tile_decompositions - .decompositions - .len() - .checked_sub(usize::from(header.skipped_resolution_levels)) - .filter(|levels| *levels > 0) - .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let active_levels = active_levels(storage, header, component_idx)?; let mut rings = Vec::new(); rings .try_reserve_exact(1 + 3 * active_levels) @@ -1193,7 +1325,7 @@ struct State<'c, 'w, W> { /// Bands that can start, or hand out another part, soonest read first, /// then largest first. Entries for bands with nothing left to hand out /// are skipped. - ready: BinaryHeap, usize)>>, + ready: BinaryHeap, /// Each ring's next band to make ready. next_ready: Vec, /// Parts of each band handed out, and parts still decoding or to hand @@ -1445,7 +1577,7 @@ fn run_bands( }; state .ready - .try_reserve(bands.len()) + .try_reserve_exact(queue_entries(bands)) .map_err(|_| DecodingError::HostAllocationFailed)?; state .decoded @@ -2162,11 +2294,14 @@ mod tests { /// The bytes a streamed tile's pool bands charge beyond the decode's /// baseline, in the order they charge them: the list of routes, every - /// route's plan, the executor's records and the threads' scratch. + /// route's plan, the rings and bands, with the most planning them holds + /// beside them, the executor's records and the threads' scratch. #[derive(Clone, Copy, Debug)] struct Steps { routes: usize, plans: usize, + rings: usize, + planning: usize, executor: usize, scratch: usize, /// The tile's readers, beside its workers. @@ -2257,10 +2392,16 @@ mod tests { plans >= tables, "planning charged {plans} for {tables} of tables" ); - let (rings, bands, _, _) = + let (_, planning) = + tile_ring_plan_bytes(&routes, storage, &image.header).expect("planning bytes"); + let (rings, mut bands, strip_starts, active_levels) = plan_tile_rings(&routes, storage, &image.header).expect("plan rings"); + let ring_bytes = rings.capacity() * size_of::() + + bands.capacity() * size_of::() + + (strip_starts.capacity() + active_levels.capacity()) * size_of::(); // The threads the tile streamed on: the most workers its routes - // planned that the bank holds beside its readers. + // planned that the bank holds beside its readers, which split its + // bands. let readers = tile_ctx.debug_counters.pool_band_readers; let route_workers = routes .iter() @@ -2268,14 +2409,27 @@ mod tests { .max() .unwrap_or(0); let workers = route_workers.min(tile_ctx.classic_task_workspaces.len() - readers); + split_bands( + tile_ctx, + &rings, + &mut bands, + workers + readers, + |band, parts| { + let (route, strip) = route_strip(&routes, &strip_starts, band.strip)?; + route.strip_parts(strip, parts) + }, + ) + .expect("split bands"); let steps = Steps { routes: routes_bytes, plans, + rings: ring_bytes, + planning, executor: executor_records(&rings, &bands, workers, readers), scratch: tile_ctx.debug_counters.thread_scratch_held, readers, }; - drop((routes, rings, bands)); + drop((routes, rings, bands, strip_starts, active_levels)); storage.structural_workspace_bytes += DEFAULT_MAX_DECODE_BYTES - start - headroom; let mut sink = Collected::new(width, height); let pooled = decode_tile_rows( @@ -2345,16 +2499,20 @@ mod tests { let Steps { routes, plans, + rings, + planning, executor, scratch, readers, } = steps; assert_eq!(readers, expected_readers, "{steps:?}"); assert!( - routes > 0 && plans > 0 && executor > 0 && scratch > 0, + routes > 0 && plans > 0 && rings > 0 && executor > 0 && scratch > 0, "{steps:?}" ); - let all = routes + plans + executor + scratch; + // Planning the rings holds more beside them only until the executor + // and the scratch take its place. + let all = routes + plans + rings + planning.max(executor + scratch); let (_, warm, exact) = decode_streamed(image, all); assert!(exact.pooled, "pool bands within {all} bytes: {steps:?}"); let rows = exact.rows.expect("pool bands decode"); @@ -2364,7 +2522,8 @@ mod tests { for headroom in [ routes - 1, routes + plans - 1, - routes + plans + executor - 1, + routes + plans + rings - 1, + routes + plans + rings + executor - 1, all - 1, ] { let (_, _, short) = decode_streamed(image, headroom); @@ -2399,30 +2558,339 @@ mod tests { /// The bytes of the records the band executor allocates while `workers` /// and `readers` threads decode `bands` of `rings`, from their types: per - /// band its record, status, held and owned slot, readiness entry, part - /// counters, decoded entry and shared output; per ring its record, - /// readiness and freed counters, slot list and slots, and each reader's - /// band being read and next band; per thread its workspace pair; per - /// reader its view and its entry in the readers' list. + /// band its status, held and owned slot, part counters, decoded entry + /// and shared output; a queue entry per part; per ring its readiness and + /// freed counters, slot list and slots, and each reader's band being read + /// and next band; per thread its workspace pair; per reader its view and + /// its entry in the readers' list; and the pool's job per worker and + /// what the locks of bands in parts and of the shared state allocate, + /// which `band_executor_allocates_within_its_charge` measures. fn executor_records(rings: &[Ring], bands: &[Band], workers: usize, readers: usize) -> usize { - let per_band = size_of::() - + size_of::() + let per_band = size_of::() + 2 * size_of::>() - + size_of::, usize)>>() + 3 * size_of::() + size_of::>(); - let per_ring = size_of::() - + 2 * size_of::() + let entries: usize = bands.iter().map(|band| band.parts).sum(); + let per_ring = 2 * size_of::() + size_of::>() + readers * (size_of::>() + size_of::()); let slots: usize = rings.iter().map(|ring| ring.slots).sum(); let per_thread = size_of::<(&mut (), &mut BlockScratch)>(); let per_reader = size_of::>() + size_of::<&mut (dyn crate::j2c::idwt::SubBandRows + Send)>(); + let locks = bands.iter().filter(|band| band.parts > 1).count() + 2; bands.len() * per_band + + entries * size_of::, usize)>>() + rings.len() * per_ring + slots * size_of::<&mut [f32]>() + (workers + readers) * per_thread + readers * per_reader + + workers * SENT_WORKER_BYTES + + locks * LOCK_BYTES + } + + /// What the band executor allocated for a tile, measured under the + /// counting allocator, beside what the decode charges for it. + #[derive(Debug)] + struct Measured { + /// The most planning the rings and bands held at once, and what it + /// kept, against the bytes charged for them and for the most beside + /// them. + planning: (usize, usize), + planning_charge: (usize, usize), + /// The most the executor held at once while the bands decoded, + /// against its charge. + running: usize, + running_charge: usize, + readers: usize, + parted: usize, + bands: usize, + deep: bool, + } + + /// Decode the tile of `bytes`, or its `roi`, streaming its rows on the + /// current pool, with its classic bands in `parts` parts when given: + /// once to warm the bank, then on the band executor as the streamed + /// tile drives it, under the counting allocator, reading every row of + /// every ring through one reader. + fn measure_executor( + bytes: &[u8], + roi: Option<(u32, u32, u32, u32)>, + parts: Option, + ) -> Measured { + let image = Image::new(bytes, &DecodeSettings::default()).expect("parse fixture"); + let roi = roi.unwrap_or((0, 0, image.width(), image.height())); + let mut context = DecoderContext::default(); + context.set_output_region(Some(roi)); + context.tile_decode_context.line_region_idwt = true; + context.tile_decode_context.stream_rows = true; + context.tile_decode_context.band_parts = parts; + let tiles = tile::parse(&mut BitReader::new(image.codestream), &image.header, 0) + .expect("parse tile"); + let baseline = context + .reset( + &image.header, + &tiles[0], + tiles.structural_workspace_bytes(), + 0, + false, + None, + ) + .expect("prepare output planes"); + let (tile_ctx, storage) = (&mut context.tile_decode_context, &mut context.storage); + super::super::decode_tile( + &tiles[0], + &image.header, + tile_ctx, + storage, + &mut None, + CpuDecodeParallelism::Auto, + false, + &mut DecodeProfileTimings::default(), + baseline, + super::super::TileOutput::Sink(&mut CountingSink(0)), + ) + .expect("decode with room to spare"); + let components = tiles[0].component_infos.len(); + assert_eq!(tile_ctx.debug_counters.pool_band_components, components); + let mut routes = Vec::new(); + let mut live = None; + for component in 0..components { + let BandRouting::Route(route) = plan_band_route_beside( + &tiles[0], + component, + tile_ctx, + storage, + &image.header, + live, + true, + ) + .expect("plan route") else { + panic!("the tile takes the pool's bands"); + }; + live = Some(route.budget()); + routes.push(route); + } + let planning_charge = + tile_ring_plan_bytes(&routes, storage, &image.header).expect("planning bytes"); + let base = j2k_alloc_probe::live_bytes(); + j2k_alloc_probe::reset_peak(); + let (rings, mut bands, strip_starts, active_levels) = + plan_tile_rings(&routes, storage, &image.header).expect("plan rings"); + let planning = ( + j2k_alloc_probe::peak_bytes() - base, + j2k_alloc_probe::live_bytes() - base, + ); + let line_plans = + plan_tile_lines(&tiles[0], &image.header, tile_ctx, storage, &active_levels) + .expect("plan lines"); + let copy = line_plans + .first() + .map_or(IntRect::from_xywh(0, 0, 0, 0), |plan| plan.copy); + let coding = routes[0].coding(); + let bank = match coding { + CodeBlockCoding::Classic => tile_ctx.classic_task_workspaces.len(), + CodeBlockCoding::HighThroughput => tile_ctx.ht_task_workspaces.len(), + }; + let route_workers = routes + .iter() + .map(|route| route.workers()) + .max() + .unwrap_or(0); + let samples = copy.width() as usize * copy.height() as usize; + let readers = if parallel_synthesis(components, coding, bank, route_workers, samples) { + components + } else { + 1 + }; + let workers = route_workers.min(bank - readers); + if coding == CodeBlockCoding::Classic { + split_bands( + tile_ctx, + &rings, + &mut bands, + workers + readers, + |band, parts| { + let (route, strip) = route_strip(&routes, &strip_starts, band.strip)?; + route.strip_parts(strip, parts) + }, + ) + .expect("split bands"); + } + let running_charge = + executor_bytes(&rings, &bands, (workers, readers)).expect("executor bytes"); + let mut scratches = BlockScratch::for_threads( + workers + readers, + BandRoute::shared_scratch(routes.iter().map(|route| &**route)).expect("sizes"), + &mut DecodeAllocationBudget::for_storage(storage).expect("budget"), + ) + .expect("thread scratch"); + let helps = readers == 1 && coding == CodeBlockCoding::Classic; + let mut coefficients = core::mem::take(&mut storage.coefficients); + let base = j2k_alloc_probe::live_bytes(); + j2k_alloc_probe::reset_peak(); + let slots = split_slots(&rings, &bands, storage, &mut coefficients).expect("slots"); + let source = PayloadSource::new(storage); + let read = |readers: &mut [&mut (dyn SubBandRows + Send)]| -> Result<()> { + for ring in &rings { + for y in ring.stored.y0..ring.stored.y1 { + readers[0].prepare_row(ring.sub_band, y)?; + } + } + Ok(()) + }; + let decoded = match coding { + CodeBlockCoding::Classic => run_bands( + (workers, readers, helps), + &rings, + &bands, + slots, + (&mut tile_ctx.classic_task_workspaces, &mut scratches), + |strip, output, workspace, scratch| { + let (route, strip) = route_strip(&routes, &strip_starts, strip)?; + route.decode_classic(strip, output, workspace, (scratch, source)) + }, + read, + false, + &mut 0, + false, + ), + CodeBlockCoding::HighThroughput => run_bands( + (workers, readers, helps), + &rings, + &bands, + slots, + (&mut tile_ctx.ht_task_workspaces, &mut scratches), + |strip, output, workspace, scratch| { + let (route, strip) = route_strip(&routes, &strip_starts, strip)?; + route.decode_ht(strip, output, workspace, (scratch, source)) + }, + read, + false, + &mut 0, + false, + ), + }; + let running = j2k_alloc_probe::peak_bytes() - base; + assert_eq!(decoded.ok(), Some(true), "the executor decodes every band"); + storage.coefficients = coefficients; + Measured { + planning, + planning_charge, + running, + running_charge, + readers, + parted: bands.iter().filter(|band| band.parts > 1).count(), + bands: bands.len(), + deep: storage.deep_band_rings, + } + } + + /// Under the counting allocator, the band executor holds at most what + /// the decode charges for it, planning its rings and bands and while + /// they decode: for classic and HTJ2K tiles, whole and in a region, + /// with bands in parts, on pools of 2, 4 and 12 threads, with one reader + /// and with one per component, with shallow rings and deep ones. + #[test] + fn band_executor_allocates_within_its_charge() { + const PROBE: &str = "J2K_NATIVE_BAND_EXECUTOR_PROBE"; + if std::env::var_os(PROBE).is_none() { + // The allocator's high-water mark is the process's: measure in a + // process that runs this test alone. + let name = module_path!() + .split_once("::") + .map(|(_, path)| path) + .expect("module path"); + let output = std::process::Command::new(std::env::current_exe().expect("test binary")) + .args([ + &format!("{name}::band_executor_allocates_within_its_charge"), + "--exact", + "--test-threads=1", + "--nocapture", + ]) + .env(PROBE, "1") + .output() + .expect("run the measuring process"); + let report = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.status.success(), "{report}"); + assert!(report.contains("1 passed"), "{report}"); + return; + } + let classic = include_bytes!("../../../fixtures/multi_segment/bypass_termall_rgb_53.j2k"); + let ht = ht_fixture(); + let (mut readers, mut parted, mut clipped, mut deep) = (Vec::new(), 0, false, false); + for threads in [2, 4, 12] { + rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("build pool") + .install(|| { + for (label, bytes, splits) in [ + ("classic", &classic[..], &[None, Some(4)][..]), + ("HTJ2K", &ht[..], &[None][..]), + ] { + let whole = measure_executor(bytes, None, None); + for &roi in &[None, Some((48, 48, 16, 16))] { + for &parts in splits { + let measured = measure_executor(bytes, roi, parts); + let case = format!("{label} {roi:?} parts {parts:?} on {threads}"); + let (peak, kept) = measured.planning; + let (planned, beside) = measured.planning_charge; + assert!( + kept <= planned && peak <= planned + beside, + "{case}: {measured:?}" + ); + assert!( + measured.running <= measured.running_charge, + "{case}: {measured:?}" + ); + readers.push(measured.readers); + parted += measured.parted; + clipped |= measured.bands < whole.bands; + deep |= measured.deep; + } + } + } + }); + } + assert!(readers.contains(&1) && readers.contains(&3), "{readers:?}"); + assert!( + parted > 0 && clipped && deep, + "parts {parted}, region {clipped}, deep {deep}" + ); + } + + /// A three-component 64 x 64 HTJ2K image in 16 x 16 code-blocks. + fn ht_fixture() -> Vec { + let mut state = 0x2545_f491_u32; + let pixels: Vec = (0..64 * 64 * 3) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state.to_le_bytes()[3] + }) + .collect(); + encode( + &pixels, + 64, + 64, + 3, + 8, + false, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + code_block_width_exp: 2, + code_block_height_exp: 2, + use_ht_block_coding: true, + ..EncodeOptions::default() + }, + ) + .expect("encode HTJ2K fixture") } } From 5fcca630ab72ef93ef22b78880bc646e4b2ffbb5 Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 02:29:19 -0500 Subject: [PATCH 67/94] fix(j2k-native): charge the pool's retiring jobs, first-use locks and line lists The band executor's charge left out a few allocations that the pool and the platform's locks make on its behalf, and the tile charged nothing for the streamed IDWT's per-component lists: - The pool frees a worker's job only after the worker returns, so beside the workers running or queued, each pool thread can hold the job of a worker that has finished. One job per pool thread is now charged too. - Called from outside a pool, the executor's workers queue on the pool's shared queue, which may take a new block for them; one is charged. - Where locks allocate when first used, threads that race to first use a condition variable can each allocate one. The executor now uses its condition variable once before any worker starts. - The tile route charges what each shared output's lock and its queue's lock allocate when first used, and locks each once before its workers start. - Each route's box is charged with its plan, and the streamed IDWT's plan, ring length and reconstruction of each component before they are made. The measuring test now also plans the routes under the counting allocator, reads each component's rings through its own reader, on a thread of its own in a pool, measures a decode from outside a pool, and keeps its pools up, their threads started, across every measurement. --- .../src/j2c/decode/parallel_bands.rs | 263 +++++++++++++----- crates/j2k-native/src/j2c/decode/subband.rs | 2 +- .../src/j2c/decode/subband/tile_parallel.rs | 20 ++ 3 files changed, 219 insertions(+), 66 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index 554e7d8d..c0b05a03 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -48,7 +48,7 @@ use std::sync::{Condvar, Mutex, MutexGuard}; use super::banded_tier1::{self, RowOutput}; use super::subband::{ plan_band_route, plan_band_route_beside, routes_tile, BandOutput, BandRoute, BandRouting, - BlockScratch, PayloadSource, + BlockScratch, PayloadSource, LOCK_BYTES, }; use super::workspace::{TaskScratch, TaskWorkspace}; use super::{ @@ -383,6 +383,16 @@ pub(super) fn decode_tile_rows<'a>( } let (rings, mut bands, strip_starts, active_levels) = plan_tile_rings(&routes, storage, header)?; + // So do the streamed IDWT's plans, its rings' lengths and its + // reconstructions, one of each per component. + let line_lists = components + .checked_mul( + size_of::() + size_of::() + size_of::>(), + ) + .ok_or(ValidationError::ImageTooLarge)?; + if declines(live.include_bytes(line_lists))? { + return Ok(false); + } let line_plans = plan_tile_lines(tile, header, tile_ctx, storage, &active_levels)?; let empty = IntRect::from_xywh(0, 0, 0, 0); let (copy, region) = line_plans @@ -867,9 +877,10 @@ fn plan_tile_lines( /// its parts put in it; each ring's cursors, slot list and slots; each /// reader's view, with its band being read and next band in every ring, and /// its entry in the readers' list; each thread's workspace and scratch -/// pair; the pool's job for each worker it sends; and what the shared -/// output of each band decoding in parts, the shared state's lock and its -/// condition variable allocate when first used. +/// pair; the pool's job for each worker running or queued, and for each +/// pool thread whose worker finished before the pool freed its job; and +/// what the shared output of each band decoding in parts, the shared +/// state's lock and its condition variable allocate when first used. fn executor_bytes( rings: &[Ring], bands: &[Band], @@ -899,7 +910,16 @@ fn executor_bytes( workers .checked_add(readers) .and_then(|threads| threads.checked_mul(per_thread)), - workers.checked_mul(SENT_WORKER_BYTES), + workers + .checked_add(rayon::current_num_threads()) + .and_then(|jobs| jobs.checked_mul(SENT_WORKER_BYTES)), + // Called from outside the pool, its workers queue on the pool's + // shared queue, which may take a new block for them. + Some(if rayon::current_thread_index().is_none() { + INJECTED_BLOCK_BYTES + } else { + 0 + }), bands .iter() .filter(|band| band.parts > 1) @@ -931,10 +951,9 @@ fn queue_entries(bands: &[Band]) -> usize { /// job's context. const SENT_WORKER_BYTES: usize = 4 * size_of::(); -/// Bytes a lock or a condition variable may allocate when first used: none -/// where the platform's locks are futexes, a boxed `pthread` mutex of 64 -/// bytes or condition variable of 48 where they are not. -const LOCK_BYTES: usize = 64; +/// Bytes of a block of the pool's shared queue: 63 jobs of two words, each +/// with its state, and a link. +const INJECTED_BLOCK_BYTES: usize = 64 * 3 * size_of::(); /// The bytes the rings and bands of the component of `route` hold once /// planned, and the rows' scales planning them holds beside them. @@ -1593,6 +1612,10 @@ fn run_bands( state: Mutex::new(state), decoded: Condvar::new(), }; + // Where locks allocate when first used, the state's is first locked + // below, and the condition variable is first used here, before any + // worker races to allocate it. + shared.decoded.notify_all(); { let mut state = shared.lock()?; for ring in 0..rings.len() { @@ -2302,6 +2325,9 @@ mod tests { plans: usize, rings: usize, planning: usize, + /// The streamed IDWT's plan, ring length and reconstruction of each + /// component. + lines: usize, executor: usize, scratch: usize, /// The tile's readers, beside its workers. @@ -2425,6 +2451,10 @@ mod tests { plans, rings: ring_bytes, planning, + lines: components + * (size_of::() + + size_of::() + + size_of::>()), executor: executor_records(&rings, &bands, workers, readers), scratch: tile_ctx.debug_counters.thread_scratch_held, readers, @@ -2501,6 +2531,7 @@ mod tests { plans, rings, planning, + lines, executor, scratch, readers, @@ -2512,7 +2543,7 @@ mod tests { ); // Planning the rings holds more beside them only until the executor // and the scratch take its place. - let all = routes + plans + rings + planning.max(executor + scratch); + let all = routes + plans + rings + planning.max(lines + executor + scratch); let (_, warm, exact) = decode_streamed(image, all); assert!(exact.pooled, "pool bands within {all} bytes: {steps:?}"); let rows = exact.rows.expect("pool bands decode"); @@ -2523,7 +2554,8 @@ mod tests { routes - 1, routes + plans - 1, routes + plans + rings - 1, - routes + plans + rings + executor - 1, + routes + plans + rings + lines - 1, + routes + plans + rings + lines + executor - 1, all - 1, ] { let (_, _, short) = decode_streamed(image, headroom); @@ -2562,9 +2594,11 @@ mod tests { /// and shared output; a queue entry per part; per ring its readiness and /// freed counters, slot list and slots, and each reader's band being read /// and next band; per thread its workspace pair; per reader its view and - /// its entry in the readers' list; and the pool's job per worker and - /// what the locks of bands in parts and of the shared state allocate, - /// which `band_executor_allocates_within_its_charge` measures. + /// its entry in the readers' list; and the pool's job per worker and per + /// pool thread that may retire one, and what the locks of bands in parts + /// and of the shared state allocate, which + /// `band_executor_allocates_within_its_charge` measures. It decodes in a + /// pool, so its workers take no block of the pool's shared queue. fn executor_records(rings: &[Ring], bands: &[Band], workers: usize, readers: usize) -> usize { let per_band = size_of::() + 2 * size_of::>() @@ -2585,7 +2619,7 @@ mod tests { + slots * size_of::<&mut [f32]>() + (workers + readers) * per_thread + readers * per_reader - + workers * SENT_WORKER_BYTES + + (workers + rayon::current_num_threads()) * SENT_WORKER_BYTES + locks * LOCK_BYTES } @@ -2593,6 +2627,10 @@ mod tests { /// counting allocator, beside what the decode charges for it. #[derive(Debug)] struct Measured { + /// The most planning the routes held at once, and what they kept, + /// against what their budgets charged. + routing: (usize, usize), + routing_charge: usize, /// The most planning the rings and bands held at once, and what it /// kept, against the bytes charged for them and for the most beside /// them. @@ -2608,11 +2646,48 @@ mod tests { deep: bool, } + /// Read every row of the rings that `ring_readers` gives to `readers`, + /// the first of which is reader `first`: each reader on a thread of its + /// own in a pool, and one after another outside one, where handing them + /// to the pool would allocate. + fn read_rings( + rings: &[Ring], + ring_readers: &[usize], + readers: &mut [&mut (dyn SubBandRows + Send)], + first: usize, + ) -> Result<()> { + let Some((reader, rest)) = readers.split_first_mut() else { + return Ok(()); + }; + let mut read = || -> Result<()> { + for (ring, _) in rings + .iter() + .zip(ring_readers) + .filter(|(_, reader)| **reader == first) + { + for y in ring.stored.y0..ring.stored.y1 { + reader.prepare_row(ring.sub_band, y)?; + } + } + Ok(()) + }; + if rest.is_empty() { + return read(); + } + let mut next = || read_rings(rings, ring_readers, rest, first + 1); + if rayon::current_thread_index().is_none() { + return read().and_then(|()| next()); + } + let (mine, theirs) = rayon::join(read, next); + mine.and(theirs) + } + /// Decode the tile of `bytes`, or its `roi`, streaming its rows on the /// current pool, with its classic bands in `parts` parts when given: - /// once to warm the bank, then on the band executor as the streamed - /// tile drives it, under the counting allocator, reading every row of - /// every ring through one reader. + /// once to warm the bank, then, under the counting allocator, planning + /// its routes, rings and bands and running the band executor as the + /// streamed tile drives it, reading every row of every ring, each + /// component's through its own reader when it has one. fn measure_executor( bytes: &[u8], roi: Option<(u32, u32, u32, u32)>, @@ -2653,8 +2728,11 @@ mod tests { .expect("decode with room to spare"); let components = tiles[0].component_infos.len(); assert_eq!(tile_ctx.debug_counters.pool_band_components, components); - let mut routes = Vec::new(); - let mut live = None; + let mut routes = Vec::with_capacity(components); + let mut live = DecodeAllocationBudget::for_storage(storage).expect("baseline"); + let start = live.live_bytes(); + let base = j2k_alloc_probe::live_bytes(); + j2k_alloc_probe::reset_peak(); for component in 0..components { let BandRouting::Route(route) = plan_band_route_beside( &tiles[0], @@ -2662,15 +2740,20 @@ mod tests { tile_ctx, storage, &image.header, - live, + Some(live), true, ) .expect("plan route") else { panic!("the tile takes the pool's bands"); }; - live = Some(route.budget()); + live = route.budget(); routes.push(route); } + let routing = ( + j2k_alloc_probe::peak_bytes() - base, + j2k_alloc_probe::live_bytes() - base, + ); + let routing_charge = live.live_bytes() - start; let planning_charge = tile_ring_plan_bytes(&routes, storage, &image.header).expect("planning bytes"); let base = j2k_alloc_probe::live_bytes(); @@ -2727,18 +2810,26 @@ mod tests { .expect("thread scratch"); let helps = readers == 1 && coding == CodeBlockCoding::Classic; let mut coefficients = core::mem::take(&mut storage.coefficients); + // Each component's rings read through its own reader, when it has + // one. + let mut ring_readers = Vec::new(); + for (component_idx, route) in routes.iter().enumerate() { + let (component_rings, _) = + ring_counts(route, storage, &image.header, component_idx).expect("rings"); + let reader = if readers == components { + component_idx + } else { + 0 + }; + ring_readers.extend(core::iter::repeat_n(reader, component_rings)); + } + let read = |readers: &mut [&mut (dyn SubBandRows + Send)]| -> Result<()> { + read_rings(&rings, &ring_readers, readers, 0) + }; let base = j2k_alloc_probe::live_bytes(); j2k_alloc_probe::reset_peak(); let slots = split_slots(&rings, &bands, storage, &mut coefficients).expect("slots"); let source = PayloadSource::new(storage); - let read = |readers: &mut [&mut (dyn SubBandRows + Send)]| -> Result<()> { - for ring in &rings { - for y in ring.stored.y0..ring.stored.y1 { - readers[0].prepare_row(ring.sub_band, y)?; - } - } - Ok(()) - }; let decoded = match coding { CodeBlockCoding::Classic => run_bands( (workers, readers, helps), @@ -2775,6 +2866,8 @@ mod tests { assert_eq!(decoded.ok(), Some(true), "the executor decodes every band"); storage.coefficients = coefficients; Measured { + routing, + routing_charge, planning, planning_charge, running, @@ -2786,11 +2879,24 @@ mod tests { } } + /// Wait until the process has held the same bytes for a while: until + /// idle threads have finished starting. + fn settle() { + let (mut last, mut still) = (j2k_alloc_probe::live_bytes(), 0); + while still < 20 { + std::thread::sleep(std::time::Duration::from_millis(5)); + let now = j2k_alloc_probe::live_bytes(); + still = if now == last { still + 1 } else { 0 }; + last = now; + } + } + /// Under the counting allocator, the band executor holds at most what - /// the decode charges for it, planning its rings and bands and while - /// they decode: for classic and HTJ2K tiles, whole and in a region, - /// with bands in parts, on pools of 2, 4 and 12 threads, with one reader - /// and with one per component, with shallow rings and deep ones. + /// the decode charges for it, planning its routes, rings and bands and + /// while they decode: for classic and HTJ2K tiles, whole and in a + /// region, with bands in parts, on pools of 2, 4 and 12 threads and + /// from outside a pool, with one reader and with one per component, + /// with shallow rings and deep ones. #[test] fn band_executor_allocates_within_its_charge() { const PROBE: &str = "J2K_NATIVE_BAND_EXECUTOR_PROBE"; @@ -2823,40 +2929,67 @@ mod tests { let classic = include_bytes!("../../../fixtures/multi_segment/bypass_termall_rgb_53.j2k"); let ht = ht_fixture(); let (mut readers, mut parted, mut clipped, mut deep) = (Vec::new(), 0, false, false); - for threads in [2, 4, 12] { - rayon::ThreadPoolBuilder::new() - .num_threads(threads) - .build() - .expect("build pool") - .install(|| { - for (label, bytes, splits) in [ - ("classic", &classic[..], &[None, Some(4)][..]), - ("HTJ2K", &ht[..], &[None][..]), - ] { - let whole = measure_executor(bytes, None, None); - for &roi in &[None, Some((48, 48, 16, 16))] { - for &parts in splits { - let measured = measure_executor(bytes, roi, parts); - let case = format!("{label} {roi:?} parts {parts:?} on {threads}"); - let (peak, kept) = measured.planning; - let (planned, beside) = measured.planning_charge; - assert!( - kept <= planned && peak <= planned + beside, - "{case}: {measured:?}" - ); - assert!( - measured.running <= measured.running_charge, - "{case}: {measured:?}" - ); - readers.push(measured.readers); - parted += measured.parted; - clipped |= measured.bands < whole.bands; - deep |= measured.deep; - } + let mut check = |case: &str, measured: Measured, whole: &Measured| { + assert!( + measured.routing.0 <= measured.routing_charge, + "{case}: {measured:?}" + ); + let (peak, kept) = measured.planning; + let (planned, beside) = measured.planning_charge; + assert!( + kept <= planned && peak <= planned + beside, + "{case}: {measured:?}" + ); + assert!( + measured.running <= measured.running_charge, + "{case}: {measured:?}" + ); + readers.push(measured.readers); + parted += measured.parted; + clipped |= measured.bands < whole.bands; + deep |= measured.deep; + }; + // The pools, and the global one, stay up until every measurement + // ends, so no pool's teardown frees bytes inside another's window, + // and every measurement waits until their threads have started, + // which allocates their queues' state and their locks. + let pools: Vec<_> = [2, 4, 12] + .into_iter() + .map(|threads| { + rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("build pool") + }) + .collect(); + assert!(rayon::current_num_threads() > 0); + settle(); + for pool in &pools { + let threads = pool.current_num_threads(); + pool.install(|| { + for (label, bytes, splits) in [ + ("classic", &classic[..], &[None, Some(4)][..]), + ("HTJ2K", &ht[..], &[None][..]), + ] { + let whole = measure_executor(bytes, None, None); + for &roi in &[None, Some((48, 48, 16, 16))] { + for &parts in splits { + let case = format!("{label} {roi:?} parts {parts:?} on {threads}"); + check(&case, measure_executor(bytes, roi, parts), &whole); } } - }); + } + }); } + // Called from outside a pool, the executor sends its workers through + // the global pool's shared queue. + let whole = measure_executor(classic, None, None); + check( + "classic outside the pool", + measure_executor(classic, None, Some(4)), + &whole, + ); + drop(pools); assert!(readers.contains(&1) && readers.contains(&3), "{readers:?}"); assert!( parted > 0 && clipped && deep, diff --git a/crates/j2k-native/src/j2c/decode/subband.rs b/crates/j2k-native/src/j2c/decode/subband.rs index 3014cee3..c31b8bda 100644 --- a/crates/j2k-native/src/j2c/decode/subband.rs +++ b/crates/j2k-native/src/j2c/decode/subband.rs @@ -45,7 +45,7 @@ pub(crate) use self::tile_parallel::MIN_BAND_WORK; #[cfg(feature = "parallel")] pub(in crate::j2c::decode) use self::tile_parallel::{ plan_band_route, plan_band_route_beside, routes_tile, BandOutput, BandRoute, BandRouting, - BlockScratch, PayloadSource, + BlockScratch, PayloadSource, LOCK_BYTES, }; /// Decode the code-blocks of the tile's `components` into their sub-bands. diff --git a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs index e106a511..19c53f20 100644 --- a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs +++ b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs @@ -323,6 +323,11 @@ impl Drop for BlockScratch { } } +/// Bytes a lock or a condition variable may allocate when first used: none +/// where the platform's locks are futexes, a boxed `pthread` mutex of 64 +/// bytes or condition variable of 48 where they are not. +pub(in crate::j2c::decode) const LOCK_BYTES: usize = 64; + /// The first `samples` of a thread's block scratch, within the capacity /// [`BlockScratch::for_threads`] reserved: it never grows. fn block_scratch(scratch: &mut Vec, samples: usize) -> Result<&mut [f32]> { @@ -487,6 +492,13 @@ pub(in crate::j2c::decode) fn plan_tile_route<'a>( // `plan_tile` charged one task per strip. budget.include_elements::>(counts.tasks - plan.strips.len())?; budget.include_elements::>(counts.shared)?; + // The shared outputs' locks and the queue's, where locking first + // allocates. + budget.include_bytes( + (counts.shared + 1) + .checked_mul(LOCK_BYTES) + .ok_or(ValidationError::ImageTooLarge)?, + )?; if has_later_components { // Component-at-a-time decoding retains this serial context while // preparing the next component. Do not commit a parallel bank @@ -601,6 +613,7 @@ fn record_route_peak( workers.saturating_mul(plan.scratch_sizes(counts.scratch_samples).thread_bytes()), counts.tasks * size_of::>(), counts.shared * size_of::>(), + (counts.shared + 1) * LOCK_BYTES, ] .into_iter() .fold(0, usize::saturating_add); @@ -1481,6 +1494,11 @@ fn decode_strips<'plan, 'coefficients, W: Send>( // Largest tasks first so stragglers are small; zeroing tasks last. tasks.sort_unstable_by_key(|task| core::cmp::Reverse(task.work())); let queue = Mutex::new(tasks.into_iter()); + // Lock each lock once here, where no worker races to allocate it. + for lock in &shared { + drop(lock.lock()); + } + drop(queue.lock()); let failed = AtomicBool::new(false); // Each worker owns one already-budgeted workspace for the whole tile. // A single queue pop per task keeps work balanced across unequal cores @@ -1944,6 +1962,8 @@ pub(in crate::j2c::decode) fn plan_band_route_beside<'a>( } let workers = threads.min(plan.strips.len()); let prepared = (|| -> Result { + // The route itself, boxed beside the others of its tile. + budget.include_bytes(size_of::>())?; // Each thread holds scratch of its own, reserved before it starts. if !lockstep { band_scratch_sizes(&plan)?.include(&mut budget, workers + 1)?; From 8ff37568580ba8ef99f749bb9546b1b3cd0ee9d1 Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 02:47:12 -0500 Subject: [PATCH 68/94] fix(j2k-native): charge what the pool takes for a decode entered from outside it A parallel decode entered from outside the pool hands its jobs to the pool's shared queue, which may take a new block for them, and waits on a lock and condition variable of the calling thread, which allocate when first used where locks are not futexes, the lock twice when the waiter and the worker that wakes it first use it at once. The band executor charged only the block, and the tile route neither. Both now charge all three. The measuring test builds the global pool it decodes on from outside a pool with 12 threads, so the case streams whatever the machine or its environment; has every thread of every pool run a job before it measures; and waits, before each measurement and at most 30 s, until the process's allocations have settled. --- .../src/j2c/decode/parallel_bands.rs | 55 +++++++++++-------- crates/j2k-native/src/j2c/decode/subband.rs | 4 +- .../src/j2c/decode/subband/tile_parallel.rs | 17 +++++- 3 files changed, 49 insertions(+), 27 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index c0b05a03..9377df04 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -47,8 +47,8 @@ use std::sync::{Condvar, Mutex, MutexGuard}; use super::banded_tier1::{self, RowOutput}; use super::subband::{ - plan_band_route, plan_band_route_beside, routes_tile, BandOutput, BandRoute, BandRouting, - BlockScratch, PayloadSource, LOCK_BYTES, + plan_band_route, plan_band_route_beside, pool_entry_bytes, routes_tile, BandOutput, BandRoute, + BandRouting, BlockScratch, PayloadSource, LOCK_BYTES, }; use super::workspace::{TaskScratch, TaskWorkspace}; use super::{ @@ -878,9 +878,10 @@ fn plan_tile_lines( /// reader's view, with its band being read and next band in every ring, and /// its entry in the readers' list; each thread's workspace and scratch /// pair; the pool's job for each worker running or queued, and for each -/// pool thread whose worker finished before the pool freed its job; and -/// what the shared output of each band decoding in parts, the shared -/// state's lock and its condition variable allocate when first used. +/// pool thread whose worker finished before the pool freed its job; what +/// the shared output of each band decoding in parts, the shared state's +/// lock and its condition variable allocate when first used; and what the +/// pool takes when entered from outside it. fn executor_bytes( rings: &[Ring], bands: &[Band], @@ -914,12 +915,8 @@ fn executor_bytes( .checked_add(rayon::current_num_threads()) .and_then(|jobs| jobs.checked_mul(SENT_WORKER_BYTES)), // Called from outside the pool, its workers queue on the pool's - // shared queue, which may take a new block for them. - Some(if rayon::current_thread_index().is_none() { - INJECTED_BLOCK_BYTES - } else { - 0 - }), + // shared queue, and this thread waits for them to finish. + Some(pool_entry_bytes()), bands .iter() .filter(|band| band.parts > 1) @@ -951,10 +948,6 @@ fn queue_entries(bands: &[Band]) -> usize { /// job's context. const SENT_WORKER_BYTES: usize = 4 * size_of::(); -/// Bytes of a block of the pool's shared queue: 63 jobs of two words, each -/// with its state, and a link. -const INJECTED_BLOCK_BYTES: usize = 64 * 3 * size_of::(); - /// The bytes the rings and bands of the component of `route` hold once /// planned, and the rows' scales planning them holds beside them. fn ring_plan_bytes( @@ -2731,6 +2724,7 @@ mod tests { let mut routes = Vec::with_capacity(components); let mut live = DecodeAllocationBudget::for_storage(storage).expect("baseline"); let start = live.live_bytes(); + settle(); let base = j2k_alloc_probe::live_bytes(); j2k_alloc_probe::reset_peak(); for component in 0..components { @@ -2879,11 +2873,17 @@ mod tests { } } - /// Wait until the process has held the same bytes for a while: until - /// idle threads have finished starting. + /// Wait until the process has held the same bytes for 100 ms, so the + /// pools' threads have gone idle: a thread allocates its queue's state + /// and the locks it sleeps on when it first idles. fn settle() { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); let (mut last, mut still) = (j2k_alloc_probe::live_bytes(), 0); while still < 20 { + assert!( + std::time::Instant::now() < deadline, + "the process's allocations never settled" + ); std::thread::sleep(std::time::Duration::from_millis(5)); let now = j2k_alloc_probe::live_bytes(); still = if now == last { still + 1 } else { 0 }; @@ -2926,6 +2926,12 @@ mod tests { assert!(report.contains("1 passed"), "{report}"); return; } + // The decode from outside a pool streams on the global pool, of + // enough threads whatever the machine. + rayon::ThreadPoolBuilder::new() + .num_threads(12) + .build_global() + .expect("the measuring process's first use of the global pool"); let classic = include_bytes!("../../../fixtures/multi_segment/bypass_termall_rgb_53.j2k"); let ht = ht_fixture(); let (mut readers, mut parted, mut clipped, mut deep) = (Vec::new(), 0, false, false); @@ -2950,20 +2956,21 @@ mod tests { deep |= measured.deep; }; // The pools, and the global one, stay up until every measurement - // ends, so no pool's teardown frees bytes inside another's window, - // and every measurement waits until their threads have started, - // which allocates their queues' state and their locks. + // ends, so no pool's teardown frees bytes inside another's window. + // Every thread of each runs a job before any measurement, and every + // measurement waits until they idle (see `settle`). let pools: Vec<_> = [2, 4, 12] .into_iter() .map(|threads| { - rayon::ThreadPoolBuilder::new() + let pool = rayon::ThreadPoolBuilder::new() .num_threads(threads) .build() - .expect("build pool") + .expect("build pool"); + pool.broadcast(|_| ()); + pool }) .collect(); - assert!(rayon::current_num_threads() > 0); - settle(); + rayon::broadcast(|_| ()); for pool in &pools { let threads = pool.current_num_threads(); pool.install(|| { diff --git a/crates/j2k-native/src/j2c/decode/subband.rs b/crates/j2k-native/src/j2c/decode/subband.rs index c31b8bda..7363434d 100644 --- a/crates/j2k-native/src/j2c/decode/subband.rs +++ b/crates/j2k-native/src/j2c/decode/subband.rs @@ -44,8 +44,8 @@ pub(crate) use self::placement::BlockPlacement; pub(crate) use self::tile_parallel::MIN_BAND_WORK; #[cfg(feature = "parallel")] pub(in crate::j2c::decode) use self::tile_parallel::{ - plan_band_route, plan_band_route_beside, routes_tile, BandOutput, BandRoute, BandRouting, - BlockScratch, PayloadSource, LOCK_BYTES, + plan_band_route, plan_band_route_beside, pool_entry_bytes, routes_tile, BandOutput, BandRoute, + BandRouting, BlockScratch, PayloadSource, LOCK_BYTES, }; /// Decode the code-blocks of the tile's `components` into their sub-bands. diff --git a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs index 19c53f20..50006df2 100644 --- a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs +++ b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs @@ -328,6 +328,19 @@ impl Drop for BlockScratch { /// bytes or condition variable of 48 where they are not. pub(in crate::j2c::decode) const LOCK_BYTES: usize = 64; +/// Bytes the pool may take for a parallel decode entered from outside it: +/// a block of its shared queue for the jobs the decode hands it, 63 of two +/// words with their states and a link, and the lock and condition variable +/// the calling thread waits on, whose lock the waiter and the worker that +/// wakes it may each allocate when they first use it at once. +pub(in crate::j2c::decode) fn pool_entry_bytes() -> usize { + if rayon::current_thread_index().is_some() { + 0 + } else { + 64 * 3 * size_of::() + 3 * LOCK_BYTES + } +} + /// The first `samples` of a thread's block scratch, within the capacity /// [`BlockScratch::for_threads`] reserved: it never grows. fn block_scratch(scratch: &mut Vec, samples: usize) -> Result<&mut [f32]> { @@ -493,10 +506,11 @@ pub(in crate::j2c::decode) fn plan_tile_route<'a>( budget.include_elements::>(counts.tasks - plan.strips.len())?; budget.include_elements::>(counts.shared)?; // The shared outputs' locks and the queue's, where locking first - // allocates. + // allocates, and what the pool takes when entered from outside it. budget.include_bytes( (counts.shared + 1) .checked_mul(LOCK_BYTES) + .and_then(|locks| locks.checked_add(pool_entry_bytes())) .ok_or(ValidationError::ImageTooLarge)?, )?; if has_later_components { @@ -614,6 +628,7 @@ fn record_route_peak( counts.tasks * size_of::>(), counts.shared * size_of::>(), (counts.shared + 1) * LOCK_BYTES, + pool_entry_bytes(), ] .into_iter() .fold(0, usize::saturating_add); From 2bc8d3aab137c735739d555a6ba190c26468ae85 Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 03:38:23 -0500 Subject: [PATCH 69/94] fix(j2k-native): decode a component in serial bands when its executor does not fit A component decoding into planes whose band route planned within the budget, but whose rings, executor records or threads' scratch then did not fit beside the plan, failed with the budget's error even where the serial bands fit: the error reached the whole-window replay, which reports the original error when the replay succeeds. The executor now declines before anything decodes, and the component decodes in serial bands, as when its route declines while planning. The budget test now finds the least room a single component's route plans in, which leaves none for its executor, and requires the component to decode there in serial bands, bit for bit as the pool decodes it with room. A new test measures the whole-window tile route under the counting allocator, entered from a thread outside the pool that never entered it before: what the route holds stays within what it charges, its shared locks and the pool's entry included. --- .../src/j2c/decode/parallel_bands.rs | 314 ++++++++++++++---- .../src/j2c/decode/subband/tile_parallel.rs | 2 +- 2 files changed, 255 insertions(+), 61 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index 9377df04..64756b24 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -164,7 +164,7 @@ pub(super) fn decode_component<'a>( &mut tier1_us, ) .and_then(|completed| { - if completed { + if completed == Some(true) { route.record(tile_ctx, storage)?; } Ok(completed) @@ -175,11 +175,26 @@ pub(super) fn decode_component<'a>( profile_timings.codeblock_us += tier1_us; profile_timings.idwt_us += elapsed.saturating_sub(tier1_us); match result { - Ok(true) => { + Ok(Some(true)) => { tile_ctx.debug_counters.pool_band_components += 1; Ok(true) } - Ok(false) => { + Ok(None) => { + // The executor did not fit beside the plan, before anything + // decoded; the serial bands need neither. + banded_tier1::decode_component( + tile, + component_idx, + header, + tile_ctx, + storage, + transform, + profile_enabled, + profile_timings, + )?; + Ok(true) + } + Ok(Some(false)) => { // A band was left running with no worker to finish it. #[cfg(test)] let simulated = tile_ctx.strand_pool_band; @@ -202,6 +217,11 @@ pub(super) fn decode_component<'a>( } } +/// Decode the bands of `route`'s component on the pool while the streamed +/// IDWT reads them. Returns `None`, before anything decodes, when the +/// rings and bands, the executor's records or its threads' scratch do not +/// fit beside the route's plan, and `Some(false)` when a band was left +/// running with no worker to finish it. #[expect( clippy::too_many_arguments, reason = "the banded stage shares decode_tile's state and profiling explicitly" @@ -215,14 +235,17 @@ fn decode_route( transform: WaveletTransform, profile_enabled: bool, tier1_us: &mut u128, -) -> Result { +) -> Result> { // The rings and bands live beside the route's plan and scratch while // the bands decode; planning them holds their row scales beside them. let mut budget = route.budget(); let (planned, scales) = ring_plan_bytes(route, storage, header, component_idx)?; let mut planning = budget; - planning.include_bytes(planned + scales)?; - budget.include_bytes(planned)?; + if declines(planning.include_bytes(planned + scales))? + || declines(budget.include_bytes(planned))? + { + return Ok(None); + } let (rings, mut bands, active_levels) = plan_rings(route, storage, header, component_idx)?; if route.coding() == CodeBlockCoding::Classic { split_bands( @@ -235,9 +258,14 @@ fn decode_route( } // So do the executor's records and each thread's scratch, the route's // workers' and this thread's. - budget.include_bytes(executor_bytes(&rings, &bands, (route.workers(), 1))?)?; + if declines(budget.include_bytes(executor_bytes(&rings, &bands, (route.workers(), 1))?))? { + return Ok(None); + } let mut scratches = - BlockScratch::for_threads(route.workers() + 1, route.scratch_sizes()?, &mut budget)?; + match BlockScratch::for_threads(route.workers() + 1, route.scratch_sizes()?, &mut budget) { + Err(error) if super::reuse::is_capacity_error(&error) => return Ok(None), + scratches => scratches?, + }; tile_ctx.idwt_output.placement = crate::j2c::idwt::IdwtPlacement::Coefficients; tile_ctx.idwt_output.coefficients = Vec::new(); tile_ctx.idwt_output.rect = IntRect::from_xywh(0, 0, 0, 0); @@ -299,17 +327,26 @@ fn decode_route( // theirs written. storage.stale_coefficients = true; let Some(lines) = lines else { - return result; + return result.map(Some); }; match result { - Ok(true) => lines.finish(tile_ctx, Ok(())).map(|()| true), + Ok(true) => lines.finish(tile_ctx, Ok(())).map(|()| Some(true)), Ok(false) => { // Return the rings and plane; the whole-window path refills them. match lines.finish(tile_ctx, Err(DecodingError::CodeBlockDecodeFailure.into())) { - Ok(()) | Err(_) => Ok(false), + Ok(()) | Err(_) => Ok(Some(false)), } } - Err(error) => lines.finish(tile_ctx, Err(error)).map(|()| true), + Err(error) => lines.finish(tile_ctx, Err(error)).map(|()| Some(true)), + } +} + +/// Whether `result` failed for want of budget, which declines the pool's +/// bands; other errors pass on. +fn declines(result: Result<()>) -> Result { + match result { + Err(error) if super::reuse::is_capacity_error(&error) => Ok(true), + result => result.map(|()| false), } } @@ -339,10 +376,6 @@ pub(super) fn decode_tile_rows<'a>( // executor runs them all, so each is planned in the budget the last one // left, as one budget, which starts with the list of routes. // A budget too small for the pool leaves the tile to serial bands. - let declines = |result: Result<()>| match result { - Err(error) if super::reuse::is_capacity_error(&error) => Ok(true), - result => result.map(|()| false), - }; let mut live = DecodeAllocationBudget::for_storage(storage)?; let mut routes = Vec::new(); if declines(live.reserve_new(&mut routes, components))? { @@ -2059,7 +2092,7 @@ mod tests { use crate::reader::BitReader; use crate::{encode, DecodeSettings, EncodeOptions, Image, DEFAULT_MAX_DECODE_BYTES}; - fn budget_fixture(width: u32, height: u32, components: u16) -> Vec { + fn budget_fixture(width: u32, height: u32, components: u16, block_exp: u8) -> Vec { let mut state = 0x2545_f491_u32; let pixels: Vec = (0..width * height * u32::from(components)) .map(|_| { @@ -2079,8 +2112,8 @@ mod tests { &EncodeOptions { reversible: true, num_decomposition_levels: 4, - code_block_width_exp: 0, - code_block_height_exp: 0, + code_block_width_exp: block_exp, + code_block_height_exp: block_exp, ..EncodeOptions::default() }, ) @@ -2109,8 +2142,10 @@ mod tests { )] fn pool_bands_respect_the_combined_plan_and_executor_budget() { let (width, height) = (203, 157); - for components in [1_u16, 3] { - let bytes = budget_fixture(width, height, components); + // A component in 16 x 16 blocks plans its route in less room than + // its executor then needs beside it; 4 x 4 blocks plan in more. + for (components, block_exp) in [(1_u16, 2), (3, 0)] { + let bytes = budget_fixture(width, height, components, block_exp); let image = Image::new(&bytes, &DecodeSettings::default()).expect("parse budget fixture"); rayon::ThreadPoolBuilder::new() @@ -2213,24 +2248,41 @@ mod tests { let baseline = DecodeAllocationBudget::for_storage(storage) .expect("control budget") .live_bytes(); + let structural = storage.structural_workspace_bytes; storage.structural_workspace_bytes += DEFAULT_MAX_DECODE_BYTES - baseline - headroom; sink.0 = 0; let result = if components == 1 { - let route = match plan_band_route( - &tiles[0], - 0, - tile_ctx, - storage, - &image.header, - false, - ) { - Ok(BandRouting::Route(route)) => route, - Ok(BandRouting::Serial | BandRouting::WholeWindows) => return, - Err(error) if super::super::reuse::is_capacity_error(&error) => return, - Err(error) => panic!("unexpected planning error: {error:?}"), + // The least room the component's route plans in, + // which leaves none for its executor. + let (mut short, mut room) = (route_bytes, 4 * headroom); + let mut plans = |headroom: usize| { + storage.structural_workspace_bytes = + structural + DEFAULT_MAX_DECODE_BYTES - baseline - headroom; + match plan_band_route( + &tiles[0], + 0, + tile_ctx, + storage, + &image.header, + false, + ) { + Ok(BandRouting::Route(route)) => Some(route), + Ok(BandRouting::Serial | BandRouting::WholeWindows) => None, + Err(error) => panic!("planning within {headroom}: {error:?}"), + } }; - decode_route( + assert!(plans(short).is_none() && plans(room).is_some()); + while room - short > 1 { + let middle = short + (room - short) / 2; + if plans(middle).is_some() { + room = middle; + } else { + short = middle; + } + } + let route = plans(room).expect("the route plans"); + let declined = decode_route( &route, 0, &image.header, @@ -2239,7 +2291,49 @@ mod tests { tiles[0].component_infos[0].wavelet_transform(), false, &mut 0, - ) + ); + assert!( + matches!(declined, Ok(None)), + "the executor must decline before decoding: {declined:?}" + ); + drop(route); + // Where the executor does not fit, the serial bands + // decode the component as the pool does with room. + let mut decode = |headroom: usize| { + storage.structural_workspace_bytes = + structural + DEFAULT_MAX_DECODE_BYTES - baseline - headroom; + tile_ctx.channel_data[0].container.fill(f32::NAN); + let pooled = tile_ctx.debug_counters.pool_band_components; + let decoded = decode_component( + &tiles[0], + 0, + &image.header, + tile_ctx, + storage, + tiles[0].component_infos[0].wavelet_transform(), + false, + &mut DecodeProfileTimings::default(), + ); + assert!( + matches!(decoded, Ok(true)), + "the component decodes within {headroom}: {decoded:?}" + ); + let bits: Vec = tile_ctx.channel_data[0] + .container + .iter() + .map(|sample| sample.to_bits()) + .collect(); + (tile_ctx.debug_counters.pool_band_components - pooled, bits) + }; + let (pooled, with_room) = decode(4 * headroom); + assert_eq!(pooled, 1, "the pool decodes the component with room"); + let (pooled, serial) = decode(room); + assert_eq!(pooled, 0, "the serial bands decode it within {room}"); + assert!( + serial == with_room, + "the serial bands decode it bit for bit" + ); + return; } else { decode_tile_rows( &tiles[0], @@ -2341,6 +2435,10 @@ mod tests { /// pool's bands or, when they decline the tile, on serial bands, with all /// but `headroom` bytes beyond the baseline taken as if retained, which /// every budget of the decode sees. + #[expect( + clippy::too_many_lines, + reason = "the warm pass measures the steps the budgeted pass is then checked against, on one context" + )] fn decode_streamed(image: &Image<'_>, headroom: usize) -> (Steps, Collected, Streamed) { let (width, height) = (image.width(), image.height()); let mut context = DecoderContext::default(); @@ -2681,6 +2779,10 @@ mod tests { /// its routes, rings and bands and running the band executor as the /// streamed tile drives it, reading every row of every ring, each /// component's through its own reader when it has one. + #[expect( + clippy::too_many_lines, + reason = "the planning and executor phases are measured in order under the counting allocator, on one warm bank" + )] fn measure_executor( bytes: &[u8], roi: Option<(u32, u32, u32, u32)>, @@ -2873,6 +2975,122 @@ mod tests { } } + /// Whether this process is the one that measures the test `name`: the + /// allocator's high-water mark is the process's, so when it is not, + /// this runs the test alone in a process of its own and checks that it + /// passed there. + fn measures_alone(name: &str) -> bool { + const PROBE: &str = "J2K_NATIVE_BAND_EXECUTOR_PROBE"; + if std::env::var_os(PROBE).is_some() { + return true; + } + let module = module_path!() + .split_once("::") + .map(|(_, path)| path) + .expect("module path"); + let output = std::process::Command::new(std::env::current_exe().expect("test binary")) + .args([ + &format!("{module}::{name}"), + "--exact", + "--test-threads=1", + "--nocapture", + ]) + .env(PROBE, "1") + .output() + .expect("run the measuring process"); + let report = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.status.success(), "{report}"); + assert!(report.contains("1 passed"), "{report}"); + false + } + + /// Under the counting allocator, the whole-window tile route, entered + /// from a thread outside the pool that never entered it before, holds + /// at most what it charges for its plan, tasks and threads' scratch, + /// the locks it shares and what the pool takes for it. + #[test] + fn tile_route_allocates_within_its_charge() { + use core::sync::atomic::{AtomicBool, Ordering}; + + static ROUTED: AtomicBool = AtomicBool::new(false); + if !measures_alone("tile_route_allocates_within_its_charge") { + return; + } + rayon::ThreadPoolBuilder::new() + .num_threads(4) + .build_global() + .expect("the measuring process's first use of the global pool"); + rayon::broadcast(|_| ()); + let bytes = budget_fixture(203, 157, 1, 2); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("parse fixture"); + let mut context = DecoderContext::default(); + let tiles = tile::parse(&mut BitReader::new(image.codestream), &image.header, 0) + .expect("parse tile"); + let baseline = context + .reset( + &image.header, + &tiles[0], + tiles.structural_workspace_bytes(), + 0, + false, + None, + ) + .expect("prepare output planes"); + let (tile_ctx, storage) = (&mut context.tile_decode_context, &mut context.storage); + // Warm the bank on this thread, so the route reuses it. + super::super::decode_tile( + &tiles[0], + &image.header, + tile_ctx, + storage, + &mut None, + CpuDecodeParallelism::Auto, + false, + &mut DecodeProfileTimings::default(), + baseline, + super::super::TileOutput::Planes, + ) + .expect("decode with room to spare"); + tile_ctx.tile_route_post_decode = Some(|_| { + ROUTED.store(true, Ordering::Relaxed); + Ok(()) + }); + let retained = tile_ctx.tier1_capacity_bytes().expect("bank bytes"); + tile_ctx.debug_counters.workspace_peak = WorkspacePeak::default(); + let held = std::thread::scope(|scope| { + scope + .spawn(|| { + settle(); + let base = j2k_alloc_probe::live_bytes(); + j2k_alloc_probe::reset_peak(); + super::super::subband::decode_component_tile_bit_planes( + &tiles[0], + 0..1, + tile_ctx, + storage, + &image.header, + &mut None, + CpuDecodeParallelism::Auto, + false, + ) + .expect("decode the tile"); + j2k_alloc_probe::peak_bytes() - base + }) + .join() + .expect("the measuring thread") + }); + assert!(ROUTED.load(Ordering::Relaxed), "the tile route decodes it"); + let charged = tile_ctx.debug_counters.workspace_peak.bytes().tier1 - retained; + assert!( + held <= charged, + "the tile route held {held} of {charged} bytes" + ); + } + /// Wait until the process has held the same bytes for 100 ms, so the /// pools' threads have gone idle: a thread allocates its queue's state /// and the locks it sleeps on when it first idles. @@ -2899,31 +3117,7 @@ mod tests { /// with shallow rings and deep ones. #[test] fn band_executor_allocates_within_its_charge() { - const PROBE: &str = "J2K_NATIVE_BAND_EXECUTOR_PROBE"; - if std::env::var_os(PROBE).is_none() { - // The allocator's high-water mark is the process's: measure in a - // process that runs this test alone. - let name = module_path!() - .split_once("::") - .map(|(_, path)| path) - .expect("module path"); - let output = std::process::Command::new(std::env::current_exe().expect("test binary")) - .args([ - &format!("{name}::band_executor_allocates_within_its_charge"), - "--exact", - "--test-threads=1", - "--nocapture", - ]) - .env(PROBE, "1") - .output() - .expect("run the measuring process"); - let report = format!( - "{}{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - assert!(output.status.success(), "{report}"); - assert!(report.contains("1 passed"), "{report}"); + if !measures_alone("band_executor_allocates_within_its_charge") { return; } // The decode from outside a pool streams on the global pool, of diff --git a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs index 50006df2..d0ee8726 100644 --- a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs +++ b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs @@ -2249,7 +2249,7 @@ mod tests { let segment = |len: usize| Segment { idx: 0, coding_pases: 1, - data_length: len as u32, + data_length: u32::try_from(len).expect("test segment lengths fit u32"), data: &data[..len], }; let segments = [segment(500), segment(500), segment(1000), segment(500)]; From 323851e9473303151f1a42f3f002be80e7473e56 Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 03:38:23 -0500 Subject: [PATCH 70/94] docs(changelog): note multi-segment classic blocks on the pool routes --- CHANGELOG.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8989bbe..7027b8fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,24 @@ Staged workspace version: `0.13.2`. status, detail and batch source index. CUDA keeps its typed source and job attribution. Unsupported statuses, raw runtime failures and combined cleanup or release failures keep their existing classes. +- Classic code-blocks of several codeword segments, with selective bypass + or per-pass termination (as archival images and multi-layer encodes of + this crate often have), decode on the worker pool's tile route and pool + bands like single-segment ones: each worker collects a block's segment + descriptors into its own scratch, reserved before it starts, from the + segment records it already reads, with no per-block allocation or plan + growth. Region decodes and full-image decodes that stream their rows now + stream such images on the pool without component planes, and full-image + decodes into planes decode their blocks in place instead of on the + per-sub-band path. Output is unchanged. +- A region decode on the worker pool that streams its rows to a sink, such + as an 8-bit full-image decode whose component planes would pass the + decode budget, now streams them from serial bands when the pool's bands + cannot decode a tile but serial bands can, and the region's planes would + pass the budget. Such decodes of classic images with BYPASS or per-pass + termination failed with a resource limit on pools of two or more threads, + and decode as on one thread now. Decodes whose planes fit keep their + route. - Parallel band decoding charges each component's plan and the band executor's records to one decode budget, across all components when a tile's rows stream together, and releases the plans and the records before From 1535444e2692623b85eb0b2838a5b7e9625e73af Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 15:45:18 -0400 Subject: [PATCH 71/94] perf(j2k): decode streamed batch images on a pool of the explicit worker count With an explicit worker count, each streamed image of a multi-image group decoded serially on its worker thread, and the static chunks of the batch plan left workers idle: 8 images on 6 workers ran on 4 threads, 8 on 18 on 8, and the last images ran alone. Such a group now decodes every image inside a rayon pool of exactly that many threads, kept by the `CpuBatchDecoder`, with automatic inner parallelism: idle pool threads take Tier-1 bands of the images still in flight. The worker threads only wait for their images and keep their admission claims, so the images in flight stay one per worker and the group never runs more threads than the count allows. `rayon` becomes a normal dependency of j2k; j2k-native's default `parallel` feature already links it. On an 18-core ARM64 machine, 8 copies of a 2048 x 2048 RGB image (NHWC) decode in 395 ms instead of 555 ms on 6 workers and in 192 ms instead of 301 ms on 18, and four half-scale decodes of a large RGB JP2 scan in 395 ms instead of 1057 ms on 18 workers. Eight workers for eight images were already busy (268 ms either way). Peak heap grows by the pool's band working sets (192 instead of 116 MiB for the RGB group on 6 workers), still far below the staged route's 1407 MiB. Output is unchanged. --- crates/j2k/Cargo.toml | 2 +- crates/j2k/src/owned_batch.rs | 2 +- crates/j2k/src/owned_batch/cpu.rs | 22 ++++- crates/j2k/src/owned_batch/cpu_execute.rs | 86 +++++++++++++++++-- .../prepare/cpu_preflight_tests.rs | 43 ++++++++++ 5 files changed, 142 insertions(+), 13 deletions(-) diff --git a/crates/j2k/Cargo.toml b/crates/j2k/Cargo.toml index 6905aac6..3f324f80 100644 --- a/crates/j2k/Cargo.toml +++ b/crates/j2k/Cargo.toml @@ -24,12 +24,12 @@ j2k-core = { path = "../j2k-core", version = "=0.13.2" } j2k-native = { path = "../j2k-native", version = "=0.13.2" } j2k-types = { path = "../j2k-types", version = "=0.13.2" } moxcms = { workspace = true } +rayon = { workspace = true } thiserror = { workspace = true } [dev-dependencies] proptest = { workspace = true } criterion = { workspace = true } -rayon = { workspace = true } j2k-native = { path = "../j2k-native", version = "=0.13.2" } j2k-test-support = { path = "../j2k-test-support", features = ["j2k-native-fixtures"] } diff --git a/crates/j2k/src/owned_batch.rs b/crates/j2k/src/owned_batch.rs index 593c02bf..4dab7ec1 100644 --- a/crates/j2k/src/owned_batch.rs +++ b/crates/j2k/src/owned_batch.rs @@ -55,7 +55,7 @@ use self::cpu_fast::CpuGroupFastWorkspace; mod cpu_staged; use self::cpu_staged::{CpuEntropyOutcome, CpuStagedWorkspace, CPU_ENTROPY_IMAGES_PER_WINDOW}; mod cpu_execute; -use self::cpu_execute::decode_cpu_group; +use self::cpu_execute::{decode_cpu_group, CpuStreamPool}; mod cpu; mod cpu_materialize; mod cpu_staged_execute; diff --git a/crates/j2k/src/owned_batch/cpu.rs b/crates/j2k/src/owned_batch/cpu.rs index 73e84638..e034f88d 100644 --- a/crates/j2k/src/owned_batch/cpu.rs +++ b/crates/j2k/src/owned_batch/cpu.rs @@ -6,8 +6,9 @@ use super::{ decode_cpu_group, fmt, generic_batch_worker_limit, prepare_batch_from_images_for, prepare_batch_with_workers, BatchDecodeOptions, BatchDecoder, BatchGroupInfo, BatchInfrastructureError, BatchWorker, CpuBatchWorkspaceStats, CpuGroupFastWorkspace, - CpuStagedWorkspace, EncodedImage, IndexedBatchError, J2kDecodeWarning, Mutex, NativeSampleType, - NonZeroUsize, PrepareFor, PreparedBatch, PreparedBatchGroup, PreparedImage, Rect, Vec, + CpuStagedWorkspace, CpuStreamPool, EncodedImage, IndexedBatchError, J2kDecodeWarning, Mutex, + NativeSampleType, NonZeroUsize, PrepareFor, PreparedBatch, PreparedBatchGroup, PreparedImage, + Rect, Vec, }; use crate::batch::admission::BatchAllocationBudget; use crate::batch::allocation::try_vec_with_capacity; @@ -174,6 +175,7 @@ pub struct CpuBatchDecoder { workers: Mutex>, fast_workspace: CpuGroupFastWorkspace, staged_workspace: CpuStagedWorkspace, + stream_pool: Mutex, } impl fmt::Debug for CpuBatchDecoder { @@ -202,6 +204,7 @@ impl CpuBatchDecoder { workers: Mutex::new(workers), fast_workspace: CpuGroupFastWorkspace::default(), staged_workspace: CpuStagedWorkspace::default(), + stream_pool: Mutex::default(), } } @@ -211,6 +214,15 @@ impl CpuBatchDecoder { self.options } + /// Threads of the session's streamed-group pool, once a group used it. + #[cfg(test)] + pub(super) fn stream_pool_threads(&mut self) -> Option { + self.stream_pool + .get_mut() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .threads() + } + /// Number of reusable worker context/scratch owners retained by this session. #[must_use] pub fn retained_worker_count(&self) -> usize { @@ -373,6 +385,10 @@ impl CpuBatchDecoder { .workers .get_mut() .map_err(|_| BatchInfrastructureError::SchedulerPoisoned)?; + let stream_pool = self + .stream_pool + .get_mut() + .map_err(|_| BatchInfrastructureError::SchedulerPoisoned)?; let input_count = prepared .groups() .iter() @@ -398,6 +414,7 @@ impl CpuBatchDecoder { workers, &mut self.fast_workspace, &mut self.staged_workspace, + stream_pool, group, prepared.options, self.options.workers, @@ -435,6 +452,7 @@ impl CpuBatchDecoder { workers, &mut self.fast_workspace, &mut self.staged_workspace, + stream_pool, &part, prepared.options, self.options.workers, diff --git a/crates/j2k/src/owned_batch/cpu_execute.rs b/crates/j2k/src/owned_batch/cpu_execute.rs index 9cb45bb8..7001579b 100644 --- a/crates/j2k/src/owned_batch/cpu_execute.rs +++ b/crates/j2k/src/owned_batch/cpu_execute.rs @@ -26,6 +26,7 @@ pub(super) fn decode_cpu_group( workers: &mut [BatchWorker], fast_workspace: &mut CpuGroupFastWorkspace, staged_workspace: &mut CpuStagedWorkspace, + stream_pool: &mut CpuStreamPool, group: &PreparedBatchGroup, options: BatchDecodeOptions, workers_option: Option, @@ -47,6 +48,16 @@ pub(super) fn decode_cpu_group( && fast_workspace.prepare_group(group)? && retained_inner_parallelism(group.images.len(), tile_options)? == CpuDecodeParallelism::Serial; + // With an explicit worker count, the images of such a group decode in a + // pool of that many threads (see `CpuStreamPool`). + let stream_pool = if streams_rows + && group.images.len() > 1 + && workers_option.is_some_and(|count| count.get() > 1) + { + Some(stream_pool.get(workers.len())?) + } else { + None + }; let samples_per_image = group .info @@ -77,6 +88,7 @@ pub(super) fn decode_cpu_group( &mut values, has_flattened_payloads.then_some(&mut *fast_workspace), staged_workspace, + stream_pool, admission, // The only image of a Gray8 group is the whole in-flight set. if group.images.len() == 1 && group.info.color == BatchColor::Gray { @@ -104,6 +116,7 @@ pub(super) fn decode_cpu_group( &mut values, has_flattened_payloads.then_some(&mut *fast_workspace), staged_workspace, + stream_pool, admission, decode_image_u16, convert_u16, @@ -126,6 +139,7 @@ pub(super) fn decode_cpu_group( &mut values, has_flattened_payloads.then_some(&mut *fast_workspace), staged_workspace, + stream_pool, admission, decode_image_i16, round_signed, @@ -169,6 +183,44 @@ pub(super) fn decode_cpu_group( ))) } +/// The threads a group's streamed images decode on when the caller set the +/// worker count, created on first use and kept by the session. +/// +/// A streamed image hands its Tier-1 bands to the rayon pool it decodes in. +/// An explicit worker count used to keep each image serial on its worker +/// thread instead, so `M` images on `N` workers kept +/// `ceil(M / ceil(M / N))` threads busy (8 images on 6 workers: 4) and the +/// last images alone at the end. In a pool of exactly `N` threads idle +/// threads take bands of the images still in flight, so the group keeps all +/// `N` busy and never runs more: the worker threads only wait for their +/// images, and the images in flight stay one per worker. +#[derive(Default)] +pub(super) struct CpuStreamPool { + pool: Option, +} + +impl CpuStreamPool { + fn get(&mut self, threads: usize) -> Result<&rayon::ThreadPool, BatchInfrastructureError> { + let pool = match self.pool.take() { + Some(pool) if pool.current_num_threads() == threads => pool, + _ => rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .thread_name(|index| format!("j2k-batch-stream-{index}")) + .build() + .map_err(|_| BatchInfrastructureError::WorkerSpawnFailed { worker: 0 })?, + }; + Ok(self.pool.insert(pool)) + } + + /// Threads of the pool, once a group has used it. + #[cfg(test)] + pub(super) fn threads(&self) -> Option { + self.pool + .as_ref() + .map(rayon::ThreadPool::current_num_threads) + } +} + pub(super) struct CpuTypedDecodeJob<'image, 'output, T> { pub(super) slot: usize, pub(super) image: &'image PreparedImage, @@ -199,6 +251,7 @@ fn run_typed_group( output: &mut Vec, flattened_payloads: Option<&mut CpuGroupFastWorkspace>, staged_workspace: &mut CpuStagedWorkspace, + stream_pool: Option<&rayon::ThreadPool>, admission: &Arc, decode: CpuDecodeFn, convert: fn(f32, u8) -> T, @@ -248,15 +301,30 @@ fn run_typed_group( &mut claim, job.image.cpu_execution().claim_bytes(), )?; - *slot = Some(decode( - job.image, - options, - &group.info, - parallelism, - &mut context, - worker, - job.output, - )); + *slot = Some(match stream_pool { + // This thread waits while a pool thread decodes the + // image and idle ones take its bands. + Some(pool) => pool.install(|| { + decode( + job.image, + options, + &group.info, + CpuDecodeParallelism::Auto, + &mut context, + worker, + job.output, + ) + }), + None => decode( + job.image, + options, + &group.info, + parallelism, + &mut context, + worker, + job.output, + ), + }); } worker.restore_native_workspace(context.into_workspace()); Ok(()) diff --git a/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs b/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs index 7bd84e44..2f1d2964 100644 --- a/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs +++ b/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs @@ -364,3 +364,46 @@ fn a_lone_gray8_image_decodes_its_plan_and_gray8_images_in_flight_stream() { assert_eq!(decode(2, &gray), 0, "Gray8 images in flight stream"); assert_eq!(decode(1, &rgb), 0, "a lone RGB8 image streams"); } + +#[test] +fn explicit_workers_decode_streamed_images_on_a_pool_of_that_many_threads() { + let rgb = classic_rgb8(96, 64); + let gray = classic_fixture(80, 72, 1, 8); + for image in [&rgb, &gray] { + let inputs = |copies: usize| vec![EncodedImage::full(Arc::clone(image)); copies]; + // Prepared on this thread, where the threshold override applies. + let mut preparer = CpuBatchDecoder::new(single_worker_options()); + // (workers, images, pool threads): a multi-image group with an + // explicit count of two or more workers decodes on a pool of that + // many threads; one worker, the codec's own count, and a lone image + // keep their threads. + for (workers, copies, pool) in [ + (Some(1), 4, None), + (Some(2), 3, Some(2)), + (Some(3), 4, Some(3)), + (Some(3), 1, None), + (None, 4, None), + ] { + let reference = preparer + .decode_prepared(&preparer.prepare(inputs(copies)).expect("prepare plans")) + .expect("decode plans"); + let _threshold = StreamAbove::hold(0); + let prepared = preparer.prepare(inputs(copies)).expect("prepare"); + assert!(planned_bytes(&prepared) + .iter() + .all(|(_, execution)| matches!( + execution, + CpuExecution::StreamedRowsOrPlan { .. } + ))); + let mut decoder = CpuBatchDecoder::new(BatchDecodeOptions { + workers: workers.and_then(NonZeroUsize::new), + ..BatchDecodeOptions::default() + }); + let decoded = decoder.decode_prepared(&prepared).expect("decode"); + let label = format!("{workers:?} workers, {copies} images"); + assert_eq!(decoded.errors(), &[] as &[IndexedBatchError], "{label}"); + assert_eq!(decoded.groups(), reference.groups(), "{label}"); + assert_eq!(decoder.stream_pool_threads(), pool, "{label}"); + } + } +} From 14b91c8add42234119ec02e600997603141b79fd Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 15:49:41 -0400 Subject: [PATCH 72/94] perf(j2k): keep classic batch plans on the staged route at one worker With one explicit worker (up to four for Gray8), a classic group of several Gray8/RGB8 images whose plans each fit the per-decode bound, and together the batch execution cap, decodes them on the staged route again instead of streaming; larger groups and HTJ2K groups stream. The staged route decodes each code-block of a window's images back to back, which speeds up classic Tier-1 when the images repeat, and at these counts it keeps its workers busy, so the stream pool has no idle thread to gain from. Staged windows claim such an image's plan bytes, as they claim an offset plan's. On an 18-core ARM64 machine, 8 copies of one classic image streamed in 1.13-1.32 times the staged time at one worker, and 8 Gray8 copies in 1.09-1.24 times at two to four workers; those settings now measure 0.97-1.02 of the staged route (0.99-1.01 at one worker on a 10-core ARM64 machine). Distinct images streamed 4-5% faster at one worker, which the staged route gives up. RGB8 streams 10-40% faster from two workers on and Gray8 from six, and HTJ2K images stream about 20% faster even when they repeat, so those keep streaming. The plans' working memory returns for the staged settings (950-4302 MiB peak heap for the 8-image groups instead of 66-303 MiB). Output is unchanged. --- crates/j2k/src/owned_batch/cpu_execute.rs | 78 ++++++++++++--- .../j2k/src/owned_batch/cpu_staged_execute.rs | 9 +- .../prepare/cpu_preflight_tests.rs | 95 ++++++++++++++++--- crates/j2k/src/owned_batch/prepared.rs | 12 ++- 4 files changed, 166 insertions(+), 28 deletions(-) diff --git a/crates/j2k/src/owned_batch/cpu_execute.rs b/crates/j2k/src/owned_batch/cpu_execute.rs index 7001579b..f99cc5d2 100644 --- a/crates/j2k/src/owned_batch/cpu_execute.rs +++ b/crates/j2k/src/owned_batch/cpu_execute.rs @@ -8,14 +8,14 @@ use super::cpu_materialize::{ }; use super::cpu_staged_execute::run_staged_typed_group; use super::{ - decode_warnings_for_recovery, retained_inner_parallelism, run_retained_chunks, Arc, BatchColor, - BatchDecodeOptions, BatchErrorStage, BatchGroupInfo, BatchInfrastructureError, BatchItemError, - BatchWorker, CpuBatchGroup, CpuBatchSamples, CpuDecodeParallelism, CpuGroupFastWorkspace, - CpuStagedWorkspace, IndexedBatchError, J2kError, NativeSampleType, NonZeroUsize, - PreparedBatchGroup, PreparedImage, TileBatchOptions, Vec, + decode_warnings_for_recovery, retained_inner_parallelism, run_retained_chunks, Arc, + BatchCodecRoute, BatchColor, BatchDecodeOptions, BatchErrorStage, BatchGroupInfo, + BatchInfrastructureError, BatchItemError, BatchWorker, CpuBatchGroup, CpuBatchSamples, + CpuDecodeParallelism, CpuGroupFastWorkspace, CpuStagedWorkspace, IndexedBatchError, J2kError, + NativeSampleType, NonZeroUsize, PreparedBatchGroup, PreparedImage, TileBatchOptions, Vec, }; use crate::batch::admission::{hold_at_least, BatchAllocationBudget}; -use crate::batch::allocation::try_vec_with_capacity; +use crate::batch::allocation::{batch_execution_cap_bytes, try_vec_with_capacity}; #[expect( clippy::too_many_lines, @@ -39,11 +39,12 @@ pub(super) fn decode_cpu_group( // plan decodes faster on its own: its code-block rows and wavelet bands // spread over every core, still without reparsing. // Streamed images never hold whole-image scratch, so a group with one - // never enters the staged route. - let streams_rows = group - .images - .iter() - .any(|image| image.cpu_execution().streams()); + // enters the staged route only to keep its plans. + let streams_rows = !keeps_staged_plans(group, workers_option) + && group + .images + .iter() + .any(|image| image.cpu_execution().streams()); let has_flattened_payloads = !streams_rows && fast_workspace.prepare_group(group)? && retained_inner_parallelism(group.images.len(), tile_options)? @@ -183,6 +184,61 @@ pub(super) fn decode_cpu_group( ))) } +/// Most explicit workers for which a group whose images may stream keeps +/// their plans on the staged route (see [`keeps_staged_plans`]): one, or +/// four for Gray8, whose staged route keeps up to four workers busy. +const fn staged_plans_max_workers(color: BatchColor) -> usize { + match color { + BatchColor::Gray => 4, + _ => 1, + } +} + +#[cfg(test)] +std::thread_local! { + /// Test-only planned bytes above which a group streams instead of + /// keeping its plans on the staged route, on this thread. + pub(super) static STAGED_PLANS_CAP_OVERRIDE: core::cell::Cell> = + const { core::cell::Cell::new(None) }; +} + +/// Whether a group of images that may stream decodes their plans on the +/// staged route instead: the caller set one worker (up to four for Gray8), +/// the group holds several classic images, and each has a plan within the +/// per-decode bound, all together within the batch execution cap. Larger +/// groups stream. +/// +/// The staged route decodes each code-block of a window's images back to +/// back, which speeds up classic Tier-1 when the images repeat, and while it +/// keeps its workers busy the stream pool has no idle thread to gain from: +/// 8 copies of one image stream in 1.13-1.32 times the staged time at one +/// worker, and 8 Gray8 copies in 1.12-1.24 times at two to four workers. +/// Distinct images stream as fast or faster, RGB8 streams faster from two +/// workers on and Gray8 from six, and HTJ2K images always stream faster. +/// The window keeps each image's plan scratch, as these settings did before +/// images streamed. +fn keeps_staged_plans(group: &PreparedBatchGroup, workers: Option) -> bool { + group.info.route == BatchCodecRoute::Classic + && group.images.len() > 1 + && workers + .is_some_and(|workers| workers.get() <= staged_plans_max_workers(group.info.color)) + && group + .images + .iter() + .try_fold(0usize, |total, image| { + total.checked_add(image.cpu_execution().plan_bytes()?) + }) + .is_some_and(|total| total <= staged_plans_cap()) +} + +fn staged_plans_cap() -> usize { + #[cfg(test)] + if let Some(cap) = STAGED_PLANS_CAP_OVERRIDE.with(core::cell::Cell::get) { + return cap; + } + batch_execution_cap_bytes() +} + /// The threads a group's streamed images decode on when the caller set the /// worker count, created on first use and kept by the session. /// diff --git a/crates/j2k/src/owned_batch/cpu_staged_execute.rs b/crates/j2k/src/owned_batch/cpu_staged_execute.rs index a280aaab..8d0d0207 100644 --- a/crates/j2k/src/owned_batch/cpu_staged_execute.rs +++ b/crates/j2k/src/owned_batch/cpu_staged_execute.rs @@ -47,9 +47,12 @@ pub(super) fn run_staged_typed_group( let mut window_start = 0; while window_start < image_jobs.len() { let (window_end, window_bytes) = staged_window_len( - image_jobs[window_start..] - .iter() - .map(|job| job.image.cpu_execution().claim_bytes()), + image_jobs[window_start..].iter().map(|job| { + let execution = job.image.cpu_execution(); + execution + .plan_bytes() + .unwrap_or_else(|| execution.claim_bytes()) + }), window_budget, ); let window_end = window_start + window_end; diff --git a/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs b/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs index 2f1d2964..9d765e8e 100644 --- a/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs +++ b/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs @@ -3,6 +3,7 @@ use alloc::{sync::Arc, vec, vec::Vec}; use core::num::NonZeroUsize; +use super::super::cpu_execute::STAGED_PLANS_CAP_OVERRIDE; use super::super::{ prepare_batch, BatchDecodeOptions, BatchErrorStage, BatchItemError, BatchLayout, CpuBatchDecoder, CpuBatchSamples, CpuExecution, DecodeRequest, EncodedImage, IndexedBatchError, @@ -12,6 +13,10 @@ use super::image::{PLANNED_DECODE_BOUND, STREAM_ABOVE_OVERRIDE, STREAM_ABOVE_PLA use j2k_core::Downscale; fn classic_fixture(width: u32, height: u32, components: u16, precision: u8) -> Arc<[u8]> { + fixture(width, height, components, precision, false) +} + +fn fixture(width: u32, height: u32, components: u16, precision: u8, ht: bool) -> Arc<[u8]> { let count = width * height * u32::from(components); let samples = if precision <= 8 { (0..count) @@ -29,6 +34,7 @@ fn classic_fixture(width: u32, height: u32, components: u16, precision: u8) -> A let options = j2k_native::EncodeOptions { reversible: true, num_decomposition_levels: 3, + use_ht_block_coding: ht, ..j2k_native::EncodeOptions::default() }; Arc::from( @@ -82,6 +88,21 @@ impl Drop for StreamAbove { } } +struct StagedPlansCap; + +impl StagedPlansCap { + fn hold(bytes: Option) -> Self { + STAGED_PLANS_CAP_OVERRIDE.with(|cap| cap.set(bytes)); + Self + } +} + +impl Drop for StagedPlansCap { + fn drop(&mut self) { + STAGED_PLANS_CAP_OVERRIDE.with(|cap| cap.set(None)); + } +} + fn planned_bytes(prepared: &PreparedBatch) -> Vec<(usize, CpuExecution)> { let mut planned = prepared .groups() @@ -356,6 +377,9 @@ fn a_lone_gray8_image_decodes_its_plan_and_gray8_images_in_flight_stream() { assert!(planned_bytes(&prepared) .iter() .all(|(_, execution)| matches!(execution, CpuExecution::StreamedRowsOrPlan { .. }))); + // One worker keeps several images' plans within the cap (see + // `explicit_workers_choose_the_staged_plans_or_the_stream_pool`). + let _cap = StagedPlansCap::hold(Some(0)); let output = decoder.decode_prepared(&prepared).expect("decode"); assert_eq!(output.groups(), reference.groups(), "{copies} copies"); decoder.workspace_stats().prepared_plan_decode_calls() - plan_calls @@ -366,23 +390,30 @@ fn a_lone_gray8_image_decodes_its_plan_and_gray8_images_in_flight_stream() { } #[test] -fn explicit_workers_decode_streamed_images_on_a_pool_of_that_many_threads() { +fn explicit_workers_choose_the_staged_plans_or_the_stream_pool() { let rgb = classic_rgb8(96, 64); let gray = classic_fixture(80, 72, 1, 8); - for image in [&rgb, &gray] { + for (image, is_gray) in [(&rgb, false), (&gray, true)] { let inputs = |copies: usize| vec![EncodedImage::full(Arc::clone(image)); copies]; // Prepared on this thread, where the threshold override applies. let mut preparer = CpuBatchDecoder::new(single_worker_options()); - // (workers, images, pool threads): a multi-image group with an - // explicit count of two or more workers decodes on a pool of that - // many threads; one worker, the codec's own count, and a lone image - // keep their threads. - for (workers, copies, pool) in [ - (Some(1), 4, None), - (Some(2), 3, Some(2)), - (Some(3), 4, Some(3)), - (Some(3), 1, None), - (None, 4, None), + // (workers, images, cap on the group's plans, staged windows, pool + // threads): one worker (up to four for Gray8) keeps a multi-image + // group's plans on the staged route while they fit the cap and + // streams them otherwise; more workers stream the images on a pool of + // that many threads; the codec's own count and a lone image keep + // their threads. + let streams = |workers: usize| (!is_gray).then_some(workers); + let five_workers = 5.min(crate::batch::allocation::generic_batch_worker_limit()); + for (workers, copies, cap, staged, pool) in [ + (Some(1), 4, None, true, None), + (Some(2), 3, None, is_gray, streams(2)), + (Some(3), 4, None, is_gray, streams(3)), + (Some(1), 4, Some(1), false, None), + (Some(2), 3, Some(1), false, Some(2)), + (Some(5), 6, None, false, Some(five_workers)), + (Some(3), 1, None, false, None), + (None, 4, None, false, None), ] { let reference = preparer .decode_prepared(&preparer.prepare(inputs(copies)).expect("prepare plans")) @@ -399,11 +430,49 @@ fn explicit_workers_decode_streamed_images_on_a_pool_of_that_many_threads() { workers: workers.and_then(NonZeroUsize::new), ..BatchDecodeOptions::default() }); + let _cap = StagedPlansCap::hold(cap); let decoded = decoder.decode_prepared(&prepared).expect("decode"); - let label = format!("{workers:?} workers, {copies} images"); + let label = + format!("gray {is_gray}, {workers:?} workers, {copies} images, cap {cap:?}"); assert_eq!(decoded.errors(), &[] as &[IndexedBatchError], "{label}"); assert_eq!(decoded.groups(), reference.groups(), "{label}"); + assert_eq!( + decoder.workspace_stats().cross_image_entropy_windows() > 0, + staged, + "{label}" + ); assert_eq!(decoder.stream_pool_threads(), pool, "{label}"); } } } + +#[test] +fn htj2k_groups_stream_with_one_or_two_workers() { + let ht = fixture(96, 64, 3, 8, true); + let inputs = || vec![EncodedImage::full(Arc::clone(&ht)); 3]; + let mut preparer = CpuBatchDecoder::new(single_worker_options()); + let reference = preparer + .decode_prepared(&preparer.prepare(inputs()).expect("prepare plans")) + .expect("decode plans"); + let _threshold = StreamAbove::hold(0); + let prepared = preparer.prepare(inputs()).expect("prepare"); + assert!(planned_bytes(&prepared) + .iter() + .all(|(_, execution)| matches!(execution, CpuExecution::StreamedRowsOrPlan { .. }))); + // HTJ2K gains nothing from the staged route's back-to-back code-blocks. + for (workers, pool) in [(1, None), (2, Some(2))] { + let mut decoder = CpuBatchDecoder::new(BatchDecodeOptions { + workers: NonZeroUsize::new(workers), + ..BatchDecodeOptions::default() + }); + let decoded = decoder.decode_prepared(&prepared).expect("decode"); + assert_eq!(decoded.errors(), &[] as &[IndexedBatchError], "{workers}"); + assert_eq!(decoded.groups(), reference.groups(), "{workers}"); + assert_eq!( + decoder.workspace_stats().cross_image_entropy_windows(), + 0, + "{workers}" + ); + assert_eq!(decoder.stream_pool_threads(), pool, "{workers}"); + } +} diff --git a/crates/j2k/src/owned_batch/prepared.rs b/crates/j2k/src/owned_batch/prepared.rs index aecf4fba..4028f6cf 100644 --- a/crates/j2k/src/owned_batch/prepared.rs +++ b/crates/j2k/src/owned_batch/prepared.rs @@ -76,8 +76,18 @@ impl CpuExecution { } } + /// The planned peak of the offset plan this image may decode, if any: + /// what the staged route, which decodes plans only, claims for it. + pub(super) const fn plan_bytes(self) -> Option { + match self { + Self::OffsetPlan { bytes } | Self::StreamedRowsOrPlan { bytes } => Some(bytes), + Self::ComponentPlanes | Self::ExceedsDecodeBound | Self::StreamedRows => None, + } + } + /// Whether the CPU decoder may stream this image's rows; groups holding - /// such an image never take the staged route. + /// such an image take the staged route only to keep their plans (see + /// `keeps_staged_plans`). pub(super) const fn streams(self) -> bool { matches!(self, Self::StreamedRows | Self::StreamedRowsOrPlan { .. }) } From f20c62e5398f610d747175b05406f43b8fe3d6be Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 17:06:54 -0400 Subject: [PATCH 73/94] fix(j2k): stream batch images only where the region decode streams rows Batch preparation labelled every Gray8/RGB8 image whose plan passed the streaming threshold or the per-decode bound as streamed, from its output format alone. The region decode streams rows only for images without a palette, channel definition, sYCC or CIELab, with unsubsampled components of at most 24 bits, and without image or tile offsets; any other image decodes whole component planes and hands them to the row sink at the end. An RGB JP2 with identity channel definitions, which batch accepts, was therefore admitted with the streamed claim while it held whole-image planes. Preparation now asks `Image::region_decode_streams_rows` (doc-hidden), which collects those header-level conditions, and applies the streaming threshold and the lone-Gray8 rule only to images for which it holds; the rest keep their offset plan, or are rejected up front when it exceeds the bound, like the other outputs. The test fixtures cover an RGB JP2 with a cdef box and RGB8/Gray8 codestreams with offsets: they decode the same samples as the plain images, keep their plans past the threshold, and are rejected under the bound while the plain images stream. --- .../prepare/cpu_preflight_tests.rs | 127 ++++++++++++++++++ crates/j2k/src/owned_batch/prepare/image.rs | 27 ++-- 2 files changed, 146 insertions(+), 8 deletions(-) diff --git a/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs b/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs index 9d765e8e..25736846 100644 --- a/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs +++ b/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs @@ -255,6 +255,133 @@ fn cpu_prepare_rejects_plans_over_the_bound_without_a_streamed_route() { )); } +/// `codestream` as JP2, with `channel_definitions` in a cdef box. +fn wrap_jp2(codestream: &[u8], channel_definitions: &[crate::J2kChannelDefinition]) -> Arc<[u8]> { + let color = if channel_definitions.is_empty() { + j2k_core::Colorspace::SGray + } else { + j2k_core::Colorspace::SRgb + }; + Arc::from( + crate::wrap_j2k_codestream( + codestream, + crate::J2kFileWrapOptions::jp2() + .with_color(crate::J2kFileColorSpec::Enumerated(color)) + .with_metadata(crate::J2kFileBoxMetadata { + palette: None, + component_mappings: &[], + channel_definitions, + }), + ) + .expect("wrap JP2"), + ) +} + +fn identity_rgb_channel_definitions() -> [crate::J2kChannelDefinition; 3] { + [1, 2, 3].map(|color| crate::J2kChannelDefinition { + channel_index: color - 1, + channel_type: crate::J2kChannelType::Color, + association: crate::J2kChannelAssociation::Color { index: color }, + }) +} + +/// `codestream` with the image area and tile grid moved to `origin` on +/// both axes. A multiple of every code-block and DWT alignment keeps the +/// same coded samples. +fn with_siz_origin(codestream: &[u8], origin: u32) -> Arc<[u8]> { + let mut bytes = codestream.to_vec(); + assert_eq!(&bytes[..4], &[0xff, 0x4f, 0xff, 0x51], "SOC then SIZ"); + let mut add = |offset: usize| { + let field = <[u8; 4]>::try_from(&bytes[offset..offset + 4]).expect("SIZ field"); + let value = u32::from_be_bytes(field) + origin; + bytes[offset..offset + 4].copy_from_slice(&value.to_be_bytes()); + }; + // Xsiz, Ysiz, XOsiz, YOsiz, then XTOsiz, YTOsiz. + for offset in [8, 12, 16, 20, 32, 36] { + add(offset); + } + Arc::from(bytes) +} + +#[test] +fn gray8_and_rgb8_images_whose_region_decode_cannot_stream_keep_the_plan_route() { + let rgb = classic_rgb8(96, 64); + let gray = classic_fixture(80, 72, 1, 8); + let streams = [Arc::clone(&rgb), Arc::clone(&gray), wrap_jp2(&gray, &[])]; + // A channel definition, or offsets, keep the region decode on whole + // component planes, which it hands over at the end. + let planes = [ + wrap_jp2(&rgb, &identity_rgb_channel_definitions()), + with_siz_origin(&rgb, 4096), + with_siz_origin(&gray, 4096), + ]; + let inputs = || { + streams + .iter() + .chain(&planes) + .map(|image| EncodedImage::full(Arc::clone(image))) + .collect::>() + }; + let mut decoder = CpuBatchDecoder::new(single_worker_options()); + let reference = all_u8_samples( + &decoder + .decode_prepared(&decoder.prepare(inputs()).expect("prepare plans")) + .expect("decode plans"), + ); + // Moved, wrapped and plain images decode the same samples. + let samples = |index: usize| { + reference + .iter() + .find(|(indices, _)| indices.contains(&index)) + .map(|(indices, samples)| { + let per_image = samples.len() / indices.len(); + let slot = indices.iter().position(|&i| i == index).expect("slot"); + samples[slot * per_image..(slot + 1) * per_image].to_vec() + }) + .expect("decoded image") + }; + for (index, plain) in [(2, 1), (3, 0), (4, 0), (5, 1)] { + assert_eq!(samples(index), samples(plain), "input {index}"); + } + + { + let _threshold = StreamAbove::hold(0); + let prepared = decoder + .prepare(inputs()) + .expect("prepare over the threshold"); + let planned = planned_bytes(&prepared); + for (index, execution) in &planned { + assert_eq!( + matches!(execution, CpuExecution::StreamedRowsOrPlan { .. }), + *index < streams.len(), + "input {index}: {execution:?}" + ); + assert_eq!( + matches!(execution, CpuExecution::OffsetPlan { .. }), + *index >= streams.len(), + "input {index}: {execution:?}" + ); + } + let decoded = all_u8_samples(&decoder.decode_prepared(&prepared).expect("decode")); + assert_eq!(decoded, reference); + } + + // Over the bound, only the images that stream their rows are admitted. + let _bound = PlannedBound::hold(0); + let bounded = decoder.prepare(inputs()).expect("prepare under the bound"); + assert!(planned_bytes(&bounded) + .iter() + .all(|(_, execution)| *execution == CpuExecution::StreamedRows)); + let rejected = bounded.errors(); + assert_eq!(rejected.len(), planes.len(), "{rejected:?}"); + for (error, index) in rejected.iter().zip(streams.len()..) { + assert!( + is_decode_bound_error(error, index, BatchErrorStage::Prepare), + "{error:?}" + ); + } +} + #[test] fn cpu_batch_streams_large_rgb8_and_gray8_plans_below_the_decode_bound() { let small = classic_rgb8(16, 16); diff --git a/crates/j2k/src/owned_batch/prepare/image.rs b/crates/j2k/src/owned_batch/prepare/image.rs index aa932a6a..70b48458 100644 --- a/crates/j2k/src/owned_batch/prepare/image.rs +++ b/crates/j2k/src/owned_batch/prepare/image.rs @@ -54,7 +54,11 @@ pub(super) fn prepare_image( .map_or(PreparedCodecPlan::MetadataOnly, PreparedCodecPlan::Htj2k), }; reconcile_codec_plan_metadata(&mut info, &codec_plan)?; - let cpu_execution = choose_cpu_execution(&codec_plan, &info); + let cpu_execution = choose_cpu_execution( + &codec_plan, + &info, + native_image.region_decode_streams_rows(), + ); let preparation_depth = codec_plan.preparation_depth(); let execution_shape = batch_execution_shape(&support, plan, preparation_depth); let image = PreparedImage { @@ -79,17 +83,21 @@ pub(super) fn prepare_image( /// enforces. /// /// A Gray8 or RGB8 image whose planned execution exceeds -/// [`STREAM_ABOVE_PLANNED_BYTES`] streams its rows: the offset plan holds +/// [`STREAM_ABOVE_PLANNED_BYTES`] streams its rows when the region decode +/// can stream them (`region_streams_rows`): the offset plan holds /// whole-image f32 planes and coefficients (about 14 times the 8-bit /// output), which every image in flight kept until its decode ended, while /// the streamed route holds a band of rows. While its plan fits the /// per-decode bound, the image keeps it for decoding alone in a Gray8 group /// ([`CpuExecution::StreamedRowsOrPlan`]). Other outputs keep their offset /// plan unless its serial execution exceeds the per-decode bound, which -/// rejects them. The claim of an offset plan is its parallel peak. +/// rejects them, as it does images whose region decode would hand over +/// whole component planes instead of rows. The claim of an offset plan is +/// its parallel peak. pub(super) fn choose_cpu_execution( codec_plan: &PreparedCodecPlan, info: &BatchGroupInfo, + region_streams_rows: bool, ) -> CpuExecution { let planned = |parallel| match codec_plan { PreparedCodecPlan::Classic(plan) => { @@ -103,7 +111,8 @@ pub(super) fn choose_cpu_execution( if matches!(codec_plan, PreparedCodecPlan::MetadataOnly) { return CpuExecution::ComponentPlanes; } - let exceeds_bound = if streams_rows(info) { + let streams_rows = region_streams_rows && has_streamed_output(info); + let exceeds_bound = if streams_rows { CpuExecution::StreamedRows } else { CpuExecution::ExceedsDecodeBound @@ -114,12 +123,12 @@ pub(super) fn choose_cpu_execution( _, ) => exceeds_bound, (Ok(serial), _) if serial > planned_decode_bound() => exceeds_bound, - (Ok(_), Ok(bytes)) if !streams_rows(info) || bytes <= stream_above_planned_bytes() => { + (Ok(_), Ok(bytes)) if !streams_rows || bytes <= stream_above_planned_bytes() => { CpuExecution::OffsetPlan { bytes } } (Ok(_), Ok(bytes)) => CpuExecution::StreamedRowsOrPlan { bytes }, // An 8-bit plan that fits only serially streams. - (Ok(_), _) if streams_rows(info) => CpuExecution::StreamedRows, + (Ok(_), _) if streams_rows => CpuExecution::StreamedRows, // Any other plan that fits only serially, or a malformed plan, keeps // the worst-case claim and reports any error from execution. _ => CpuExecution::OffsetPlan { @@ -129,8 +138,10 @@ pub(super) fn choose_cpu_execution( } /// Gray8 and RGB8 outputs have a streamed route: the region decode hands -/// over finished rows of 8-bit components. Other outputs keep the bound. -fn streams_rows(info: &BatchGroupInfo) -> bool { +/// over finished rows of 8-bit components where the image lets it (see +/// `j2k_native::Image::region_decode_streams_rows`). Other outputs keep the +/// bound. +fn has_streamed_output(info: &BatchGroupInfo) -> bool { info.sample_type == NativeSampleType::U8 && matches!(info.color, BatchColor::Gray | BatchColor::Rgb) } From 7c9ceabdb3e8eb9084c4c58ecc5226ba65302427 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 17:06:54 -0400 Subject: [PATCH 74/94] perf(j2k): flatten batch payloads only for groups on the staged route `decode_cpu_group` flattened a group's code-block payloads before it checked whether the group's images decode serially, which the staged route needs. A group whose images decode with inner parallelism, such as any lone image or a group under automatic workers on a host with more cores than images, discarded that work. The parallelism check now runs first. Two tests that counted the discarded flattening of lone images now expect none, and the HTJ2K pass-bucket test sets one worker so its group takes the staged route that decodes the flattened jobs. --- crates/j2k/src/owned_batch/cpu_execute.rs | 6 ++++-- crates/j2k/tests/owned_batch/cpu_reuse.rs | 6 ++++-- crates/j2k/tests/owned_batch/native_types_and_requests.rs | 4 +++- .../owned_batch_fixtures/ht_matrix/pass_bucket_parity.rs | 5 ++++- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/crates/j2k/src/owned_batch/cpu_execute.rs b/crates/j2k/src/owned_batch/cpu_execute.rs index f99cc5d2..ea915b5c 100644 --- a/crates/j2k/src/owned_batch/cpu_execute.rs +++ b/crates/j2k/src/owned_batch/cpu_execute.rs @@ -45,10 +45,12 @@ pub(super) fn decode_cpu_group( .images .iter() .any(|image| image.cpu_execution().streams()); + // Payloads are flattened only for the staged route, so a group whose + // images decode with inner parallelism skips that work. let has_flattened_payloads = !streams_rows - && fast_workspace.prepare_group(group)? && retained_inner_parallelism(group.images.len(), tile_options)? - == CpuDecodeParallelism::Serial; + == CpuDecodeParallelism::Serial + && fast_workspace.prepare_group(group)?; // With an explicit worker count, the images of such a group decode in a // pool of that many threads (see `CpuStreamPool`). let stream_pool = if streams_rows diff --git a/crates/j2k/tests/owned_batch/cpu_reuse.rs b/crates/j2k/tests/owned_batch/cpu_reuse.rs index f0b4b50e..bcfeb19c 100644 --- a/crates/j2k/tests/owned_batch/cpu_reuse.rs +++ b/crates/j2k/tests/owned_batch/cpu_reuse.rs @@ -356,8 +356,10 @@ fn cpu_session_retained_workspace_stabilizes_during_thousand_batch_soak() { assert_eq!(soaked.prepared_plan_decode_calls(), 1_001); assert_eq!(warmed.decode_calls(), 0); assert_eq!(soaked.decode_calls(), 0); - assert_eq!(warmed.flattened_group_plans(), 1); - assert_eq!(soaked.flattened_group_plans(), 1_001); + // A lone image decodes its plan with inner parallelism, so its payloads + // are never flattened for the staged route. + assert_eq!(warmed.flattened_group_plans(), 0); + assert_eq!(soaked.flattened_group_plans(), 0); assert!(warmed.retained_prepared_plan_classic_workspace_bytes() > 0); assert_eq!(soaked.scratch_capacity_retries(), 0); assert_eq!( diff --git a/crates/j2k/tests/owned_batch/native_types_and_requests.rs b/crates/j2k/tests/owned_batch/native_types_and_requests.rs index c79697b9..e3ea4e33 100644 --- a/crates/j2k/tests/owned_batch/native_types_and_requests.rs +++ b/crates/j2k/tests/owned_batch/native_types_and_requests.rs @@ -86,7 +86,9 @@ fn prepared_htj2k_gray_and_rgb_support_native_types_and_requests_exactly() { let stats = decoder.workspace_stats(); assert_eq!(stats.preparation_calls(), requests.len() as u64); assert_eq!(stats.prepared_plan_decode_calls(), requests.len() as u64); - assert_eq!(stats.flattened_group_plans(), requests.len() as u64); + // Each request is a group of one image, which decodes its + // plan with inner parallelism and flattens no payloads. + assert_eq!(stats.flattened_group_plans(), 0); assert_eq!(stats.output_group_allocations(), requests.len() as u64); assert_eq!(stats.output_compaction_copied_samples(), 0); } diff --git a/crates/j2k/tests/owned_batch_fixtures/ht_matrix/pass_bucket_parity.rs b/crates/j2k/tests/owned_batch_fixtures/ht_matrix/pass_bucket_parity.rs index 533b8359..d4488fc3 100644 --- a/crates/j2k/tests/owned_batch_fixtures/ht_matrix/pass_bucket_parity.rs +++ b/crates/j2k/tests/owned_batch_fixtures/ht_matrix/pass_bucket_parity.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use std::{collections::BTreeSet, sync::Arc}; +use std::{collections::BTreeSet, num::NonZeroUsize, sync::Arc}; use j2k::{ prepare_batch, BatchDecodeOptions, BatchLayout, CpuBatchDecoder, CpuBatchSamples, EncodedImage, @@ -87,9 +87,12 @@ fn external_cleanup_magref_and_generated_sigprop_jobs_decode_in_batches() { let (cleanup, cleanup_pixels) = layered_ht_fixture(1); let (sigprop, sigprop_pixels) = layered_ht_fixture(2); let (magref, magref_pixels) = layered_ht_fixture(3); + // One worker decodes a group's images serially, on the staged route that + // flattens their payloads into pass-bucket jobs. let options = { let mut options = BatchDecodeOptions::default(); options.layout = BatchLayout::Nhwc; + options.workers = NonZeroUsize::new(1); options }; let prepared = prepare_batch( From 7003943f69c3250d0d944556df95b3d4531ee97e Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 19:13:25 -0400 Subject: [PATCH 75/94] perf(j2k): stream every batch image whose rows stream With one explicit worker (up to four for Gray8), a classic group whose images may stream decoded all of its plans on the staged route whenever they fit the batch execution cap. The staged route only gains where identical code-blocks decode back to back, and it holds each staged image's whole-image plan scratch; distinct images stream faster at every worker count and hold a band of rows instead of whole-image planes. Every image whose region decode streams its rows now streams at every worker count, repeated or not. Groups of plans that do not stream keep the staged route at one worker. Their flattened payloads are now checked against the flattening and staged-window caps first: a group beyond them decodes its plans one image at a time instead of failing the batch. A planned image among streamed ones (`b` in `[a, b, a]`: an RGB8 JP2 with identity channel definitions, or a Gray8 image with offsets) took the group's policy only in the parallelism handed to the decode call. Its plan reads the policy from its worker, which the scheduler had set for a single job, so it executed with nested parallelism: on the global pool at one worker, and across the stream pool at two. The policy is now set on the worker before the decode, so images decoded one at a time run serially at one worker and in the explicit pool otherwise. The tests pin the route: copies, distinct and mixed groups stream at one to eight workers and with automatic workers, on a pool of the worker count, while the same images as plans stage at one worker; plans beyond the flattening cap decode one at a time with the same samples; and `[a, b, a]` with a planned `b` decodes it without nested parallelism at one and two workers. --- crates/j2k/src/batch/worker.rs | 5 + crates/j2k/src/batch/worker/owned.rs | 23 +- crates/j2k/src/owned_batch/cpu.rs | 12 + crates/j2k/src/owned_batch/cpu_execute.rs | 121 +++---- crates/j2k/src/owned_batch/cpu_fast.rs | 81 ++++- crates/j2k/src/owned_batch/cpu_staged.rs | 6 + .../prepare/cpu_preflight_tests.rs | 336 ++++++++++++++---- crates/j2k/src/owned_batch/prepared.rs | 5 +- 8 files changed, 430 insertions(+), 159 deletions(-) diff --git a/crates/j2k/src/batch/worker.rs b/crates/j2k/src/batch/worker.rs index 437404c4..8e8073e6 100644 --- a/crates/j2k/src/batch/worker.rs +++ b/crates/j2k/src/batch/worker.rs @@ -35,6 +35,9 @@ pub(crate) struct BatchWorker { /// Parallelism one owned-batch image decode may use; set per submission /// by the retained scheduler. owned_inner_parallelism: CpuDecodeParallelism, + /// Test-only count of offset-plan executions with nested parallelism. + #[cfg(test)] + parallel_plan_decode_calls: u64, } impl BatchWorker { @@ -56,6 +59,8 @@ impl BatchWorker { prepared_plan_decode_calls: 0, allocation_budget, owned_inner_parallelism: CpuDecodeParallelism::Serial, + #[cfg(test)] + parallel_plan_decode_calls: 0, } } diff --git a/crates/j2k/src/batch/worker/owned.rs b/crates/j2k/src/batch/worker/owned.rs index 3f239838..59fdc306 100644 --- a/crates/j2k/src/batch/worker/owned.rs +++ b/crates/j2k/src/batch/worker/owned.rs @@ -119,8 +119,7 @@ impl BatchWorker { signed: bool, ) -> j2k_native::Result> { self.prepared_plan_decode_calls = self.prepared_plan_decode_calls.saturating_add(1); - self.prepared_plan_scratch - .set_parallel_decode(self.owned_inner_parallelism == CpuDecodeParallelism::Auto); + self.prepare_plan_parallelism(); j2k_native::execute_referenced_htj2k_plan( plan, encoded_input, @@ -136,8 +135,7 @@ impl BatchWorker { signed: bool, ) -> j2k_native::Result> { self.prepared_plan_decode_calls = self.prepared_plan_decode_calls.saturating_add(1); - self.prepared_plan_scratch - .set_parallel_decode(self.owned_inner_parallelism == CpuDecodeParallelism::Auto); + self.prepare_plan_parallelism(); j2k_native::execute_referenced_classic_plan( plan, encoded_input, @@ -146,6 +144,23 @@ impl BatchWorker { ) } + /// Nested parallelism of the next offset-plan execution: only where the + /// owned scheduler allowed it for this worker's decodes. + fn prepare_plan_parallelism(&mut self) { + let parallel = self.owned_inner_parallelism == CpuDecodeParallelism::Auto; + #[cfg(test)] + { + self.parallel_plan_decode_calls += u64::from(parallel); + } + self.prepared_plan_scratch.set_parallel_decode(parallel); + } + + /// Offset-plan executions with nested parallelism so far. + #[cfg(test)] + pub(crate) const fn parallel_plan_decode_calls(&self) -> u64 { + self.parallel_plan_decode_calls + } + pub(crate) const fn prepared_plan_decode_calls(&self) -> u64 { self.prepared_plan_decode_calls } diff --git a/crates/j2k/src/owned_batch/cpu.rs b/crates/j2k/src/owned_batch/cpu.rs index e034f88d..94ff7ea7 100644 --- a/crates/j2k/src/owned_batch/cpu.rs +++ b/crates/j2k/src/owned_batch/cpu.rs @@ -223,6 +223,18 @@ impl CpuBatchDecoder { .threads() } + /// Offset-plan executions of this session's workers with nested + /// parallelism. + #[cfg(test)] + pub(super) fn parallel_plan_decodes(&mut self) -> u64 { + self.workers + .get_mut() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .iter() + .map(BatchWorker::parallel_plan_decode_calls) + .sum() + } + /// Number of reusable worker context/scratch owners retained by this session. #[must_use] pub fn retained_worker_count(&self) -> usize { diff --git a/crates/j2k/src/owned_batch/cpu_execute.rs b/crates/j2k/src/owned_batch/cpu_execute.rs index ea915b5c..6954a560 100644 --- a/crates/j2k/src/owned_batch/cpu_execute.rs +++ b/crates/j2k/src/owned_batch/cpu_execute.rs @@ -2,6 +2,7 @@ //! CPU homogeneous-group execution and final materialization. +use super::cpu_fast::classic_staging_fits; use super::cpu_materialize::{ convert_u16, convert_u8, decode_image_i16, decode_image_u16, decode_image_u8, decode_lone_gray_u8, ensure_batch_output_within_cap, round_signed, try_zeroed_vec, @@ -15,7 +16,7 @@ use super::{ NativeSampleType, NonZeroUsize, PreparedBatchGroup, PreparedImage, TileBatchOptions, Vec, }; use crate::batch::admission::{hold_at_least, BatchAllocationBudget}; -use crate::batch::allocation::{batch_execution_cap_bytes, try_vec_with_capacity}; +use crate::batch::allocation::try_vec_with_capacity; #[expect( clippy::too_many_lines, @@ -37,26 +38,26 @@ pub(super) fn decode_cpu_group( // The staged route flattens entropy work across images for a few serial // workers. When workers may parallelize inside one image, each prepared // plan decodes faster on its own: its code-block rows and wavelet bands - // spread over every core, still without reparsing. - // Streamed images never hold whole-image scratch, so a group with one - // enters the staged route only to keep its plans. - let streams_rows = !keeps_staged_plans(group, workers_option) - && group - .images - .iter() - .any(|image| image.cpu_execution().streams()); - // Payloads are flattened only for the staged route, so a group whose - // images decode with inner parallelism skips that work. - let has_flattened_payloads = !streams_rows + // spread over every core, still without reparsing. A group with an image + // whose rows stream never stages: streamed images hold a band of rows + // instead of whole-image scratch, at every worker count. + let streams_rows = group + .images + .iter() + .any(|image| image.cpu_execution().streams()); + // Payloads are flattened only for the staged route, and only within the + // flattening and staged-window caps (see `classic_staging_fits`): a + // classic group beyond them decodes its plans one image at a time + // instead of failing. + let staged = !streams_rows && retained_inner_parallelism(group.images.len(), tile_options)? == CpuDecodeParallelism::Serial + && (group.info.route != BatchCodecRoute::Classic + || classic_staging_fits(group.images.iter())?) && fast_workspace.prepare_group(group)?; - // With an explicit worker count, the images of such a group decode in a - // pool of that many threads (see `CpuStreamPool`). - let stream_pool = if streams_rows - && group.images.len() > 1 - && workers_option.is_some_and(|count| count.get() > 1) - { + let (pooled, image_parallelism) = + image_policy(group.images.len(), streams_rows, workers_option); + let stream_pool = if pooled { Some(stream_pool.get(workers.len())?) } else { None @@ -89,9 +90,10 @@ pub(super) fn decode_cpu_group( tile_options, samples_per_image, &mut values, - has_flattened_payloads.then_some(&mut *fast_workspace), + staged.then_some(&mut *fast_workspace), staged_workspace, stream_pool, + image_parallelism, admission, // The only image of a Gray8 group is the whole in-flight set. if group.images.len() == 1 && group.info.color == BatchColor::Gray { @@ -117,9 +119,10 @@ pub(super) fn decode_cpu_group( tile_options, samples_per_image, &mut values, - has_flattened_payloads.then_some(&mut *fast_workspace), + staged.then_some(&mut *fast_workspace), staged_workspace, stream_pool, + image_parallelism, admission, decode_image_u16, convert_u16, @@ -140,9 +143,10 @@ pub(super) fn decode_cpu_group( tile_options, samples_per_image, &mut values, - has_flattened_payloads.then_some(&mut *fast_workspace), + staged.then_some(&mut *fast_workspace), staged_workspace, stream_pool, + image_parallelism, admission, decode_image_i16, round_signed, @@ -186,59 +190,23 @@ pub(super) fn decode_cpu_group( ))) } -/// Most explicit workers for which a group whose images may stream keeps -/// their plans on the staged route (see [`keeps_staged_plans`]): one, or -/// four for Gray8, whose staged route keeps up to four workers busy. -const fn staged_plans_max_workers(color: BatchColor) -> usize { - match color { - BatchColor::Gray => 4, - _ => 1, - } -} - -#[cfg(test)] -std::thread_local! { - /// Test-only planned bytes above which a group streams instead of - /// keeping its plans on the staged route, on this thread. - pub(super) static STAGED_PLANS_CAP_OVERRIDE: core::cell::Cell> = - const { core::cell::Cell::new(None) }; -} - -/// Whether a group of images that may stream decodes their plans on the -/// staged route instead: the caller set one worker (up to four for Gray8), -/// the group holds several classic images, and each has a plan within the -/// per-decode bound, all together within the batch execution cap. Larger -/// groups stream. -/// -/// The staged route decodes each code-block of a window's images back to -/// back, which speeds up classic Tier-1 when the images repeat, and while it -/// keeps its workers busy the stream pool has no idle thread to gain from: -/// 8 copies of one image stream in 1.13-1.32 times the staged time at one -/// worker, and 8 Gray8 copies in 1.12-1.24 times at two to four workers. -/// Distinct images stream as fast or faster, RGB8 streams faster from two -/// workers on and Gray8 from six, and HTJ2K images always stream faster. -/// The window keeps each image's plan scratch, as these settings did before -/// images streamed. -fn keeps_staged_plans(group: &PreparedBatchGroup, workers: Option) -> bool { - group.info.route == BatchCodecRoute::Classic - && group.images.len() > 1 - && workers - .is_some_and(|workers| workers.get() <= staged_plans_max_workers(group.info.color)) - && group - .images - .iter() - .try_fold(0usize, |total, image| { - total.checked_add(image.cpu_execution().plan_bytes()?) - }) - .is_some_and(|total| total <= staged_plans_cap()) -} - -fn staged_plans_cap() -> usize { - #[cfg(test)] - if let Some(cap) = STAGED_PLANS_CAP_OVERRIDE.with(core::cell::Cell::get) { - return cap; - } - batch_execution_cap_bytes() +/// How the images of a group that decode one at a time run: in the stream +/// pool, and with which inner parallelism otherwise (`None`: the +/// scheduler's own). With an explicit worker count and several images, +/// streamed images decode in the pool of that many threads (two or more) and +/// plans serially, also on the worker itself, where offset-plan executions +/// read it; a single worker decodes serially. +pub(super) fn image_policy( + image_count: usize, + streamed: bool, + workers: Option, +) -> (bool, Option) { + let explicit_group = image_count > 1 && workers.is_some(); + let pooled = explicit_group && streamed && workers.is_some_and(|count| count.get() > 1); + ( + pooled, + explicit_group.then_some(CpuDecodeParallelism::Serial), + ) } /// The threads a group's streamed images decode on when the caller set the @@ -310,6 +278,7 @@ fn run_typed_group( flattened_payloads: Option<&mut CpuGroupFastWorkspace>, staged_workspace: &mut CpuStagedWorkspace, stream_pool: Option<&rayon::ThreadPool>, + image_parallelism: Option, admission: &Arc, decode: CpuDecodeFn, convert: fn(f32, u8) -> T, @@ -349,6 +318,10 @@ fn run_typed_group( &mut results, tile_options, |worker, jobs, results| { + // On the worker itself, where offset-plan executions read it. + if let Some(parallelism) = image_parallelism { + worker.set_owned_inner_parallelism(parallelism); + } let parallelism = worker.prepare_owned_decode(); let workspace = worker.take_native_workspace(); let mut context = j2k_native::DecoderContext::from_workspace(workspace); diff --git a/crates/j2k/src/owned_batch/cpu_fast.rs b/crates/j2k/src/owned_batch/cpu_fast.rs index 9b344ee9..b5786127 100644 --- a/crates/j2k/src/owned_batch/cpu_fast.rs +++ b/crates/j2k/src/owned_batch/cpu_fast.rs @@ -10,14 +10,89 @@ use j2k_native::{ J2kDirectCodeBlockIndex, }; +use super::cpu_staged::{staged_window_fits, CPU_ENTROPY_IMAGES_PER_WINDOW}; use super::{BatchCodecRoute, PreparedBatchGroup, PreparedImage}; -use crate::batch::allocation::J2K_BATCH_METADATA_ALLOWANCE_BYTES; +use crate::batch::allocation::{try_vec_with_capacity, J2K_BATCH_METADATA_ALLOWANCE_BYTES}; mod classic; mod ht; mod plan; use self::plan::{checked_metadata_bytes, reserve_reused}; +#[cfg(test)] +std::thread_local! { + /// Test-only cap on a group's flattened metadata, on this thread. + pub(super) static FLATTENED_METADATA_CAP_OVERRIDE: core::cell::Cell> = + const { core::cell::Cell::new(None) }; +} + +fn flattened_metadata_cap() -> usize { + #[cfg(test)] + if let Some(cap) = FLATTENED_METADATA_CAP_OVERRIDE.with(core::cell::Cell::get) { + return cap; + } + J2K_BATCH_METADATA_ALLOWANCE_BYTES +} + +/// The flattened metadata bytes of a classic group of `image_count` images +/// and `payload_count` payloads, as `prepare_group` charges them. +#[cfg(test)] +pub(super) fn classic_flattened_metadata_bytes(image_count: usize, payload_count: usize) -> usize { + checked_metadata_bytes::<(J2kClassicCodeBlockPayload, J2kCodestreamRange)>( + image_count, + payload_count, + ) + .unwrap_or(usize::MAX) +} + +/// Whether flattening `images`, a classic group for the staged route in its +/// order, stays within the caps `CpuGroupFastWorkspace::prepare_group` and +/// the staged windows enforce: the flattened metadata and compressed arena +/// of the whole group, and the entropy jobs of its widest window, counted as +/// if a window's code-blocks shared one tile. A group that does not fit +/// decodes its plans one image at a time instead, which needs none of them. +/// +/// # Errors +/// Only when the per-image counts cannot be allocated. +pub(super) fn classic_staging_fits<'a>( + images: impl ExactSizeIterator, +) -> Result { + let image_count = images.len(); + let mut counts = try_vec_with_capacity(image_count, "J2K CPU staged payload counts")?; + let mut payload_bytes = 0usize; + for image in images { + let Some(plan) = image.classic_plan() else { + return Ok(false); + }; + counts.push(plan.payload_count()); + for payload in plan.cpu_plan().payloads() { + payload_bytes = payload_bytes.saturating_add(payload.combined_length); + } + } + let payload_count = counts + .iter() + .fold(0usize, |total, &count| total.saturating_add(count)); + let window = CPU_ENTROPY_IMAGES_PER_WINDOW.min(image_count); + let widest_window = counts + .windows(window.max(1)) + .map(|counts| { + counts + .iter() + .fold(0usize, |total, &count| total.saturating_add(count)) + }) + .max() + .unwrap_or(0); + Ok( + checked_metadata_bytes::<(J2kClassicCodeBlockPayload, J2kCodestreamRange)>( + image_count, + payload_count, + ) + .is_ok_and(|bytes| bytes <= flattened_metadata_cap()) + && payload_bytes <= DEFAULT_MAX_HOST_ALLOCATION_BYTES + && staged_window_fits(window, widest_window), + ) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum CpuPayloadBucket { Cleanup, @@ -183,11 +258,11 @@ impl CpuGroupFastWorkspace { payload_bytes: usize, ) -> Result<(), BatchInfrastructureError> { let metadata_bytes = checked_metadata_bytes::(image_count, payload_count)?; - if metadata_bytes > J2K_BATCH_METADATA_ALLOWANCE_BYTES { + if metadata_bytes > flattened_metadata_cap() { return Err(BatchInfrastructureError::AllocationTooLarge { what: "J2K CPU flattened group metadata", requested: metadata_bytes, - cap: J2K_BATCH_METADATA_ALLOWANCE_BYTES, + cap: flattened_metadata_cap(), }); } if payload_bytes > DEFAULT_MAX_HOST_ALLOCATION_BYTES { diff --git a/crates/j2k/src/owned_batch/cpu_staged.rs b/crates/j2k/src/owned_batch/cpu_staged.rs index 940c966d..6f408032 100644 --- a/crates/j2k/src/owned_batch/cpu_staged.rs +++ b/crates/j2k/src/owned_batch/cpu_staged.rs @@ -130,6 +130,12 @@ impl CpuStagedWorkspace { } } +/// Whether a staged window of `image_count` images and `entropy_count` +/// entropy jobs in one tile fits the staged metadata cap. +pub(super) fn staged_window_fits(image_count: usize, entropy_count: usize) -> bool { + ensure_metadata_capacity(image_count, entropy_count).is_ok() +} + fn ensure_metadata_capacity( image_count: usize, entropy_count: usize, diff --git a/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs b/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs index 25736846..202a8f8d 100644 --- a/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs +++ b/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs @@ -3,11 +3,10 @@ use alloc::{sync::Arc, vec, vec::Vec}; use core::num::NonZeroUsize; -use super::super::cpu_execute::STAGED_PLANS_CAP_OVERRIDE; use super::super::{ prepare_batch, BatchDecodeOptions, BatchErrorStage, BatchItemError, BatchLayout, - CpuBatchDecoder, CpuBatchSamples, CpuExecution, DecodeRequest, EncodedImage, IndexedBatchError, - PreparedBatch, + CpuBatchDecoder, CpuBatchSamples, CpuDecodeParallelism, CpuExecution, DecodeRequest, + EncodedImage, IndexedBatchError, PreparedBatch, }; use super::image::{PLANNED_DECODE_BOUND, STREAM_ABOVE_OVERRIDE, STREAM_ABOVE_PLANNED_BYTES}; use j2k_core::Downscale; @@ -17,10 +16,26 @@ fn classic_fixture(width: u32, height: u32, components: u16, precision: u8) -> A } fn fixture(width: u32, height: u32, components: u16, precision: u8, ht: bool) -> Arc<[u8]> { + seeded_fixture(width, height, components, precision, ht, 0) +} + +/// A fixture whose samples differ for every `seed`: the same size, other +/// coded bytes. +fn seeded_fixture( + width: u32, + height: u32, + components: u16, + precision: u8, + ht: bool, + seed: u32, +) -> Arc<[u8]> { let count = width * height * u32::from(components); let samples = if precision <= 8 { (0..count) - .map(|index| u8::try_from((index * 7 + index / 97) & 0xff).expect("masked sample")) + .map(|index| { + u8::try_from((index * 7 + index / 97 + seed * index / 13) & 0xff) + .expect("masked sample") + }) .collect::>() } else { (0..count) @@ -88,21 +103,6 @@ impl Drop for StreamAbove { } } -struct StagedPlansCap; - -impl StagedPlansCap { - fn hold(bytes: Option) -> Self { - STAGED_PLANS_CAP_OVERRIDE.with(|cap| cap.set(bytes)); - Self - } -} - -impl Drop for StagedPlansCap { - fn drop(&mut self) { - STAGED_PLANS_CAP_OVERRIDE.with(|cap| cap.set(None)); - } -} - fn planned_bytes(prepared: &PreparedBatch) -> Vec<(usize, CpuExecution)> { let mut planned = prepared .groups() @@ -255,13 +255,12 @@ fn cpu_prepare_rejects_plans_over_the_bound_without_a_streamed_route() { )); } -/// `codestream` as JP2, with `channel_definitions` in a cdef box. -fn wrap_jp2(codestream: &[u8], channel_definitions: &[crate::J2kChannelDefinition]) -> Arc<[u8]> { - let color = if channel_definitions.is_empty() { - j2k_core::Colorspace::SGray - } else { - j2k_core::Colorspace::SRgb - }; +/// `codestream` as JP2 in `color`, with `channel_definitions` in a cdef box. +fn wrap_jp2( + codestream: &[u8], + color: j2k_core::Colorspace, + channel_definitions: &[crate::J2kChannelDefinition], +) -> Arc<[u8]> { Arc::from( crate::wrap_j2k_codestream( codestream, @@ -307,11 +306,19 @@ fn with_siz_origin(codestream: &[u8], origin: u32) -> Arc<[u8]> { fn gray8_and_rgb8_images_whose_region_decode_cannot_stream_keep_the_plan_route() { let rgb = classic_rgb8(96, 64); let gray = classic_fixture(80, 72, 1, 8); - let streams = [Arc::clone(&rgb), Arc::clone(&gray), wrap_jp2(&gray, &[])]; + let streams = [ + Arc::clone(&rgb), + Arc::clone(&gray), + wrap_jp2(&gray, j2k_core::Colorspace::SGray, &[]), + ]; // A channel definition, or offsets, keep the region decode on whole // component planes, which it hands over at the end. let planes = [ - wrap_jp2(&rgb, &identity_rgb_channel_definitions()), + wrap_jp2( + &rgb, + j2k_core::Colorspace::SRgb, + &identity_rgb_channel_definitions(), + ), with_siz_origin(&rgb, 4096), with_siz_origin(&gray, 4096), ]; @@ -362,8 +369,8 @@ fn gray8_and_rgb8_images_whose_region_decode_cannot_stream_keep_the_plan_route() "input {index}: {execution:?}" ); } - let decoded = all_u8_samples(&decoder.decode_prepared(&prepared).expect("decode")); - assert_eq!(decoded, reference); + let streamed = all_u8_samples(&decoder.decode_prepared(&prepared).expect("decode")); + assert_eq!(streamed, reference); } // Over the bound, only the images that stream their rows are admitted. @@ -504,9 +511,6 @@ fn a_lone_gray8_image_decodes_its_plan_and_gray8_images_in_flight_stream() { assert!(planned_bytes(&prepared) .iter() .all(|(_, execution)| matches!(execution, CpuExecution::StreamedRowsOrPlan { .. }))); - // One worker keeps several images' plans within the cap (see - // `explicit_workers_choose_the_staged_plans_or_the_stream_pool`). - let _cap = StagedPlansCap::hold(Some(0)); let output = decoder.decode_prepared(&prepared).expect("decode"); assert_eq!(output.groups(), reference.groups(), "{copies} copies"); decoder.workspace_stats().prepared_plan_decode_calls() - plan_calls @@ -516,37 +520,83 @@ fn a_lone_gray8_image_decodes_its_plan_and_gray8_images_in_flight_stream() { assert_eq!(decode(1, &rgb), 0, "a lone RGB8 image streams"); } +#[derive(Debug, Clone, Copy)] +enum Pattern { + /// Copies of one image. + Copies(usize), + /// Four different images. + Distinct, + /// One image twice and two others: `[a, b, a, c]`. + Mixed, + /// One image twice around another: `[a, b, a]`. + AroundOne, +} + #[test] -fn explicit_workers_choose_the_staged_plans_or_the_stream_pool() { - let rgb = classic_rgb8(96, 64); - let gray = classic_fixture(80, 72, 1, 8); - for (image, is_gray) in [(&rgb, false), (&gray, true)] { - let inputs = |copies: usize| vec![EncodedImage::full(Arc::clone(image)); copies]; +fn groups_whose_images_stream_never_stage_and_others_stage_at_one_worker() { + let limit = crate::batch::allocation::generic_batch_worker_limit(); + for is_gray in [false, true] { + let (width, height, components) = if is_gray { (80, 72, 1) } else { (96, 64, 3) }; + let images = (0..4) + .map(|seed| seeded_fixture(width, height, components, 8, false, seed)) + .collect::>(); + let inputs = |pattern: Pattern| -> Vec { + let picks: Vec = match pattern { + Pattern::Copies(copies) => vec![0; copies], + Pattern::Distinct => vec![0, 1, 2, 3], + Pattern::Mixed => vec![0, 1, 0, 2], + Pattern::AroundOne => vec![0, 1, 0], + }; + picks + .into_iter() + .map(|pick| EncodedImage::full(Arc::clone(&images[pick]))) + .collect() + }; // Prepared on this thread, where the threshold override applies. - let mut preparer = CpuBatchDecoder::new(single_worker_options()); - // (workers, images, cap on the group's plans, staged windows, pool - // threads): one worker (up to four for Gray8) keeps a multi-image - // group's plans on the staged route while they fit the cap and - // streams them otherwise; more workers stream the images on a pool of - // that many threads; the codec's own count and a lone image keep - // their threads. - let streams = |workers: usize| (!is_gray).then_some(workers); - let five_workers = 5.min(crate::batch::allocation::generic_batch_worker_limit()); - for (workers, copies, cap, staged, pool) in [ - (Some(1), 4, None, true, None), - (Some(2), 3, None, is_gray, streams(2)), - (Some(3), 4, None, is_gray, streams(3)), - (Some(1), 4, Some(1), false, None), - (Some(2), 3, Some(1), false, Some(2)), - (Some(5), 6, None, false, Some(five_workers)), - (Some(3), 1, None, false, None), - (None, 4, None, false, None), + let batch_preparer = CpuBatchDecoder::new(single_worker_options()); + // Copies, distinct and mixed images stream at every worker count, on + // a pool of that many threads when there are several; the codec's + // own count and a lone image keep their threads. + for (workers, pattern, pool) in [ + (Some(1), Pattern::Copies(4), None), + (Some(1), Pattern::Distinct, None), + (Some(1), Pattern::Mixed, None), + (Some(1), Pattern::AroundOne, None), + (Some(2), Pattern::Copies(3), Some(2)), + (Some(2), Pattern::Mixed, Some(2)), + (Some(2), Pattern::AroundOne, Some(2)), + (Some(3), Pattern::Distinct, Some(3)), + (Some(4), Pattern::Mixed, Some(4)), + (Some(6), Pattern::Copies(3), Some(6)), + (Some(8), Pattern::Mixed, Some(8)), + (Some(3), Pattern::Copies(1), None), + (None, Pattern::Copies(4), None), ] { - let reference = preparer - .decode_prepared(&preparer.prepare(inputs(copies)).expect("prepare plans")) + let label = format!("gray {is_gray}, {workers:?} workers, {pattern:?}"); + // Plans that do not stream stage at one worker, as before. + let mut plan_decoder = CpuBatchDecoder::new(single_worker_options()); + let planned = batch_preparer + .prepare(inputs(pattern)) + .expect("prepare plans"); + assert!(planned_bytes(&planned) + .iter() + .all(|(_, execution)| matches!(execution, CpuExecution::OffsetPlan { .. }))); + let reference = plan_decoder + .decode_prepared(&planned) .expect("decode plans"); + let payloads = planned.groups()[0] + .images() + .iter() + .map(|image| image.classic_plan().expect("classic plan").payload_count() as u64) + .sum::(); + let staged = planned.groups()[0].images().len() > 1; + assert_eq!( + plan_decoder.workspace_stats().flattened_payload_jobs(), + if staged { payloads } else { 0 }, + "{label}" + ); let _threshold = StreamAbove::hold(0); - let prepared = preparer.prepare(inputs(copies)).expect("prepare"); + let prepared = batch_preparer.prepare(inputs(pattern)).expect("prepare"); assert!(planned_bytes(&prepared) .iter() .all(|(_, execution)| matches!( @@ -557,18 +607,17 @@ fn explicit_workers_choose_the_staged_plans_or_the_stream_pool() { workers: workers.and_then(NonZeroUsize::new), ..BatchDecodeOptions::default() }); - let _cap = StagedPlansCap::hold(cap); - let decoded = decoder.decode_prepared(&prepared).expect("decode"); - let label = - format!("gray {is_gray}, {workers:?} workers, {copies} images, cap {cap:?}"); - assert_eq!(decoded.errors(), &[] as &[IndexedBatchError], "{label}"); - assert_eq!(decoded.groups(), reference.groups(), "{label}"); + let output = decoder.decode_prepared(&prepared).expect("decode"); + assert_eq!(output.errors(), &[] as &[IndexedBatchError], "{label}"); + assert_eq!(output.groups(), reference.groups(), "{label}"); + let stats = decoder.workspace_stats(); + assert_eq!(stats.flattened_payload_jobs(), 0, "{label}"); + assert_eq!(stats.cross_image_entropy_windows(), 0, "{label}"); assert_eq!( - decoder.workspace_stats().cross_image_entropy_windows() > 0, - staged, + decoder.stream_pool_threads(), + pool.map(|threads: usize| threads.min(limit)), "{label}" ); - assert_eq!(decoder.stream_pool_threads(), pool, "{label}"); } } } @@ -577,12 +626,12 @@ fn explicit_workers_choose_the_staged_plans_or_the_stream_pool() { fn htj2k_groups_stream_with_one_or_two_workers() { let ht = fixture(96, 64, 3, 8, true); let inputs = || vec![EncodedImage::full(Arc::clone(&ht)); 3]; - let mut preparer = CpuBatchDecoder::new(single_worker_options()); - let reference = preparer - .decode_prepared(&preparer.prepare(inputs()).expect("prepare plans")) + let mut batch_preparer = CpuBatchDecoder::new(single_worker_options()); + let reference = batch_preparer + .decode_prepared(&batch_preparer.prepare(inputs()).expect("prepare plans")) .expect("decode plans"); let _threshold = StreamAbove::hold(0); - let prepared = preparer.prepare(inputs()).expect("prepare"); + let prepared = batch_preparer.prepare(inputs()).expect("prepare"); assert!(planned_bytes(&prepared) .iter() .all(|(_, execution)| matches!(execution, CpuExecution::StreamedRowsOrPlan { .. }))); @@ -592,9 +641,9 @@ fn htj2k_groups_stream_with_one_or_two_workers() { workers: NonZeroUsize::new(workers), ..BatchDecodeOptions::default() }); - let decoded = decoder.decode_prepared(&prepared).expect("decode"); - assert_eq!(decoded.errors(), &[] as &[IndexedBatchError], "{workers}"); - assert_eq!(decoded.groups(), reference.groups(), "{workers}"); + let output = decoder.decode_prepared(&prepared).expect("decode"); + assert_eq!(output.errors(), &[] as &[IndexedBatchError], "{workers}"); + assert_eq!(output.groups(), reference.groups(), "{workers}"); assert_eq!( decoder.workspace_stats().cross_image_entropy_windows(), 0, @@ -603,3 +652,140 @@ fn htj2k_groups_stream_with_one_or_two_workers() { assert_eq!(decoder.stream_pool_threads(), pool, "{workers}"); } } + +#[test] +fn explicit_worker_counts_set_the_policy_of_images_decoded_one_at_a_time() { + use super::super::cpu_execute::image_policy; + let workers = NonZeroUsize::new; + let serial = Some(CpuDecodeParallelism::Serial); + // Several images: streamed ones in the pool of two or more threads, and + // plans serial; one worker decodes serially. + assert_eq!(image_policy(3, true, workers(1)), (false, serial)); + assert_eq!(image_policy(3, true, workers(2)), (true, serial)); + assert_eq!(image_policy(3, false, workers(2)), (false, serial)); + // A lone image and automatic workers keep the scheduler's choice. + assert_eq!(image_policy(1, true, workers(2)), (false, None)); + assert_eq!(image_policy(3, true, None), (false, None)); +} + +#[test] +fn plans_beyond_the_flattening_cap_decode_one_image_at_a_time() { + use super::super::cpu_fast::{ + classic_flattened_metadata_bytes, FLATTENED_METADATA_CAP_OVERRIDE, + }; + struct FlattenedCap; + impl FlattenedCap { + fn hold(bytes: usize) -> Self { + FLATTENED_METADATA_CAP_OVERRIDE.with(|cap| cap.set(Some(bytes))); + Self + } + } + impl Drop for FlattenedCap { + fn drop(&mut self) { + FLATTENED_METADATA_CAP_OVERRIDE.with(|cap| cap.set(None)); + } + } + let images = (0..3) + .map(|seed| seeded_fixture(96, 64, 3, 8, false, seed)) + .collect::>(); + let inputs = || { + images + .iter() + .map(|bytes| EncodedImage::full(Arc::clone(bytes))) + .collect::>() + }; + // Small plans that do not stream: one worker stages them. + let decoder = CpuBatchDecoder::new(single_worker_options()); + let prepared = decoder.prepare(inputs()).expect("prepare plans"); + assert!(planned_bytes(&prepared) + .iter() + .all(|(_, execution)| matches!(execution, CpuExecution::OffsetPlan { .. }))); + let payloads = prepared.groups()[0] + .images() + .iter() + .map(|image| image.classic_plan().expect("classic plan").payload_count()) + .sum::(); + let needed = classic_flattened_metadata_bytes(3, payloads); + let mut reference = None; + // At the cap the plans stage; one byte below it each decodes on its + // own, where flattening them would have failed the batch. + for (cap, staged) in [(needed, true), (needed - 1, false), (0, false)] { + let _cap = FlattenedCap::hold(cap); + let mut decoder = CpuBatchDecoder::new(single_worker_options()); + let (groups, errors) = decoder + .decode_prepared(&prepared) + .expect("decode") + .into_parts(); + assert_eq!(errors, [], "cap {cap}"); + match &reference { + Some(reference) => assert_eq!(&groups, reference, "cap {cap}"), + None => reference = Some(groups), + } + assert_eq!( + decoder.workspace_stats().flattened_payload_jobs(), + if staged { payloads as u64 } else { 0 }, + "cap {cap}" + ); + assert_eq!(decoder.parallel_plan_decodes(), 0, "cap {cap}"); + } +} + +#[test] +fn planned_images_among_streamed_ones_keep_the_group_worker_policy() { + let limit = crate::batch::allocation::generic_batch_worker_limit(); + let rgb = classic_rgb8(96, 64); + let gray = classic_fixture(80, 72, 1, 8); + // Copies `a` stream; `b` keeps its offset plan: for RGB8 the same JP2 + // with identity channel definitions, for Gray8 offsets. + let srgb = j2k_core::Colorspace::SRgb; + let cases = [ + ( + wrap_jp2(&rgb, srgb, &[]), + wrap_jp2(&rgb, srgb, &identity_rgb_channel_definitions()), + false, + ), + (Arc::clone(&gray), with_siz_origin(&gray, 4096), true), + ]; + for (copy, planned, is_gray) in cases { + let inputs = || { + [©, &planned, ©] + .map(|bytes| EncodedImage::full(Arc::clone(bytes))) + .to_vec() + }; + let mut batch_preparer = CpuBatchDecoder::new(single_worker_options()); + let reference = batch_preparer + .decode_prepared(&batch_preparer.prepare(inputs()).expect("prepare plans")) + .expect("decode plans"); + let _threshold = StreamAbove::hold(0); + let prepared = batch_preparer.prepare(inputs()).expect("prepare"); + assert_eq!(prepared.groups().len(), 1, "gray {is_gray}"); + let streams = planned_bytes(&prepared) + .iter() + .map(|(_, execution)| execution.streams()) + .collect::>(); + assert_eq!(streams, [true, false, true], "gray {is_gray}"); + // `[a, b, a]`: `b` decodes its plan without nested parallelism, as + // the group's other plans would, and in the pool of two threads. + for workers in [1, 2] { + let label = format!("gray {is_gray}, {workers} workers"); + let mut decoder = CpuBatchDecoder::new(BatchDecodeOptions { + workers: NonZeroUsize::new(workers), + ..BatchDecodeOptions::default() + }); + let output = decoder.decode_prepared(&prepared).expect("decode"); + assert_eq!(output.errors(), &[] as &[IndexedBatchError], "{label}"); + assert_eq!(output.groups(), reference.groups(), "{label}"); + assert_eq!( + decoder.workspace_stats().flattened_payload_jobs(), + 0, + "{label}" + ); + assert_eq!(decoder.parallel_plan_decodes(), 0, "{label}"); + assert_eq!( + decoder.stream_pool_threads(), + (workers > 1).then_some(workers.min(limit)), + "{label}" + ); + } + } +} diff --git a/crates/j2k/src/owned_batch/prepared.rs b/crates/j2k/src/owned_batch/prepared.rs index 4028f6cf..f3b38d1b 100644 --- a/crates/j2k/src/owned_batch/prepared.rs +++ b/crates/j2k/src/owned_batch/prepared.rs @@ -85,9 +85,8 @@ impl CpuExecution { } } - /// Whether the CPU decoder may stream this image's rows; groups holding - /// such an image take the staged route only to keep their plans (see - /// `keeps_staged_plans`). + /// Whether the CPU decoder may stream this image's rows; a group holding + /// such an image never takes the staged route. pub(super) const fn streams(self) -> bool { matches!(self, Self::StreamedRows | Self::StreamedRowsOrPlan { .. }) } From 938ff8b6cd010398ea41293546cdc3b948411f99 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 19:13:25 -0400 Subject: [PATCH 76/94] docs(changelog): note the batch worker pool and streamed batch images --- CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7027b8fa..7b6777ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,22 @@ Staged workspace version: `0.13.2`. decodes, output and peak heap are unchanged, and so are decodes on larger pools, where the faults would contend in the kernel. New dependency: `bytemuck`. +- `CpuBatchDecoder` streams a Gray8/RGB8 image's rows only when its region + decode can stream them. An image with a channel definition box, or with + image or tile offsets, keeps its offset plan, and is rejected up front when + that plan exceeds the per-decode bound, as other outputs are; before, it was + admitted as streamed but decoded whole component planes. A group whose + images decode with inner parallelism no longer flattens their code-block + payloads first. +- `CpuBatchDecoder` with an explicit worker count of two or more decodes a + group's streamed Gray8/RGB8 images on a pool of exactly that many threads, + so threads that finish their images help the images still running instead + of idling: eight 2048 x 2048 RGB images on six workers kept four threads + busy before. Every image whose rows stream does so at every worker count, + repeated or not. A group of plans that do not stream still decodes on the + staged route at one worker, but one image at a time when its flattened + code-block payloads would exceed the staging caps, instead of failing the + batch. Output is unchanged. - Decodes that would exceed a resource limit now fail with `J2kError::ResourceLimit { requested, cap, source }` instead of the generic `J2kError::NativeDecode`, on every decode route: `J2kDecoder`, the tile From ef020bd7e8de6904e60f7e99bf6e4aab07546f05 Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 02:02:11 -0500 Subject: [PATCH 77/94] perf(j2k): stream one-thread full-image decodes where streaming is nearly as fast On one thread (CpuDecodeParallelism::Serial or a one-thread pool) the default full-image route kept the component planes, about 12 bytes per pixel of working memory (more for reduced decodes, which hold the full-resolution coefficients), for several kinds of image the streamed route decodes as fast or within 3%. Measured on an 18-core ARM64 machine at one thread (7 interleaved rounds, one process per route and round, median of the per-round warm ratios streamed / planes; an A/A copy of the planes build stays within 0.987-1.007; bytes identical on every image), these now stream: - classic RGB below 4 coded bits per pixel from 4096 x 2048 samples, as on 2 to 6 threads: 4096 x 2048 to 5428 x 4460 images at 1.5 to 2 bpp 1.016-1.029 (cold 0.97-0.99); a 5428 x 4460 image 309.4 -> 16.0 MiB heap. 2048 x 2048 to 2896 x 2896 ones (1.035-1.041) keep the planes. - classic RGB at 4 bits per pixel or more from 512 x 512 instead of 1024 x 1024, as on 2 to 11 threads: 1.008-1.027, a fifth to a half of the heap (16 tiles of 512 x 512 decoded one by one 1.024). - tiled classic images whose tiles are 256 x 256 or more (512 x 512 before; HTJ2K tiles keep that floor): 1.008-1.014, 193 -> 0.9 MiB for 4096 x 4096 in 256 x 256 tiles. - reduced classic outputs from 512 x 256 (1024 x 512 before): 0.89-0.98, e.g. a 1/8 decode of 5428 x 4460 101.9 -> 1.6 MiB; and reduced tiled images: 0.96-0.99, 53 -> 1.2 MiB. - reduced HTJ2K decodes at 1/4 and below: 0.87-0.94 (cold 0.56-0.64), 41-83 -> 2 MiB. 1/2 decodes (1.009-1.093) keep the planes. - gray classic images from 1024 x 1024 (2896 x 2896 before): 1.004-1.026; reduced gray from 512 x 256: 0.97-1.00, 204 -> 3 MiB; tiled gray in tiles of 256 x 256 or more: 1.006-1.025. - 16-bit outputs of 512 x 512 samples or more, any coding, untiled or in tiles of 256 x 256 or more: the planes route packs them from planes of native samples it fills first, and streaming is faster: HTJ2K 0.755- 0.791, classic 1.5-2 bpp 0.913-0.941 (488 -> 16 MiB for 5428 x 4460), 0.987-1.011 elsewhere. Full-resolution 8-bit HTJ2K images (1.047-1.283 streamed, the planes' fused last level against the streamed line IDWT), HTJ2K gray images (1.048-1.147), smaller images and multi-thread routes are unchanged. The route request carries the decode's scale instead of a reduced flag. The router checks that the planes fit the decode budget, not that the streamed route's own buffers do: a very wide and shallow image (an 8,388,608 x 1 or x 2 RGB image of one level) fits its planes while its reconstruction rows pass the budget, so the stream is refused before it decodes. Such a decode now runs again on the planes, from a fresh context, when the stream was chosen by preference and the planes fit the default budget. This also covers the multi-thread preferences that predate this change (2 to 6 and 12 or more pool threads failed on that image before; one thread kept the planes). --- crates/j2k/src/decode.rs | 36 ++ crates/j2k/src/decode/route.rs | 263 ++++++++--- crates/j2k/src/decode/route/tests.rs | 654 +++++++++++++++++++++++++-- crates/j2k/src/view.rs | 61 +-- 4 files changed, 861 insertions(+), 153 deletions(-) diff --git a/crates/j2k/src/decode.rs b/crates/j2k/src/decode.rs index 6a3bcb20..5dd2b715 100644 --- a/crates/j2k/src/decode.rs +++ b/crates/j2k/src/decode.rs @@ -97,6 +97,42 @@ pub(crate) fn decode_full_image_into_with_native_context<'a>( } } +/// Decode the full `image` into `out` on `route`, which [`full_image_route`] +/// chose for `request`. A stream the budget refuses runs on the planes +/// instead, from a fresh context, when they fit the budget (see +/// `route::planes_fit`); the planes write every row, including rows the +/// stream wrote before it was refused. +pub(crate) fn decode_full_image_on_route<'a>( + image: &Image<'a>, + native_context: &mut j2k_native::DecoderContext<'a>, + out: &mut [u8], + stride: usize, + request: RouteRequest, + route: &FullImageRoute, +) -> Result<(), J2kError> { + let fmt = request.fmt; + match decode_full_image_into_with_native_context(image, native_context, out, stride, fmt, route) + { + Err(J2kError::ResourceLimit { .. }) + if route.streams() && route::planes_fit(image, request) => + { + let parallelism = native_context.cpu_decode_parallelism(); + *native_context = j2k_native::DecoderContext::default(); + native_context.set_cpu_decode_parallelism(parallelism); + let planes = FullImageRoute::Planes; + decode_full_image_into_with_native_context( + image, + native_context, + out, + stride, + fmt, + &planes, + ) + } + decoded => decoded, + } +} + pub(crate) fn decode_image_into_with_native_context<'a>( image: &Image<'a>, native_context: &mut j2k_native::DecoderContext<'a>, diff --git a/crates/j2k/src/decode/route.rs b/crates/j2k/src/decode/route.rs index 0bcb3a40..c86d212f 100644 --- a/crates/j2k/src/decode/route.rs +++ b/crates/j2k/src/decode/route.rs @@ -13,9 +13,12 @@ //! native decoder plans the planes route's charge and whether the region //! decode streams from the codestream before anything is decoded (see //! `Image::full_decode_planes_floor`), so the choice is made once, before -//! the decode, and a decode never reruns after a failure. +//! the decode. Only a stream chosen by preference whose own working memory +//! the budget refuses runs again, on the planes, and only when they fit (see +//! `planes_fit`): streaming by preference never fails a decode the planes +//! finish. -use j2k_core::{PixelFormat, Rect}; +use j2k_core::{Downscale, PixelFormat, Rect}; use j2k_native::CpuDecodeParallelism; use super::output::u16_rows_pack_floor; @@ -46,7 +49,7 @@ impl FullImageRoute { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum PlanesBudget { /// The native decode budget; an image within it still streams where - /// streaming is the faster route (see [`streams_by_default`]). + /// streaming is the better route (see [`streams_by_default`]). Default, /// These bytes and no preference: the planes route unless the planes /// would pass them. A test hook that forces either route. @@ -63,28 +66,21 @@ pub(crate) struct RouteRequest { pub(crate) components: u16, /// The bytes of the coded image, its codestream and container. pub(crate) coded_bytes: usize, - /// Whether the image is decoded below full resolution. - pub(crate) reduced: bool, + /// The resolution the image is decoded at. + pub(crate) scale: Downscale, } /// How a full-image decode of `image` runs for `request`: streamed when the /// output has 8 bits per sample, or 16 bits whose rows carry the native /// samples the planes route packs (see `u16_rows_pack_floor`), the region /// decode streams, and either the planes (or their 16-bit pack) would pass -/// the budget or the streamed route is the faster one for this image. +/// the budget or the streamed route is the better one for this image. /// /// A planning error keeps the planes route, which meets the same error and /// reports it. pub(crate) fn full_image_route(image: &Image<'_>, request: RouteRequest) -> FullImageRoute { - let pack_floor = match request.fmt { - PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Gray8 => 0, - PixelFormat::Rgb16 | PixelFormat::Rgba16 | PixelFormat::Gray16 => { - match u16_rows_pack_floor(image, request.fmt) { - Some(floor) => floor, - None => return FullImageRoute::Planes, - } - } - _ => return FullImageRoute::Planes, + let Some(pack_floor) = pack_floor(image, request.fmt) else { + return FullImageRoute::Planes; }; let planes_budget = match request.budget { PlanesBudget::Default => j2k_native::DEFAULT_MAX_DECODE_BYTES, @@ -108,16 +104,48 @@ pub(crate) fn full_image_route(image: &Image<'_>, request: RouteRequest) -> Full } } +/// What the 16-bit pack of `image` into `fmt` holds besides the planes (see +/// `u16_rows_pack_floor`), nothing for an 8-bit output, or `None` when the +/// streamed rows cannot write `fmt`. +fn pack_floor(image: &Image<'_>, fmt: PixelFormat) -> Option { + match fmt { + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Gray8 => Some(0), + PixelFormat::Rgb16 | PixelFormat::Rgba16 | PixelFormat::Gray16 => { + u16_rows_pack_floor(image, fmt) + } + _ => None, + } +} + +/// Whether a full-image decode of `image` for `request` that streams by +/// preference may run on the planes instead when the budget refuses its +/// stream: at the default budget, when the planes and their 16-bit pack fit +/// it. The planes grow with an image's area and the streamed rows' line +/// buffers with its width, so a wide and shallow image can fit the one and +/// not the other: an RGB image 8,388,608 samples wide and a row or two deep +/// fits its planes, but not its reconstruction rows. +pub(super) fn planes_fit(image: &Image<'_>, request: RouteRequest) -> bool { + let budget = j2k_native::DEFAULT_MAX_DECODE_BYTES; + request.budget == PlanesBudget::Default + && pack_floor(image, request.fmt).is_some_and(|pack_floor| { + !matches!( + image.full_decode_planes_floor(budget), + Ok(Some(floor)) if floor.max(pack_floor) > budget + ) + }) +} + /// Pool threads from which a classic tile of several components /// reconstructs each component on its own pool reader (`parallel_synthesis` /// in the native decoder's pool bands). From there, a full-resolution /// untiled image decodes 3-6% faster on the planes. const CLASSIC_SYNTHESIS_THREADS: usize = 12; -/// Samples from which a full-resolution classic image streams on 2 to 11 +/// Samples from which a full-resolution classic image streams on 1 to 11 /// pool threads (a 512 x 512 image at 6.6 coded bits per pixel and 768 x 512 -/// ones at 9.6 decode 5-10% faster streamed on 6 threads); smaller images -/// spend more of their time setting up the pool bands. +/// ones at 9.6 decode 5-10% faster streamed on 6 threads, 1% slower on one, +/// in a third of the memory); smaller images spend more of their time +/// setting up the pool bands, and on one thread the line IDWT's rows. const MIN_STREAMED_SAMPLES: u64 = 1 << 18; /// Samples below which a full-resolution classic image also streams from @@ -127,14 +155,19 @@ const MIN_STREAMED_SAMPLES: u64 = 1 << 18; const MAX_STREAMED_SAMPLES_ON_A_LARGE_POOL: u64 = 1 << 19; /// Samples a tile needs to stream one after another with the pool behind -/// it (measured on 1024 x 1024 tiles only), and an untiled classic image to -/// stream on one thread: 2048 x 2048 to 4096 x 4096 RGB images at 5-11 -/// coded bits per pixel decode as fast streamed on one thread (0.99-1.01), -/// in a third of the memory. +/// it (measured on 1024 x 1024 tiles only). const MIN_STREAMED_TILE_SAMPLES: u64 = 1 << 20; -/// Output samples from which a reduced-resolution decode streams on any -/// pool (a 1024 x 512 output and larger: 6-53% faster on 18 threads). On 12 +/// Samples a classic tile needs to stream one after another on one thread: +/// 4096 x 4096 and 2048 x 2048 RGB images in 256 x 256 tiles decode 1% +/// slower streamed, in a fiftieth to a two-hundredth of the memory. An HTJ2K +/// tile needs [`MIN_STREAMED_SAMPLES`] (512 x 512 tiles 0.93-0.99 of the +/// planes), the streamed IDWT's rows costing more on narrower tiles. +const MIN_STREAMED_TILE_SAMPLES_ON_ONE_THREAD: u64 = 1 << 16; + +/// Output samples from which a reduced-resolution classic decode streams on +/// any pool (a 1024 x 512 output and larger: 6-53% faster on 18 threads), +/// and an HTJ2K one on two threads or more or on one at 1/4 and below. On 12 /// pool threads or more a smaller classic output gains nothing from /// streaming (a 384 x 256 output of a 768 x 512 image 2-3% slower on 12 /// threads, 2% faster on 18), and smaller HTJ2K outputs are up to 59% slower @@ -146,14 +179,27 @@ const MIN_REDUCED_STREAMED_SAMPLES: u64 = 1 << 19; /// streamed on 6 threads, a 192 x 128 one 1-3% slower. const MIN_SMALL_REDUCED_STREAMED_SAMPLES: u64 = 1 << 16; +/// Output samples from which a smaller reduced classic decode streams on one +/// thread: 512 x 256 to 758 x 532 outputs of 4096 x 2048 to 6061 x 4252 +/// images decode 3-11% faster streamed, in a thirtieth to a sixtieth of the +/// memory (the planes route holds the full-resolution coefficients), while +/// 256 x 256 and 384 x 256 outputs of small images are 0-6% slower. +const MIN_SMALL_REDUCED_STREAMED_SAMPLES_ON_ONE_THREAD: u64 = 1 << 17; + /// Samples from which an untiled one-component classic image streams at -/// full resolution below [`CLASSIC_SYNTHESIS_THREADS`] pool threads: the -/// smallest gray image measured. With the pool bands decoding a band in -/// parts when the rings hold too few bands, 2896 x 2896 to 5792 x 5792 gray -/// images decode 12-15% faster streamed on 6 threads and as fast on one, in -/// a fifth to a seventh of the memory (5792 x 5792: 459 vs 68 MB). +/// full resolution on 2 to 11 pool threads: the smallest gray image +/// measured. With the pool bands decoding a band in parts when the rings +/// hold too few bands, 2896 x 2896 to 5792 x 5792 gray images decode 12-15% +/// faster streamed on 6 threads, in a fifth to a seventh of the memory +/// (5792 x 5792: 459 vs 68 MB). const MIN_STREAMED_GRAY_SAMPLES: u64 = 2896 * 2896; +/// Samples from which a one-component classic image streams at full +/// resolution on one thread: 1024 x 1024 to 4096 x 1024 gray images at 1.5 +/// bits per pixel to lossless decode 0-3% slower streamed, in a fifteenth to +/// a thirtieth of the memory. +const MIN_STREAMED_GRAY_SAMPLES_ON_ONE_THREAD: u64 = 1 << 20; + /// Samples from which a gray image streams on larger pools too: 5792 x 5792 /// decodes as fast streamed on 18 threads (0.96-1.02), 2896 x 2896 and /// 4096 x 4096 are 3-14% slower. @@ -162,7 +208,7 @@ const MIN_STREAMED_GRAY_SAMPLES_ON_A_LARGE_POOL: u64 = 5792 * 5792; /// Coded bits per sample from which a full-resolution untiled classic image /// streams as [`full_resolution_streams`] says: below it Tier-1 is light, the /// decode is bound by the inverse transforms and the pack, and the planes' -/// fused last level does those faster on one thread and on small images. +/// fused last level does those faster on small images. const MIN_STREAMED_CODED_BITS_PER_SAMPLE: u64 = 4; /// Samples from which a full-resolution classic image below @@ -171,8 +217,11 @@ const MIN_STREAMED_CODED_BITS_PER_SAMPLE: u64 = 4; /// [`CLASSIC_SYNTHESIS_THREADS`] or more: with its rows packed through the /// fused RGB8 pack, a 4096 x 2048 image at 2 bits per pixel decodes 4-7% /// faster streamed on 4 to 6 threads and 5% faster on 18, in less than half -/// the memory, and 1% slower on one; 512 x 512 ones at 2 bits per pixel are -/// 22% slower on 6 threads. +/// the memory; on one thread 4096 x 2048 to 5428 x 4460 images at 1.5 to 2 +/// bits per pixel decode 2-3% slower streamed, in a tenth to a twentieth of +/// the memory (5428 x 4460: 309 vs 16 MiB), and 2048 x 2048 to 2896 x 2896 +/// ones 4% slower; 512 x 512 ones at 2 bits per pixel are 22% slower on 6 +/// threads. const MIN_STREAMED_LOW_RATE_SAMPLES: u64 = 1 << 23; /// Pool threads up to which a large image below @@ -183,44 +232,69 @@ const MIN_STREAMED_LOW_RATE_SAMPLES: u64 = 1 << 23; const MAX_LOW_RATE_SHARED_READER_THREADS: usize = 6; /// Whether an image the planes route can decode streams anyway, because -/// streaming was measured at least as fast for its kind; for three -/// components: +/// streaming was measured at least as fast for its kind, or on one thread +/// nearly as fast in much less memory; for three components: /// - a full-resolution tiled image whose tiles decode at once (see /// [`tile_workers`]); /// - an untiled reduced image of [`MIN_REDUCED_STREAMED_SAMPLES`] output /// samples or more: classic at any thread count, HTJ2K on two threads or -/// more (one thread decodes a half resolution HTJ2K image 2-9% faster on -/// the planes); a smaller classic one of -/// [`MIN_SMALL_REDUCED_STREAMED_SAMPLES`] or more on 2 to 11 pool threads; +/// more and on one from 1/4 (1/4 outputs of 2896 x 2896 to 4096 x 4096 +/// images at 1 to 12 bits per pixel 6-13% faster streamed, in a fifteenth +/// of the memory; 1/2 outputs 1-9% slower); a smaller classic one of +/// [`MIN_SMALL_REDUCED_STREAMED_SAMPLES`] or more on 2 to 11 pool threads +/// and of [`MIN_SMALL_REDUCED_STREAMED_SAMPLES_ON_ONE_THREAD`] or more on +/// one; on one thread a tiled one too (4096 x 4096 RGB in 1024 x 1024 +/// tiles at 1/2 and 1/4 1-4% faster, in a fiftieth of the memory); /// - a full-resolution classic image of at least 4 coded bits per pixel, or a -/// large one at fewer on most pools (see [`full_resolution_streams`]); a tiled -/// one whose tiles do not decode at +/// large one at fewer on most pools and on one thread (see +/// [`full_resolution_streams`]); a tiled one whose tiles do not decode at /// once streams them one after another, so each of its tiles must meet /// that as an image of its own, of [`MIN_STREAMED_TILE_SAMPLES`] or more /// (4096 x 4096 RGB in 16 tiles of 1024 x 1024: 116.9 vs 124.9 ms on the -/// planes with 6 threads). +/// planes with 6 threads); on one thread any classic tile of +/// [`MIN_STREAMED_TILE_SAMPLES_ON_ONE_THREAD`] or more streams, and any +/// HTJ2K tile of [`MIN_STREAMED_SAMPLES`] or more. /// -/// An untiled one-component classic image streams at full resolution as -/// [`gray_streams`] says; a tiled three-component one on one thread streams -/// its tiles one after another from 512 x 512 tiles (as fast as the planes: -/// 0.93-1.00, in a quarter of the memory). Reduced and tiled gray images, -/// full-resolution HTJ2K images (whose streamed IDWT and pack cost more per -/// sample than the planes' fused last level), small images and reduced -/// tiled images (one tile after another was 1-13% slower on 6 threads, -/// 1.5-10 times on 18) keep the planes. +/// A one-component classic image streams as [`gray_streams`] says, or on +/// one thread at full resolution in tiles of +/// [`MIN_STREAMED_TILE_SAMPLES_ON_ONE_THREAD`] or more, and a 16-bit output +/// on one thread as [`sixteen_bit_streams_on_one_thread`] says. Otherwise +/// untiled HTJ2K images at full resolution (whose streamed IDWT and pack +/// cost more per sample than the planes' fused last level: 5-28% slower on +/// one thread), HTJ2K gray images, small images and, on pools, reduced tiled +/// images (one tile after another was 1-13% slower on 6 threads, 1.5-10 +/// times on 18) keep the planes. fn streams_by_default(image: &Image<'_>, request: RouteRequest) -> bool { let threads = pool_threads(request.parallelism); let htj2k = image.uses_high_throughput_block_coding(); let tiled = image.tile_count() > 1; let mut samples = u64::from(image.width()) * u64::from(image.height()); + if threads == 1 + && matches!(request.components, 1 | 3) + && matches!( + request.fmt, + PixelFormat::Rgb16 | PixelFormat::Rgba16 | PixelFormat::Gray16 + ) + && sixteen_bit_streams_on_one_thread(image, samples) + { + return true; + } if request.components == 1 { - return !request.reduced && !tiled && !htj2k && gray_streams(samples, threads); + let reduced = request.scale != Downscale::None; + if htj2k { + return false; + } + if tiled && !reduced && threads == 1 { + return average_tile_samples(image, samples) + .is_some_and(|tile| tile >= MIN_STREAMED_TILE_SAMPLES_ON_ONE_THREAD); + } + return gray_streams(samples, reduced, tiled, threads); } if request.components != 3 { return false; } - if request.reduced { - return !tiled && reduced_streams(samples, htj2k, threads); + if request.scale != Downscale::None { + return (!tiled || threads == 1) && reduced_streams(samples, request.scale, htj2k, threads); } let mut coded_bytes = request.coded_bytes; if tiled { @@ -234,7 +308,12 @@ fn streams_by_default(image: &Image<'_>, request: RouteRequest) -> bool { samples /= u64::try_from(tile_count).unwrap_or(u64::MAX); coded_bytes /= tile_count; if threads == 1 { - return samples >= MIN_STREAMED_SAMPLES; + return samples + >= if htj2k { + MIN_STREAMED_SAMPLES + } else { + MIN_STREAMED_TILE_SAMPLES_ON_ONE_THREAD + }; } if samples < MIN_STREAMED_TILE_SAMPLES { return false; @@ -243,19 +322,62 @@ fn streams_by_default(image: &Image<'_>, request: RouteRequest) -> bool { !htj2k && full_resolution_streams(samples, coded_bytes, threads) } -/// Whether an untiled one-component classic image of `samples` streams at -/// full resolution on `threads` pool threads. -fn gray_streams(samples: u64, threads: usize) -> bool { - samples >= MIN_STREAMED_GRAY_SAMPLES +/// Whether a 16-bit decode of `image`, of `samples` output samples, streams +/// on one thread whatever its coding: from [`MIN_STREAMED_SAMPLES`], untiled +/// or in tiles of [`MIN_STREAMED_TILE_SAMPLES_ON_ONE_THREAD`] or more. The +/// planes route packs such outputs from planes of native samples it fills +/// first, which the streamed rows skip: 4096 x 2048 HTJ2K images decode +/// 21-25% faster streamed, 2896 x 2896 to 5428 x 4460 classic images at 1.5 +/// bits per pixel 6-9% faster and 512 x 512 to 768 x 768 ones 1% faster, +/// while smaller images and tiles are 4-5% slower. +fn sixteen_bit_streams_on_one_thread(image: &Image<'_>, samples: u64) -> bool { + samples >= MIN_STREAMED_SAMPLES + && (image.tile_count() <= 1 + || average_tile_samples(image, samples) + .is_some_and(|tile| tile >= MIN_STREAMED_TILE_SAMPLES_ON_ONE_THREAD)) +} + +/// The output samples of an average tile of `image`, of `samples` output +/// samples in all, or `None` when its tiles cannot be listed. +fn average_tile_samples(image: &Image<'_>, samples: u64) -> Option { + let tiles = image.output_tile_rects().ok()?.len().max(1); + Some(samples / u64::try_from(tiles).unwrap_or(u64::MAX)) +} + +/// Whether a one-component classic image of `samples` output samples, +/// `tiled` or not, streams on `threads` pool threads: on one thread reduced +/// to [`MIN_SMALL_REDUCED_STREAMED_SAMPLES_ON_ONE_THREAD`] or more, tiled or +/// not (5100 x 6600 and 5792 x 5792 images at 1/2 to 1/8 0-3% faster, in a +/// fiftieth to a hundredth of the memory), or untiled at full resolution +/// from [`MIN_STREAMED_GRAY_SAMPLES_ON_ONE_THREAD`]; on more threads untiled +/// at full resolution from [`MIN_STREAMED_GRAY_SAMPLES`], and from 12 threads +/// only from [`MIN_STREAMED_GRAY_SAMPLES_ON_A_LARGE_POOL`]. A tiled gray image +/// at full resolution streams on one thread in tiles of +/// [`MIN_STREAMED_TILE_SAMPLES_ON_ONE_THREAD`] or more (2048 x 2048 in +/// 512 x 512 tiles to 5100 x 6600 in 1024 x 1024 ones 1-3% slower, in a +/// fortieth to a hundredth of the memory). +fn gray_streams(samples: u64, reduced: bool, tiled: bool, threads: usize) -> bool { + if threads == 1 { + return if reduced { + samples >= MIN_SMALL_REDUCED_STREAMED_SAMPLES_ON_ONE_THREAD + } else { + !tiled && samples >= MIN_STREAMED_GRAY_SAMPLES_ON_ONE_THREAD + }; + } + !reduced + && !tiled + && samples >= MIN_STREAMED_GRAY_SAMPLES && (threads < CLASSIC_SYNTHESIS_THREADS || samples >= MIN_STREAMED_GRAY_SAMPLES_ON_A_LARGE_POOL) } -/// Whether a reduced-resolution decode of an untiled three-component image -/// to `samples` output samples streams on `threads` pool threads. -fn reduced_streams(samples: u64, htj2k: bool, threads: usize) -> bool { +/// Whether a reduced-resolution decode of a three-component image at +/// `scale` to `samples` output samples streams on `threads` pool threads. +fn reduced_streams(samples: u64, scale: Downscale, htj2k: bool, threads: usize) -> bool { if samples >= MIN_REDUCED_STREAMED_SAMPLES { - !htj2k || threads >= 2 + !htj2k || threads >= 2 || (threads == 1 && scale.denominator() >= 4) + } else if threads == 1 { + !htj2k && samples >= MIN_SMALL_REDUCED_STREAMED_SAMPLES_ON_ONE_THREAD } else { !htj2k && samples >= MIN_SMALL_REDUCED_STREAMED_SAMPLES @@ -265,11 +387,10 @@ fn reduced_streams(samples: u64, htj2k: bool, threads: usize) -> bool { /// Whether a full-resolution three-component classic image (or tile) of /// `samples` pixels coded in `coded_bytes` streams on `threads` pool threads: -/// from [`MIN_STREAMED_SAMPLES`] and [`MIN_STREAMED_CODED_BITS_PER_SAMPLE`], -/// on 2 to 11 pool threads, on more when it is below -/// [`MAX_STREAMED_SAMPLES_ON_A_LARGE_POOL`], and on one from -/// [`MIN_STREAMED_TILE_SAMPLES`]; at fewer coded bits, from -/// [`MIN_STREAMED_LOW_RATE_SAMPLES`] on 2 to +/// from [`MIN_STREAMED_SAMPLES`] and [`MIN_STREAMED_CODED_BITS_PER_SAMPLE`] +/// below [`CLASSIC_SYNTHESIS_THREADS`] pool threads, and on more when it is +/// below [`MAX_STREAMED_SAMPLES_ON_A_LARGE_POOL`]; at fewer coded bits, from +/// [`MIN_STREAMED_LOW_RATE_SAMPLES`] on up to /// [`MAX_LOW_RATE_SHARED_READER_THREADS`] pool threads and from /// [`CLASSIC_SYNTHESIS_THREADS`]. fn full_resolution_streams(samples: u64, coded_bytes: usize, threads: usize) -> bool { @@ -278,13 +399,13 @@ fn full_resolution_streams(samples: u64, coded_bytes: usize, threads: usize) -> .saturating_mul(8); if coded_bits < samples.saturating_mul(MIN_STREAMED_CODED_BITS_PER_SAMPLE) { return samples >= MIN_STREAMED_LOW_RATE_SAMPLES - && (matches!(threads, 2..=MAX_LOW_RATE_SHARED_READER_THREADS) + && (matches!(threads, 1..=MAX_LOW_RATE_SHARED_READER_THREADS) || threads >= CLASSIC_SYNTHESIS_THREADS); } samples >= MIN_STREAMED_SAMPLES && match threads { - 0 | 1 => samples >= MIN_STREAMED_TILE_SAMPLES, - 2..CLASSIC_SYNTHESIS_THREADS => true, + 0 => samples >= MIN_STREAMED_TILE_SAMPLES, + 1..CLASSIC_SYNTHESIS_THREADS => true, _ => samples < MAX_STREAMED_SAMPLES_ON_A_LARGE_POOL, } } @@ -297,7 +418,7 @@ fn pool_threads(parallelism: CpuDecodeParallelism) -> usize { } } -/// Tiles from which an image decodes one tile after another rather than at +/// Tiles above which an image decodes one tile after another rather than at /// once: each tile's region decode parses the tile-part headers of the whole /// image again, a cost that grows with the square of the tiles. const MAX_TILES_AT_ONCE: usize = 1024; @@ -311,8 +432,8 @@ const MIN_SAMPLES_PER_TILE: usize = 256 * 256; /// streamed region decode runs on. Each worker decodes its tile alone, so /// the image needs a tile for every worker, and its tiles must keep at least /// 80% of the workers busy over the rounds they take: 16 tiles on 7 workers -/// take three rounds with a third of them idle, and decode no faster than -/// the planes. Other images decode one tile after another with the pool +/// take three rounds with about a quarter of them idle, and decode no faster +/// than the planes. Other images decode one tile after another with the pool /// behind each. fn tile_workers(rects: &[(u32, u32, u32, u32)], threads: usize) -> Option { let tiles = rects.len(); diff --git a/crates/j2k/src/decode/route/tests.rs b/crates/j2k/src/decode/route/tests.rs index 6586b5e3..1e917c0b 100644 --- a/crates/j2k/src/decode/route/tests.rs +++ b/crates/j2k/src/decode/route/tests.rs @@ -3,10 +3,11 @@ use super::{full_image_route, FullImageRoute, PlanesBudget, RouteRequest}; use crate::backend::{image_at_scale, Image}; use crate::decode::{ - decode_full_image_into_with_native_context, decode_native_components_with_native_context, - native_components_route, native_region_components_route, NativeRoute, + decode_full_image_into_with_native_context, decode_full_image_on_route, + decode_native_components_with_native_context, native_components_route, + native_region_components_route, NativeRoute, }; -use crate::DecodeSettings; +use crate::{DecodeSettings, J2kDecoder, J2kError, J2kScratchPool}; use j2k_core::{Downscale, PixelFormat, Rect}; use j2k_native::{encode, CpuDecodeParallelism, EncodeOptions}; use j2k_native::{DecodeError, DecoderContext, DecodingError}; @@ -31,7 +32,7 @@ fn request(fmt: PixelFormat, budget: usize, parallelism: CpuDecodeParallelism) - parallelism, components: 3, coded_bytes: 0, - reduced: false, + scale: Downscale::None, } } @@ -333,8 +334,8 @@ fn tiles_at_once_need_a_busy_tile_for_every_worker() { assert_eq!(workers(2048, 6), None); } -/// Without a fixed budget, tiny reduced images, gray, reduced tiled and -/// small full-resolution images keep the planes. +/// Without a fixed budget, tiny reduced, gray and tiled images and small +/// full-resolution images keep the planes. #[test] fn the_default_streams_where_streaming_measured_faster() { let options = |use_ht_block_coding| EncodeOptions { @@ -363,7 +364,7 @@ fn the_default_streams_where_streaming_measured_faster() { parallelism, components, coded_bytes: bytes.len(), - reduced: scale != Downscale::None, + scale, }, ) }; @@ -505,19 +506,19 @@ fn tiles_at_once_report_the_first_failing_tile() { } } -/// Full-resolution classic images stream from 4 coded bits per pixel on 2 -/// to 11 pool threads from 512 x 512 samples, on larger pools too below 2^19 -/// samples, and on one thread from a megapixel; at fewer bits per pixel from -/// 4096 x 2048 samples on pools of 2 to 6 threads and of 12 or more. +/// Full-resolution classic images stream from 4 coded bits per pixel on 1 +/// to 11 pool threads from 512 x 512 samples and on larger pools too below +/// 2^19 samples; at fewer bits per pixel from 4096 x 2048 samples on pools +/// of up to 6 threads and of 12 or more. #[test] fn full_resolution_images_stream_when_tier1_bound() { use super::full_resolution_streams; let mosaic = 4096 * 2048; // 11 MB lossless and 2.2 MB at 2 bits per pixel. assert!(full_resolution_streams(mosaic, 11_104_128, 6)); - // At 2 bits per pixel on small and large pools, but not on one thread or - // on the 7 to 11 threads where one pool reader holds the decode back. - for threads in [2, 6, 12, 18] { + // At 2 bits per pixel on one thread and on small and large pools, but + // not on the 7 to 11 threads where one pool reader holds the decode back. + for threads in [1, 2, 6, 12, 18] { assert!( full_resolution_streams(mosaic, 2_246_803, threads), "{threads}" @@ -535,14 +536,24 @@ fn full_resolution_images_stream_when_tier1_bound() { "{threads}" ); } - // One thread: from a megapixel. - assert!(full_resolution_streams(mosaic, 11_104_128, 1)); - assert!(!full_resolution_streams(mosaic, 2_246_803, 1)); - assert!(full_resolution_streams(mosaic, 11_104_128, 2)); - assert!(full_resolution_streams(mosaic, 11_104_128, 11)); + // 5428 x 4460 at 1.5 bits per pixel streams on one thread, 2896 x 2896 + // at 1.5 keeps the planes on any pool. + assert!(full_resolution_streams(5428 * 4460, 4_553_929, 1)); + for threads in [1, 6, 18] { + assert!( + !full_resolution_streams(2896 * 2896, 1_579_087, threads), + "{threads}" + ); + } + for threads in [1, 2, 11] { + assert!( + full_resolution_streams(mosaic, 11_104_128, threads), + "{threads}" + ); + } // 768 x 512 at 9.6 and 512 x 512 at 6.6 bits per pixel stream on any // pool, 4096 x 1024 at 5.9 only below 12 threads. - for threads in [2, 6, 12, 18] { + for threads in [1, 2, 6, 12, 18] { assert!( full_resolution_streams(768 * 512, 472_804, threads), "{threads}" @@ -552,51 +563,314 @@ fn full_resolution_images_stream_when_tier1_bound() { "{threads}" ); } + assert!(full_resolution_streams(4096 * 1024, 3_114_650, 1)); assert!(full_resolution_streams(4096 * 1024, 3_114_650, 6)); assert!(!full_resolution_streams(4096 * 1024, 3_114_650, 12)); - assert!(!full_resolution_streams(768 * 512, 472_804, 1)); + // No pool threads at all decide as before: from a megapixel at 4 bits per + // pixel or more, never at fewer. + assert!(full_resolution_streams(mosaic, 11_104_128, 0)); + assert!(!full_resolution_streams(768 * 512, 472_804, 0)); + assert!(!full_resolution_streams(5428 * 4460, 4_553_929, 0)); // Below 512 x 512, and 512 x 512 tiles at 2 bits per pixel. - assert!(!full_resolution_streams(500 * 500, 250_000, 6)); - assert!(!full_resolution_streams(512 * 512, 64_947, 6)); + for threads in [1, 6] { + assert!(!full_resolution_streams(500 * 500, 250_000, threads)); + assert!(!full_resolution_streams(512 * 384, 118_125, threads)); + assert!(!full_resolution_streams(512 * 512, 64_947, threads)); + } } -/// Reduced decodes stream from a 1024 x 512 output on any pool (HTJ2K on -/// two threads or more); smaller classic outputs from 256 x 256 on 2 to 11 -/// pool threads. +/// Reduced decodes stream from a 1024 x 512 output on any pool, HTJ2K on two +/// threads or more and on one from 1/4; smaller classic outputs from +/// 256 x 256 on 2 to 11 pool threads and from 512 x 256 on one. #[test] fn reduced_decodes_stream_when_large_or_on_a_mid_sized_pool() { use super::reduced_streams; let (large, small) = (1024 * 512, 384 * 256); + let (half, quarter) = (Downscale::Half, Downscale::Quarter); for threads in [1, 6, 18] { - assert!(reduced_streams(large, false, threads), "{threads}"); - assert!(!reduced_streams(small, true, threads), "{threads}"); + for scale in [half, quarter, Downscale::Eighth] { + assert!(reduced_streams(large, scale, false, threads), "{threads}"); + assert!(!reduced_streams(small, scale, true, threads), "{threads}"); + } } - assert!(!reduced_streams(large, true, 1)); + // On one thread HTJ2K streams at 1/4 and 1/8, not at 1/2. + assert!(!reduced_streams(4 * large, half, true, 1)); + assert!(reduced_streams(large, quarter, true, 1)); + assert!(reduced_streams(large, Downscale::Eighth, true, 1)); for threads in [2, 6, 11] { - assert!(reduced_streams(large, true, threads), "{threads}"); - assert!(reduced_streams(small, false, threads), "{threads}"); + assert!(reduced_streams(4 * large, half, true, threads), "{threads}"); + assert!(reduced_streams(small, half, false, threads), "{threads}"); } - for threads in [1, 12, 18] { - assert!(!reduced_streams(small, false, threads), "{threads}"); + for threads in [12, 18] { + assert!(!reduced_streams(small, half, false, threads), "{threads}"); } - // 256 x 256 streams on 6 threads, 192 x 128 keeps the planes. - assert!(reduced_streams(256 * 256, false, 6)); - assert!(!reduced_streams(192 * 128, false, 6)); + // 256 x 256 streams on 6 threads, 192 x 128 keeps the planes; on one + // thread from 512 x 256. + assert!(reduced_streams(256 * 256, half, false, 6)); + assert!(!reduced_streams(192 * 128, half, false, 6)); + assert!(reduced_streams(512 * 256, Downscale::Eighth, false, 1)); + assert!(!reduced_streams(small, half, false, 1)); + assert!(!reduced_streams(256 * 256, half, false, 1)); + // No pool threads at all decide as before. + assert!(reduced_streams(large, half, false, 0)); + assert!(!reduced_streams(large, quarter, true, 0)); + assert!(!reduced_streams(512 * 256, Downscale::Eighth, false, 0)); } -/// Gray images stream from 2896 x 2896 below 12 pool threads, and on larger -/// pools from 5792 x 5792. +/// Gray images stream at full resolution from 2896 x 2896 on 2 to 11 pool +/// threads and on larger pools from 5792 x 5792, untiled at full resolution +/// only; on one thread untiled from 1024 x 1024 and reduced from 512 x 256, +/// tiled or not. #[test] fn large_gray_images_stream() { use super::gray_streams; - for threads in [1, 6, 11] { - assert!(gray_streams(2896 * 2896, threads), "{threads}"); - assert!(!gray_streams(2048 * 2048, threads), "{threads}"); + for threads in [0, 1, 6, 11] { + assert!( + gray_streams(2896 * 2896, false, false, threads), + "{threads}" + ); + } + for threads in [0, 6, 11] { + assert!( + !gray_streams(2048 * 2048, false, false, threads), + "{threads}" + ); + assert!( + !gray_streams(2896 * 2896, true, false, threads), + "{threads}" + ); + assert!( + !gray_streams(5792 * 5792, false, true, threads), + "{threads}" + ); } for threads in [12, 18] { - assert!(gray_streams(5792 * 5792, threads), "{threads}"); - assert!(!gray_streams(4096 * 4096, threads), "{threads}"); + assert!( + gray_streams(5792 * 5792, false, false, threads), + "{threads}" + ); + assert!( + !gray_streams(4096 * 4096, false, false, threads), + "{threads}" + ); + } + // One thread: 1024 x 1024 and larger, and reduced outputs from 512 x 256. + assert!(gray_streams(1024 * 1024, false, false, 1)); + assert!(!gray_streams(1024 * 512, false, false, 1)); + for tiled in [false, true] { + assert!(gray_streams(1275 * 1650, true, tiled, 1)); + assert!(gray_streams(512 * 256, true, tiled, 1)); + assert!(!gray_streams(256 * 256, true, tiled, 1)); + } +} + +/// `bytes`, a one-tile codestream, claiming a `width` x `height` image and +/// tile. Headers only: the route reads them and plans the planes from them. +fn claiming(bytes: &[u8], (width, height): (u32, u32)) -> Vec { + let bytes = rewrite_siz(bytes, &[6, 22], |_| width); + rewrite_siz(&bytes, &[10, 26], |_| height) +} + +/// The default route of `bytes`, an image of `components`, at `scale` into +/// RGB8 or Gray8 on one thread, as if the codestream were `coded_bytes` long. +fn one_thread_route( + bytes: &[u8], + components: u16, + scale: Downscale, + coded_bytes: usize, +) -> FullImageRoute { + let fmt = if components == 1 { + PixelFormat::Gray8 + } else { + PixelFormat::Rgb8 + }; + one_thread_route_to(bytes, components, scale, coded_bytes, fmt) +} + +/// [`one_thread_route`] into `fmt`: the same with +/// `CpuDecodeParallelism::Serial` and on a pool of one thread. +fn one_thread_route_to( + bytes: &[u8], + components: u16, + scale: Downscale, + coded_bytes: usize, + fmt: PixelFormat, +) -> FullImageRoute { + let image = image_at_scale(bytes, DecodeSettings::default(), scale).expect("image"); + let route = |parallelism| { + full_image_route( + &image, + RouteRequest { + fmt, + budget: PlanesBudget::Default, + parallelism, + components, + coded_bytes, + scale, + }, + ) + }; + let serial = route(CpuDecodeParallelism::Serial); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(1) + .build() + .expect("one-thread pool"); + let one_thread_pool = pool.install(|| route(CpuDecodeParallelism::Auto)); + assert_eq!(serial, one_thread_pool, "{fmt:?} {scale:?}"); + serial +} + +/// On one thread the default streams large low-rate and mid-sized classic +/// RGB images, gray images from a megapixel, reduced and tiled images and +/// HTJ2K images from 1/4, and keeps HTJ2K images at full resolution and 1/2, +/// HTJ2K gray images and small images on the planes. +#[test] +fn one_thread_streams_wherever_streaming_is_nearly_as_fast() { + use FullImageRoute::{Planes, StreamedRegion}; + let options = |use_ht_block_coding| EncodeOptions { + num_decomposition_levels: 3, + reversible: false, + use_ht_block_coding, + ..EncodeOptions::default() + }; + let (classic, ht) = ( + codestream(3, &options(false)), + codestream(3, &options(true)), + ); + let (gray, gray_ht) = ( + codestream(1, &options(false)), + codestream(1, &options(true)), + ); + let full = Downscale::None; + // RGB at 2 and 1.5 bits per pixel, and at 6. + let mosaic = claiming(&classic, (4096, 2048)); + assert_eq!( + one_thread_route_to(&mosaic, 3, full, 2_246_803, PixelFormat::Rgba8), + StreamedRegion + ); + assert_eq!( + one_thread_route(&mosaic, 3, full, 2_246_803), + StreamedRegion + ); + let square = claiming(&classic, (2896, 2896)); + assert_eq!(one_thread_route(&square, 3, full, 1_579_087), Planes); + let mid = claiming(&classic, (1024, 768)); + assert_eq!(one_thread_route(&mid, 3, full, 589_824), StreamedRegion); + let small = claiming(&classic, (512, 384)); + assert_eq!(one_thread_route(&small, 3, full, 147_456), Planes); + // HTJ2K keeps the planes at full resolution and at 1/2, and streams + // from 1/4. + let ht_mosaic = claiming(&ht, (4096, 2048)); + assert_eq!(one_thread_route(&ht_mosaic, 3, full, 12_236_468), Planes); + assert_eq!( + one_thread_route(&ht_mosaic, 3, Downscale::Half, 12_236_468), + Planes + ); + assert_eq!( + one_thread_route(&ht_mosaic, 3, Downscale::Quarter, 2_249_731), + StreamedRegion + ); + // Reduced classic outputs from 512 x 256. + for scale in [Downscale::Quarter, Downscale::Eighth] { + assert_eq!( + one_thread_route(&mosaic, 3, scale, 2_246_803), + StreamedRegion, + "{scale:?}" + ); + } + assert_eq!( + one_thread_route(&mid, 3, Downscale::Eighth, 589_824), + Planes + ); + // Gray: classic from a megapixel and reduced, HTJ2K never. + let gray_square = claiming(&gray, (1024, 1024)); + assert_eq!( + one_thread_route(&gray_square, 1, full, 196_608), + StreamedRegion + ); + let gray_page = claiming(&gray, (5100, 6600)); + assert_eq!( + one_thread_route(&gray_page, 1, Downscale::Quarter, 6_319_456), + StreamedRegion + ); + let gray_strip = claiming(&gray, (1024, 512)); + assert_eq!(one_thread_route(&gray_strip, 1, full, 98_304), Planes); + let gray_ht_square = claiming(&gray_ht, (2048, 2048)); + assert_eq!( + one_thread_route(&gray_ht_square, 1, full, 1_114_149), + Planes + ); +} + +/// On one thread a tiled classic image streams its tiles one after another +/// at full resolution from 256 x 256 tiles, gray or RGB, and an HTJ2K one +/// from 512 x 512 tiles, or from 256 x 256 into 16 bits. A reduced classic +/// decode streams by its output's size whatever the tiles. +#[test] +fn one_thread_streams_tiles_from_256_x_256() { + let encoded = |components: u16, (width, height): (u32, u32), tile: u32, ht: bool| { + let pixels = vec![0x80_u8; (width * height) as usize * usize::from(components)]; + let options = EncodeOptions { + num_decomposition_levels: 3, + tile_size: Some((tile, tile)), + use_ht_block_coding: ht, + ..EncodeOptions::default() + }; + encode(&pixels, width, height, components, 8, false, &options).expect("encode fixture") + }; + let flat = |components, size, tile| encoded(components, size, tile, false); + for components in [3, 1] { + let bytes = flat(components, (1024, 512), 256); + let coded = bytes.len(); + for scale in [Downscale::None, Downscale::Half] { + assert_eq!( + one_thread_route(&bytes, components, scale, coded), + FullImageRoute::StreamedRegion, + "{components} {scale:?}" + ); + } + let bytes = flat(components, (512, 256), 128); + assert_eq!( + one_thread_route(&bytes, components, Downscale::None, bytes.len()), + FullImageRoute::Planes, + "{components}" + ); + } + let ht = encoded(3, (1024, 512), 256, true); + assert_eq!( + one_thread_route(&ht, 3, Downscale::None, ht.len()), + FullImageRoute::Planes + ); + let ht = encoded(3, (1024, 512), 512, true); + assert_eq!( + one_thread_route(&ht, 3, Downscale::None, ht.len()), + FullImageRoute::StreamedRegion + ); + // 16-bit outputs: HTJ2K tiles from 256 x 256. + let ht = encoded(3, (1024, 512), 256, true); + for fmt in [PixelFormat::Rgb16, PixelFormat::Rgba16] { + assert_eq!( + one_thread_route_to(&ht, 3, Downscale::None, ht.len(), fmt), + FullImageRoute::StreamedRegion, + "{fmt:?}" + ); } + let ht = encoded(3, (1024, 512), 128, true); + assert_eq!( + one_thread_route_to(&ht, 3, Downscale::None, ht.len(), PixelFormat::Rgb16), + FullImageRoute::Planes + ); + // A 512 x 256 output of 128 x 128 tiles streams, a full-resolution + // decode of them keeps the planes. + let bytes = flat(3, (1024, 512), 128); + assert_eq!( + one_thread_route(&bytes, 3, Downscale::Half, bytes.len()), + FullImageRoute::StreamedRegion + ); + assert_eq!( + one_thread_route(&bytes, 3, Downscale::None, bytes.len()), + FullImageRoute::Planes + ); } /// Images of more than 8 bits and signed images stream at 16 bits and into @@ -735,3 +1009,297 @@ fn native_region_rows_only_for_images_whose_region_decodes_stream() { } } } + +/// On one thread a 16-bit output streams from 512 x 512 samples whatever the +/// coding, untiled or in tiles of 256 x 256 or more, where an 8-bit one of +/// a full-resolution HTJ2K image keeps the planes. +#[test] +fn one_thread_streams_sixteen_bit_outputs_from_512_x_512() { + use FullImageRoute::{Planes, StreamedRegion}; + let options = |use_ht_block_coding| EncodeOptions { + num_decomposition_levels: 3, + reversible: false, + use_ht_block_coding, + ..EncodeOptions::default() + }; + let (classic, ht) = ( + codestream(3, &options(false)), + codestream(3, &options(true)), + ); + let gray_ht = codestream(1, &options(true)); + let full = Downscale::None; + let ht_mosaic = claiming(&ht, (4096, 2048)); + for fmt in [PixelFormat::Rgb16, PixelFormat::Rgba16] { + assert_eq!( + one_thread_route_to(&ht_mosaic, 3, full, 2_249_731, fmt), + StreamedRegion, + "{fmt:?}" + ); + } + assert_eq!(one_thread_route(&ht_mosaic, 3, full, 2_249_731), Planes); + // 1.5 bits per pixel below 4096 x 2048, and 512 x 512 at 6. + let square = claiming(&classic, (2896, 2896)); + assert_eq!( + one_thread_route_to(&square, 3, full, 1_579_087, PixelFormat::Rgb16), + StreamedRegion + ); + let tile = claiming(&classic, (512, 512)); + assert_eq!( + one_thread_route_to(&tile, 3, full, 196_608, PixelFormat::Rgb16), + StreamedRegion + ); + let small = claiming(&classic, (512, 384)); + assert_eq!( + one_thread_route_to(&small, 3, full, 147_456, PixelFormat::Rgb16), + Planes + ); + let gray_ht_square = claiming(&gray_ht, (2048, 2048)); + assert_eq!( + one_thread_route_to(&gray_ht_square, 1, full, 1_114_149, PixelFormat::Gray16), + StreamedRegion + ); + // The reduced output counts: 1/2 of 1024 x 512 is below 512 x 512. + let strip = claiming(&ht, (1024, 512)); + assert_eq!( + one_thread_route_to(&strip, 3, Downscale::Half, 1_000_000, PixelFormat::Rgb16), + Planes + ); +} + +/// `bytes`, a one-tile codestream of one coding style with the largest +/// precincts, as a `width` x `height` image whose packets are all empty: its +/// main header with the image and tile resized, then one tile-part of a zero +/// byte per packet. It decodes to mid-level samples whatever its size, +/// without a code-block to decode. +fn with_empty_packets(bytes: &[u8], (width, height): (u32, u32)) -> Vec { + let (mut end, mut levels, mut components) = (2, None, None); + while bytes[end..end + 2] != [0xff, 0x90] { + let length = usize::from(u16::from_be_bytes([bytes[end + 2], bytes[end + 3]])); + match bytes[end + 1] { + 0x51 => components = Some(u16::from_be_bytes([bytes[end + 38], bytes[end + 39]])), + 0x52 => { + assert_eq!(bytes[end + 4] & 1, 0, "the largest precincts"); + levels = Some(u32::from(bytes[end + 9])); + } + 0x53 => panic!("one coding style"), + _ => {} + } + end += 2 + length; + } + let (levels, components) = (levels.expect("COD"), components.expect("SIZ")); + let precincts = |extent: u32| u64::from(extent.div_ceil(1 << 15)); + let per_component: u64 = (0..=levels) + .map(|shift| precincts(width.div_ceil(1 << shift)) * precincts(height.div_ceil(1 << shift))) + .sum(); + let packets = usize::try_from(per_component * u64::from(components)).expect("packets"); + let mut out = claiming(&bytes[..end], (width, height)); + let tile_part = u32::try_from(14 + packets).expect("tile-part length"); + out.extend_from_slice(&[0xff, 0x90, 0, 10, 0, 0]); + out.extend_from_slice(&tile_part.to_be_bytes()); + out.extend_from_slice(&[0, 1, 0xff, 0x93]); + out.resize(out.len() + packets, 0); + out.extend_from_slice(&[0xff, 0xd9]); + out +} + +/// The default request for a full-resolution decode of `bytes`, an image of +/// `components`, into `fmt` with `parallelism`. +fn default_request( + bytes: &[u8], + components: u16, + fmt: PixelFormat, + parallelism: CpuDecodeParallelism, +) -> RouteRequest { + RouteRequest { + fmt, + budget: PlanesBudget::Default, + parallelism, + components, + coded_bytes: bytes.len(), + scale: Downscale::None, + } +} + +/// A wide, shallow RGB image of one 5/3 level in 1024 x 4 code-blocks +/// without a multiple-component transform, of `width` x `height`. +fn wide_and_shallow(width: u32, height: u32) -> Vec { + let options = EncodeOptions { + num_decomposition_levels: 1, + code_block_width_exp: 8, + code_block_height_exp: 0, + use_mct: false, + ..EncodeOptions::default() + }; + with_empty_packets(&codestream(3, &options), (width, height)) +} + +/// A wide, shallow image whose planes fit the decode budget while its +/// streamed rows' line buffers do not (8,388,608 x 1 RGB) is preferred for +/// streaming on one thread and on a pool of six threads. Its stream is +/// refused by the budget before it decodes, and the decode then runs on the +/// planes and writes their bytes. +#[test] +fn a_preferred_stream_past_the_budget_decodes_on_the_planes() { + use CpuDecodeParallelism::{Auto, Serial}; + let width = 1_u32 << 23; + let bytes = wide_and_shallow(width, 1); + let image = image_at_scale(&bytes, DecodeSettings::default(), Downscale::None).expect("image"); + let planes_floor = image.full_decode_planes_floor(0).expect("plan"); + assert!( + planes_floor.is_some_and(|floor| floor <= j2k_native::DEFAULT_MAX_DECODE_BYTES), + "{planes_floor:?}" + ); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(6) + .build() + .expect("pool"); + for fmt in [PixelFormat::Rgb8, PixelFormat::Rgb16] { + let request = default_request(&bytes, 3, fmt, Serial); + let route = full_image_route(&image, request); + assert_eq!(route, FullImageRoute::StreamedRegion, "{fmt:?}"); + let on_the_pool = + pool.install(|| full_image_route(&image, default_request(&bytes, 3, fmt, Auto))); + assert_eq!(on_the_pool, FullImageRoute::StreamedRegion, "{fmt:?}"); + let stride = width as usize * fmt.bytes_per_pixel(); + let mut out = vec![0; stride]; + let refused = decode_full_image_into_with_native_context( + &image, + &mut context(Serial), + &mut out, + stride, + fmt, + &route, + ); + assert!( + matches!(refused, Err(J2kError::ResourceLimit { .. })), + "{fmt:?}: {refused:?}" + ); + let planes = decode_on(&image, Serial, fmt, &FullImageRoute::Planes); + decode_full_image_on_route( + &image, + &mut context(Serial), + &mut out, + stride, + request, + &route, + ) + .unwrap_or_else(|error| panic!("{fmt:?}: {error:?}")); + assert!(out == planes, "{fmt:?}"); + // The decoder's own full-image decode, on one thread and on the pool. + for parallelism in [Some(crate::CpuDecodeParallelism::Serial), None] { + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + if let Some(parallelism) = parallelism { + decoder.set_cpu_decode_parallelism(parallelism); + } + let mut out = vec![0; stride]; + pool.install(|| decoder.decode_into(&mut out, stride, fmt)) + .unwrap_or_else(|error| panic!("{fmt:?} {parallelism:?}: {error:?}")); + assert!(out == planes, "{fmt:?} {parallelism:?}"); + } + } +} + +/// A wide, shallow image whose streamed rows fit the budget keeps its +/// preferred stream on one thread, and the stream writes the planes' bytes: +/// 65,536 x 4 RGB into 16 bits, and 1,048,576 x 8 RGB at a low rate. +#[test] +fn a_wide_image_whose_stream_fits_streams() { + use CpuDecodeParallelism::Serial; + for ((width, height), fmt) in [ + ((1 << 16, 4), PixelFormat::Rgb16), + ((1 << 20, 8), PixelFormat::Rgb8), + ] { + let bytes = wide_and_shallow(width, height); + let image = + image_at_scale(&bytes, DecodeSettings::default(), Downscale::None).expect("image"); + let route = full_image_route(&image, default_request(&bytes, 3, fmt, Serial)); + assert_eq!(route, FullImageRoute::StreamedRegion, "{width} {fmt:?}"); + let streamed = decode_on(&image, Serial, fmt, &route); + let planes = decode_on(&image, Serial, fmt, &FullImageRoute::Planes); + assert!(streamed == planes, "{width} {fmt:?}"); + } +} + +/// Rows of `components` 8-bit samples of a `width` x `height` image with +/// edges and texture. +#[expect( + clippy::cast_possible_truncation, + reason = "each sample keeps the low byte of its pattern on purpose" +)] +fn textured(components: u16, (width, height): (u32, u32)) -> Vec { + let components = u32::from(components); + (0..width * height * components) + .map(|index| { + let (pixel, component) = (index / components, index % components); + let (x, y) = (pixel % width, pixel / width); + ((x * 3 + y * 5) ^ ((x * y) >> 6)).wrapping_add(component * 85) as u8 + }) + .collect() +} + +/// On one thread the decoder's default full-image and reduced decodes of +/// each kind of image that streams there write the planes' bytes: mid-sized +/// RGB, gray from a megapixel, RGB and gray in tiles of 256 x 256, reduced +/// RGB untiled and in tiles, reduced gray, HTJ2K RGB at 1/4, and HTJ2K RGB +/// into 16 bits untiled and in tiles of 256 x 256 (large low-rate RGB in +/// `a_wide_image_whose_stream_fits_streams`). +#[test] +fn one_thread_default_streams_write_the_planes_bytes() { + let encoded = |components: u16, size: (u32, u32), tile: Option, ht: bool| { + let options = EncodeOptions { + num_decomposition_levels: 3, + reversible: false, + tile_size: tile.map(|tile| (tile, tile)), + use_ht_block_coding: ht, + ..EncodeOptions::default() + }; + let pixels = textured(components, size); + encode(&pixels, size.0, size.1, components, 8, false, &options).expect("encode fixture") + }; + let rgb = |size, tile, ht| (encoded(3, size, tile, ht), 3); + let gray = |size, tile| (encoded(1, size, tile, false), 1); + let ht_mosaic = with_empty_packets(&encoded(3, (96, 80), None, true), (4096, 2048)); + let (full, half, quarter) = (Downscale::None, Downscale::Half, Downscale::Quarter); + let (rgb8, rgb16, gray8) = (PixelFormat::Rgb8, PixelFormat::Rgb16, PixelFormat::Gray8); + let cases = [ + ("RGB", rgb((512, 512), None, false), full, rgb8), + ("gray", gray((1024, 1024), None), full, gray8), + ("RGB tiles", rgb((512, 512), Some(256), false), full, rgb8), + ("gray tiles", gray((512, 512), Some(256)), full, gray8), + ("reduced RGB", rgb((1024, 512), None, false), half, rgb8), + ("1/2 tiles", rgb((1024, 512), Some(128), false), half, rgb8), + ("reduced gray", gray((1024, 512), None), half, gray8), + ("HTJ2K at 1/4", (ht_mosaic, 3), quarter, rgb8), + ("HTJ2K 16-bit", rgb((512, 512), None, true), full, rgb16), + ( + "16-bit tiles", + rgb((1024, 512), Some(256), true), + full, + rgb16, + ), + ]; + for (label, (bytes, components), scale, fmt) in cases { + assert_eq!( + one_thread_route_to(&bytes, components, scale, bytes.len(), fmt), + FullImageRoute::StreamedRegion, + "{label}" + ); + let decode = |planes_budget: Option| { + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + decoder.set_cpu_decode_parallelism(crate::CpuDecodeParallelism::Serial); + if let Some(bytes) = planes_budget { + decoder.set_full_decode_planes_budget(bytes); + } + let (width, height) = decoder.info().dimensions; + let denominator = scale.denominator(); + let (width, height) = (width.div_ceil(denominator), height.div_ceil(denominator)); + let stride = width as usize * fmt.bytes_per_pixel(); + let mut out = vec![0; stride * height as usize]; + decoder + .decode_scaled_into(&mut J2kScratchPool::new(), &mut out, stride, fmt, scale) + .unwrap_or_else(|error| panic!("{label}: {error:?}")); + out + }; + assert!(decode(None) == decode(Some(usize::MAX)), "{label}"); + } +} diff --git a/crates/j2k/src/view.rs b/crates/j2k/src/view.rs index bd383f66..4e8cd509 100644 --- a/crates/j2k/src/view.rs +++ b/crates/j2k/src/view.rs @@ -7,7 +7,7 @@ use crate::{ inspect_info_from_image, Image, }, decode::{ - decode_full_image_into_with_native_context, decode_image_region_into_with_native_context, + decode_full_image_on_route, decode_image_region_into_with_native_context, decode_native_components_with_native_context, decode_warnings_for_image, full_image_route, native_components_route, native_region_components_route, validate_buffer, validate_region, J2kDecodeOutcome, J2kDecodedComponents, J2kDecodedNativeComponents, NativeRoute, @@ -412,25 +412,17 @@ impl<'a> J2kDecoder<'a> { let (Some(image), native_context) = (self.image.as_ref(), &mut self.native_context) else { return Err(J2kError::internal_backend("internal image cache missing")); }; - let route = full_image_route( - image, - RouteRequest { - fmt, - budget, - parallelism: native_context.cpu_decode_parallelism(), - components: self.info.components, - coded_bytes: self.bytes.len(), - reduced: false, - }, - ); - let decoded = decode_full_image_into_with_native_context( - image, - native_context, - out, - stride, + let request = RouteRequest { fmt, - &route, - ); + budget, + parallelism: native_context.cpu_decode_parallelism(), + components: self.info.components, + coded_bytes: self.bytes.len(), + scale: Downscale::None, + }; + let route = full_image_route(image, request); + let decoded = + decode_full_image_on_route(image, native_context, out, stride, request, &route); let warnings = decode_warnings_for_image(image); if route.streams() { // As after a region decode, release the decode buffers rather @@ -526,25 +518,16 @@ impl<'a> J2kDecoder<'a> { let image_dims = (image.width(), image.height()); validate_buffer(image_dims, out.len(), stride, fmt)?; let mut native_context = self.fresh_native_context(); - let route = full_image_route( - &image, - RouteRequest { - fmt, - budget: self.full_decode_planes_budget(), - parallelism: native_context.cpu_decode_parallelism(), - components: self.info.components, - coded_bytes: self.bytes.len(), - reduced: true, - }, - ); - decode_full_image_into_with_native_context( - &image, - &mut native_context, - out, - stride, + let request = RouteRequest { fmt, - &route, - )?; + budget: self.full_decode_planes_budget(), + parallelism: native_context.cpu_decode_parallelism(), + components: self.info.components, + coded_bytes: self.bytes.len(), + scale, + }; + let route = full_image_route(&image, request); + decode_full_image_on_route(&image, &mut native_context, out, stride, request, &route)?; Ok(j2k_core::DecodeOutcome::new( Rect::full(image_dims), warnings, @@ -613,8 +596,8 @@ impl<'a> J2kDecoder<'a> { /// What a full-image decode's component planes may charge: the native /// decode budget, past which the planes decode is refused, with - /// streaming wherever it is the faster route, unless a test set a fixed - /// budget. + /// streaming wherever it was measured faster, or on one thread nearly as + /// fast in far less memory, unless a test set a fixed budget. fn full_decode_planes_budget(&self) -> PlanesBudget { self.full_decode_planes_budget .map_or(PlanesBudget::Default, PlanesBudget::Fixed) From 2342422ffe14259fa738b8a40a5e8221911db909 Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 02:02:11 -0500 Subject: [PATCH 78/94] docs(changelog): note the one-thread streamed routes --- CHANGELOG.md | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b6777ac..08a76095 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -213,12 +213,27 @@ Staged workspace version: `0.13.2`. one 2.9 times as fast). Untiled gray classic images stream at full resolution from 2896 x 2896 below 12 pool threads and from 5792 x 5792 on larger pools (12-15% faster on 6 threads, as fast elsewhere, in a fifth to a - seventh of the memory), and on one thread tiled RGB images stream their - tiles one after another and untiled classic RGB images of a megapixel or - more at 4 bits per pixel or more stream too (as fast, in a quarter to a - third of the memory). Reduced and tiled gray images, full-resolution HTJ2K - images, reduced tiled images, small images and images with few tiles on a - large pool keep the planes route. Output is unchanged. + seventh of the memory). On one thread (`CpuDecodeParallelism::Serial` or a + one-thread pool) decodes stream wherever streaming was measured as fast or + nearly so in much less memory: classic RGB images of 512 x 512 or more at + 4 bits per pixel or more and of 4096 x 2048 or more at fewer (a + 5428 x 4460 RGB image at 1.5 bits per pixel 2% slower, in 16 instead of + 309 MiB); classic images in tiles of 256 x 256 or more, one tile after + another; reduced classic outputs of 512 x 256 or more, tiled or not; + reduced HTJ2K outputs of 1024 x 512 or more at 1/4 and below; gray classic + images from 1024 x 1024 or in tiles of 256 x 256 or more, and reduced gray + outputs from 512 x 256; and 16-bit outputs of 512 x 512 samples or more of + any coding, untiled or in tiles of 256 x 256 or more, which the planes + route packs from a second set of planes (HTJ2K images 21-25% faster + streamed). On one thread 8-bit decodes of untiled HTJ2K images at full + resolution, of HTJ2K images at 1/2 and of HTJ2K gray images keep the + planes route, as do small images; on two threads or more reduced and + tiled gray images, untiled full-resolution HTJ2K images, reduced tiled + images, small images and images with few tiles on a large pool do. A + stream chosen this way whose own buffers pass the decode budget, as a very + wide and shallow image's line buffers can while its planes fit, now + decodes on the planes route instead of failing, on any thread count. + Output is unchanged. - Row-streamed decodes of three 8-bit components into RGB8 output write the reconstructed rows straight into the output: 9/7 images with the multiple-component transform pack each row through the planes route's From 7c8533bb8ee1a0879a4ecbd8e70a4269bc732df2 Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 03:58:57 -0500 Subject: [PATCH 79/94] perf(j2k): decode bounded rows in one streamed pass decode_rows_u8_bounded and decode_rows_u16_bounded decoded every stripe as a region decode of its own. They share the parsed tiles, but no decoded code-block samples or reconstructed wavelet rows, so each stripe decoded again every code-block its wavelet support reaches and reconstructed the support rows around its edges: at the default 64-row stripes Tier-1 decoded 6.0 to 7.4 times the code-block samples of one full decode on 2048- to 8287-row images (2.5 to 2.9 times at 256 rows), and the horizontal inverse transform 2.2 to 2.3 times the samples. An image whose tiles span its width and whose headers let its region decode stream its rows now decodes once, as a region decode of the whole image whose rows are packed into a buffer of at most the requested stripe rows and handed to the sink a stripe at a time. A streamed decode that fails before any row reaches the sink decodes the image a stripe at a time instead, so such failures keep the rows and errors of the stripe-by-stripe decode. Rows, their bytes and order, the stripe bound and the errors are unchanged; a decode error after rows reached the sink leaves the stripes before it delivered. A sink that refuses a row still stops the rows at that row, but a one-pass decode now goes on until it completes or fails. The choice is planned from the main and tile headers: images whose tiles sit side by side, whose headers keep region decodes on component planes, or whose main header codes no decomposition level, and 16-bit rows of images deeper than 24 bits, decode a stripe at a time as before, and an image whose tiles turn out not to stream (a tile-level coding style without decomposition levels, HTJ2K code-blocks mixed with classic ones) decodes into component planes of the whole image, its rows handed over a stripe at a time. The stripe bound covers the packed output stripe, not the decoder's working memory. On an 18-core ARM64 machine (median of 7 interleaved rounds, 64-row stripes, time / peak heap / peak resident memory beside input and output, before -> after): - 5661 x 8287 RGB: 1 thread 12.9 s -> 1.34 s, 10.3 -> 22.5 MiB, 31.8 -> 25.6 MiB; 6 threads 2.54 -> 0.23 s, 12.0 -> 46.1 MiB, 76.8 -> 48.8 MiB; 18 threads 1.52 -> 0.11 s, 12.8 -> 69.0 MiB, 77.4 -> 72.8 MiB. - 5428 x 4460 and 6061 x 4252 RGB with selective bypass: 0.09 to 0.13 of the time at 1, 6 and 18 threads, heap up to that of their streamed full-image decode, resident memory down 8 to 34 MiB. - 4096 x 2048 HTJ2K: 0.13 to 0.21 of the time; on 6 and 18 threads heap 8-9 -> 72-73 MiB and resident memory 41-43 -> 76-77 MiB. - 256-row stripes: 0.21 to 0.51 of the time. - 8192 x 8192 in 1024 x 1024 tiles (stripe at a time): unchanged. --- crates/j2k/src/decode.rs | 6 + crates/j2k/src/decode/output.rs | 6 +- crates/j2k/src/decode/output/u16.rs | 57 +++- crates/j2k/src/decode/output/u8.rs | 59 ++-- crates/j2k/src/decode/stripes.rs | 263 ++++++++++++++++++ crates/j2k/src/view/rows.rs | 314 ++++++++++++++++++++- crates/j2k/tests/bounded_rows.rs | 409 ++++++++++++++++++++++++++++ 7 files changed, 1074 insertions(+), 40 deletions(-) create mode 100644 crates/j2k/src/decode/stripes.rs create mode 100644 crates/j2k/tests/bounded_rows.rs diff --git a/crates/j2k/src/decode.rs b/crates/j2k/src/decode.rs index 5dd2b715..49531480 100644 --- a/crates/j2k/src/decode.rs +++ b/crates/j2k/src/decode.rs @@ -19,6 +19,7 @@ pub(crate) use output::{ mod route; mod settings; mod srgb8; +mod stripes; use output::{ can_decode_u8_directly, expand_rgb8_to_rgba8, region_rows_preferred_u16, rows_match_native_u16, write_components_u16_output, write_components_u8_output, write_u16_output, write_u8_output, @@ -28,6 +29,11 @@ pub(crate) use route::{full_image_route, FullImageRoute, PlanesBudget, RouteRequ pub use settings::DecodeSettings; pub(crate) use srgb8::decode_image_srgb8; pub use srgb8::{J2kSrgb8Image, J2kSrgb8Layout}; +pub(crate) use stripes::{ + decode_image_rows_in_stripes, rows_stream_whole, DeliverRow, StreamedRowsError, +}; +#[cfg(test)] +pub(crate) use stripes::{InjectedFailure, INJECTED_FAILURE, STREAMED_ROW_DECODES}; /// Non-fatal JPEG 2000 decode warning surfaced through decode outcomes. #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/crates/j2k/src/decode/output.rs b/crates/j2k/src/decode/output.rs index 0c4c88f9..814dabd1 100644 --- a/crates/j2k/src/decode/output.rs +++ b/crates/j2k/src/decode/output.rs @@ -14,9 +14,9 @@ use native::{planes_pack_floor, region_rows_preferred}; pub(super) use u16::{ region_rows_preferred_u16, rows_match_native_u16, u16_rows_pack_floor, - write_components_u16_output, write_u16_output, U16RowSink, + write_components_u16_output, write_plane_rows_u16, write_u16_output, U16RowSink, }; pub(super) use u8::{ - can_decode_u8_directly, expand_rgb8_to_rgba8, write_components_u8_output, write_u8_output, - U8RowSink, + can_decode_u8_directly, expand_rgb8_to_rgba8, write_components_u8_output, write_plane_rows_u8, + write_u8_output, U8RowSink, }; diff --git a/crates/j2k/src/decode/output/u16.rs b/crates/j2k/src/decode/output/u16.rs index 519e7d53..a6cc9a84 100644 --- a/crates/j2k/src/decode/output/u16.rs +++ b/crates/j2k/src/decode/output/u16.rs @@ -4,6 +4,7 @@ use crate::backend::{ColorSpace, DecodedComponents as NativeDecodedComponents, Image, RawBitmap}; use crate::J2kError; +use core::ops::Range; use j2k_core::{PixelFormat, Unsupported}; use super::u8::{component_sample_count, output_rows, validate_component_planes}; @@ -76,30 +77,54 @@ fn write_planes_u16( out: &mut [u8], stride: usize, fmt: PixelFormat, +) -> Result<(), J2kError> { + let rows = 0..dims.1 as usize; + write_plane_rows_u16(color_space, has_alpha, planes, dims, rows, out, stride, fmt) +} + +/// Write rows `rows` of `planes`, each `dims` in size, to `out` from its +/// first byte, as [`write_planes_u16`] writes those rows of whole planes. +#[expect( + clippy::too_many_arguments, + reason = "the row-range writer keeps the planes' layout, the rows and the output layout explicit" +)] +pub(in crate::decode) fn write_plane_rows_u16( + color_space: &ColorSpace, + has_alpha: bool, + planes: &[j2k_native::ComponentPlane<'_>], + dims: (u32, u32), + rows: Range, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, ) -> Result<(), J2kError> { let expected_samples = component_sample_count(dims)?; let width = dims.0 as usize; - let height = dims.1 as usize; + if rows.start > rows.end || rows.end > dims.1 as usize { + return Err(J2kError::internal_backend( + "component rows outside their planes", + )); + } match (color_space, has_alpha, planes.len(), fmt) { (ColorSpace::Gray, false, 1, PixelFormat::Gray16) => { validate_component_planes(&planes[..1], expected_samples)?; - write_component_rows_u16(&planes[0], out, stride, width, height); + write_component_rows_u16(&planes[0], out, stride, width, rows); Ok(()) } (ColorSpace::RGB, false, 3, PixelFormat::Rgb16) | (ColorSpace::RGB, true, 4, PixelFormat::Rgb16) => { validate_component_planes(&planes[..3], expected_samples)?; - write_rgb_component_rows_u16(planes, out, stride, width, height); + write_rgb_component_rows_u16(planes, out, stride, width, rows); Ok(()) } (ColorSpace::RGB, false, 3, PixelFormat::Rgba16) => { validate_component_planes(&planes[..3], expected_samples)?; - write_rgba_component_rows_u16(planes, out, stride, width, height, true); + write_rgba_component_rows_u16(planes, out, stride, width, rows, true); Ok(()) } (ColorSpace::RGB, true, 4, PixelFormat::Rgba16) => { validate_component_planes(&planes[..4], expected_samples)?; - write_rgba_component_rows_u16(planes, out, stride, width, height, false); + write_rgba_component_rows_u16(planes, out, stride, width, rows, false); Ok(()) } _ => Err(Unsupported { @@ -177,18 +202,20 @@ impl j2k_native::ComponentRowSink for U16RowSink<'_> { } } +/// Rows `rows` of `plane`, `width` samples each, written from the first row +/// of `out` on. fn write_component_rows_u16( plane: &j2k_native::ComponentPlane<'_>, out: &mut [u8], stride: usize, width: usize, - height: usize, + rows: Range, ) { let conversion = U16Conversion::new(plane.bit_depth(), plane.signed()); - for y in 0..height { + for (index, y) in rows.enumerate() { conversion.visit(ChannelWriter { src: &plane.samples()[y * width..(y + 1) * width], - dst: &mut out[y * stride..y * stride + width * 2], + dst: &mut out[index * stride..index * stride + width * 2], channels: 1, channel: 0, }); @@ -200,9 +227,9 @@ fn write_rgb_component_rows_u16( out: &mut [u8], stride: usize, width: usize, - height: usize, + rows: Range, ) { - write_color_component_rows_u16(planes, out, stride, width, height, 3, false); + write_color_component_rows_u16(planes, out, stride, width, rows, 3, false); } fn write_rgba_component_rows_u16( @@ -210,10 +237,10 @@ fn write_rgba_component_rows_u16( out: &mut [u8], stride: usize, width: usize, - height: usize, + rows: Range, synthesize_alpha: bool, ) { - write_color_component_rows_u16(planes, out, stride, width, height, 4, synthesize_alpha); + write_color_component_rows_u16(planes, out, stride, width, rows, 4, synthesize_alpha); } fn write_color_component_rows_u16( @@ -221,7 +248,7 @@ fn write_color_component_rows_u16( out: &mut [u8], stride: usize, width: usize, - height: usize, + rows: Range, output_channels: usize, synthesize_alpha: bool, ) { @@ -235,9 +262,9 @@ fn write_color_component_rows_u16( planes[0].bit_depth(), ) .to_le_bytes(); - for y in 0..height { + for (index, y) in rows.enumerate() { let row = y * width..(y + 1) * width; - let destination = &mut out[y * stride..y * stride + width * output_channels * 2]; + let destination = &mut out[index * stride..index * stride + width * output_channels * 2]; if fused { color[0].visit(PixelWriter { src: [0, 1, 2].map(|channel| &planes[channel].samples()[row.clone()]), diff --git a/crates/j2k/src/decode/output/u8.rs b/crates/j2k/src/decode/output/u8.rs index 47bc0609..e210bbf4 100644 --- a/crates/j2k/src/decode/output/u8.rs +++ b/crates/j2k/src/decode/output/u8.rs @@ -4,6 +4,7 @@ use crate::backend::{ColorSpace, DecodedComponents as NativeDecodedComponents}; use crate::J2kError; +use core::ops::Range; use j2k_core::{PixelFormat, Unsupported}; pub(in crate::decode) fn can_decode_u8_directly( @@ -48,30 +49,54 @@ fn write_planes_u8( out: &mut [u8], stride: usize, fmt: PixelFormat, +) -> Result<(), J2kError> { + let rows = 0..dims.1 as usize; + write_plane_rows_u8(color_space, has_alpha, planes, dims, rows, out, stride, fmt) +} + +/// Write rows `rows` of `planes`, each `dims` in size, to `out` from its +/// first byte, as [`write_planes_u8`] writes those rows of whole planes. +#[expect( + clippy::too_many_arguments, + reason = "the row-range writer keeps the planes' layout, the rows and the output layout explicit" +)] +pub(in crate::decode) fn write_plane_rows_u8( + color_space: &ColorSpace, + has_alpha: bool, + planes: &[j2k_native::ComponentPlane<'_>], + dims: (u32, u32), + rows: Range, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, ) -> Result<(), J2kError> { let expected_samples = component_sample_count(dims)?; let width = dims.0 as usize; - let height = dims.1 as usize; + if rows.start > rows.end || rows.end > dims.1 as usize { + return Err(J2kError::internal_backend( + "component rows outside their planes", + )); + } match (color_space, has_alpha, planes.len(), fmt) { (ColorSpace::Gray, false, 1, PixelFormat::Gray8) => { validate_component_planes(&planes[..1], expected_samples)?; - write_component_rows_u8(&planes[0], out, stride, width, height); + write_component_rows_u8(&planes[0], out, stride, width, rows); Ok(()) } (ColorSpace::RGB, false, 3, PixelFormat::Rgb8) | (ColorSpace::RGB, true, 4, PixelFormat::Rgb8) => { validate_component_planes(&planes[..3], expected_samples)?; - write_rgb_component_rows_u8(planes, out, stride, width, height); + write_rgb_component_rows_u8(planes, out, stride, width, rows); Ok(()) } (ColorSpace::RGB, false, 3, PixelFormat::Rgba8) => { validate_component_planes(&planes[..3], expected_samples)?; - write_rgba_component_rows_u8(planes, out, stride, width, height, true); + write_rgba_component_rows_u8(planes, out, stride, width, rows, true); Ok(()) } (ColorSpace::RGB, true, 4, PixelFormat::Rgba8) => { validate_component_planes(&planes[..4], expected_samples)?; - write_rgba_component_rows_u8(planes, out, stride, width, height, false); + write_rgba_component_rows_u8(planes, out, stride, width, rows, false); Ok(()) } _ => Err(Unsupported { @@ -203,16 +228,18 @@ pub(super) fn output_rows( .map(move |row| &mut row[..row_bytes]) } +/// Rows `rows` of `plane`, `width` samples each, written from the first row +/// of `out` on. fn write_component_rows_u8( plane: &j2k_native::ComponentPlane<'_>, out: &mut [u8], stride: usize, width: usize, - height: usize, + rows: Range, ) { - for y in 0..height { + for (index, y) in rows.enumerate() { let src = &plane.samples()[y * width..(y + 1) * width]; - let dst = &mut out[y * stride..y * stride + width]; + let dst = &mut out[index * stride..index * stride + width]; write_samples_as_u8(src, plane.bit_depth(), dst); } } @@ -222,23 +249,23 @@ fn write_rgb_component_rows_u8( out: &mut [u8], stride: usize, width: usize, - height: usize, + rows: Range, ) { if planes[..3].iter().all(|plane| plane.bit_depth() == 8) { // `sample_as_u8` at 8 bits is the RGB8 pack's rounding and saturation, // which j2k-native runs at the CPU's SIMD level. - for y in 0..height { + for (index, y) in rows.enumerate() { let row = y * width..(y + 1) * width; j2k_native::pack_rgb8( [0, 1, 2].map(|channel| &planes[channel].samples()[row.clone()]), - &mut out[y * stride..y * stride + width * 3], + &mut out[index * stride..index * stride + width * 3], ); } return; } - for y in 0..height { + for (index, y) in rows.enumerate() { let row = y * width; - let dst = &mut out[y * stride..y * stride + width * 3]; + let dst = &mut out[index * stride..index * stride + width * 3]; for x in 0..width { let dst = &mut dst[x * 3..x * 3 + 3]; for channel in 0..3 { @@ -256,12 +283,12 @@ fn write_rgba_component_rows_u8( out: &mut [u8], stride: usize, width: usize, - height: usize, + rows: Range, synthesize_alpha: bool, ) { - for y in 0..height { + for (index, y) in rows.enumerate() { let row = y * width; - let dst = &mut out[y * stride..y * stride + width * 4]; + let dst = &mut out[index * stride..index * stride + width * 4]; for x in 0..width { let dst = &mut dst[x * 4..x * 4 + 4]; for channel in 0..3 { diff --git a/crates/j2k/src/decode/stripes.rs b/crates/j2k/src/decode/stripes.rs new file mode 100644 index 00000000..da2298ab --- /dev/null +++ b/crates/j2k/src/decode/stripes.rs @@ -0,0 +1,263 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Bounded row decodes of a whole image in one streamed region decode. +//! +//! A region decode of the whole image hands its rows to a sink as they are +//! finished, top to bottom when every tile spans the image's width. The +//! sink here packs them into a stripe buffer of at most the caller's +//! stripe height and hands each full stripe to the caller, so every +//! code-block and wavelet row is decoded once however short the stripes +//! are, while the packed output held stays one stripe. + +use core::ops::Range; + +use crate::backend::{ColorSpace, Image}; +use crate::J2kError; +use j2k_core::{DecodeRowsError, PixelFormat}; + +use super::output::{write_plane_rows_u16, write_plane_rows_u8}; + +#[cfg(test)] +std::thread_local! { + /// Test-only count of the streamed bounded row decodes this thread + /// started. + pub(crate) static STREAMED_ROW_DECODES: core::cell::Cell = + const { core::cell::Cell::new(0) }; + /// Test-only: the next streamed bounded row decode on this thread ends + /// with a decode error, in place of its region decode or after it. + pub(crate) static INJECTED_FAILURE: core::cell::Cell> = + const { core::cell::Cell::new(None) }; +} + +/// Where a test makes a streamed bounded row decode fail. +#[cfg(test)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum InjectedFailure { + /// The region decode fails before it hands over any row. + BeforeRows, + /// The region decode fails after it handed over every row. + AfterRows, +} + +/// Whether a bounded row decode of `image`, whose main header codes +/// `resolution_levels` resolutions, can run as one streamed region decode +/// of the whole image: the main header codes at least one decomposition +/// level, every tile spans the image's width, so its rows arrive whole and +/// in order, and the region decode streams its rows as far as the main and +/// tile headers say (see `Image::full_decode_planes_floor`, which parses +/// the tiles and so comes last). +/// +/// A planning error keeps the stripe-by-stripe decode, which meets the same +/// error and reports it. +pub(crate) fn rows_stream_whole(image: &Image<'_>, resolution_levels: u8) -> bool { + resolution_levels > 1 + && image.region_decode_streams_rows() + && (image.tile_count() == 1 + || image.output_tile_rects().is_ok_and(|rects| { + rects + .iter() + .all(|&(x, _, width, _)| x == 0 && width == image.width()) + })) + && matches!(image.full_decode_planes_floor(0), Ok(Some(_))) +} + +/// Why a streamed bounded row decode did not hand over every row. +pub(crate) enum StreamedRowsError { + /// The decode failed before any row reached the caller, who can decode + /// the image a stripe at a time instead and meet that decode's rows and + /// errors, as for an image that cannot stream. + BeforeFirstRow, + /// The caller's sink failed, or the decode failed after rows reached it. + Rows(DecodeRowsError), +} + +/// Hands one packed row, at its image row, to the caller. +pub(crate) type DeliverRow<'d, E> = dyn FnMut(u32, &[u8]) -> Result<(), E> + 'd; + +/// Writes rows `rows` of a rectangle's component planes to packed output. +type PackRows = for<'p> fn( + &ColorSpace, + bool, + &[j2k_native::ComponentPlane<'p>], + (u32, u32), + Range, + &mut [u8], + usize, + PixelFormat, +) -> Result<(), J2kError>; + +/// Decode the whole `image` in one streamed region decode, packing its rows +/// in `fmt` into `stripe`, which holds whole rows of `row_bytes` bytes, and +/// handing each full stripe, then the last, to `deliver` a row at a time. +/// +/// `image` must stream its rows in order (see [`rows_stream_whole`]); a +/// decode that falls back to component planes hands them over at the end, +/// and they are packed a stripe at a time too. The first error stops the +/// rows reaching `deliver`: a caller error is returned as such even though +/// the decode runs to its end, and a decode error after rows reached the +/// caller leaves the rows of every stripe before it delivered and the +/// stripe it failed in held back. +pub(crate) fn decode_image_rows_in_stripes<'a, E: core::error::Error + 'static>( + image: &Image<'a>, + native_context: &mut j2k_native::DecoderContext<'a>, + fmt: PixelFormat, + (stripe, row_bytes): (&mut [u8], usize), + deliver: &mut DeliverRow<'_, E>, +) -> Result<(), StreamedRowsError> { + let internal = + |what| StreamedRowsError::Rows(DecodeRowsError::Decode(J2kError::internal_backend(what))); + let pack: PackRows = match fmt { + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Gray8 => write_plane_rows_u8, + PixelFormat::Rgb16 | PixelFormat::Rgba16 | PixelFormat::Gray16 => write_plane_rows_u16, + _ => return Err(internal("bounded row decode format without a row writer")), + }; + let stripe_rows = stripe.len().checked_div(row_bytes).unwrap_or(0); + if stripe_rows == 0 { + return Err(internal("bounded row decode stripe holds no row")); + } + #[cfg(test)] + STREAMED_ROW_DECODES.with(|count| count.set(count.get() + 1)); + #[cfg(test)] + let injected = INJECTED_FAILURE.with(core::cell::Cell::take); + #[cfg(not(test))] + let skip_decode = false; + #[cfg(test)] + let skip_decode = injected == Some(InjectedFailure::BeforeRows); + let mut sink = StripeSink { + color_space: image.color_space(), + has_alpha: image.has_alpha(), + fmt, + pack, + width: image.width(), + stripe, + row_bytes, + stripe_rows, + first: 0, + held: 0, + deliver, + delivered: false, + error: None, + }; + let decoded = if skip_decode { + Ok(()) + } else { + image.decode_region_rows_for_integer_output_with_context( + (0, 0, image.width(), image.height()), + native_context, + &mut sink, + ) + }; + #[cfg(test)] + let decoded = match injected { + Some(_) => decoded.and(Err(j2k_native::DecodingError::CodeBlockDecodeFailure.into())), + None => decoded, + }; + if let Some(error) = sink.error.take() { + return Err(StreamedRowsError::Rows(error)); + } + if let Err(error) = decoded { + return Err(if sink.delivered { + StreamedRowsError::Rows(DecodeRowsError::Decode(J2kError::from_native_decode_error( + error, + ))) + } else { + StreamedRowsError::BeforeFirstRow + }); + } + sink.flush().map_err(StreamedRowsError::Rows)?; + if sink.first != image.height() { + return Err(internal("streamed region decode left rows undelivered")); + } + Ok(()) +} + +/// Packs a streamed region decode's rows into a stripe buffer and hands +/// each full stripe over. The first error stops every later row. +struct StripeSink<'s, E: core::error::Error + 'static> { + color_space: &'s ColorSpace, + has_alpha: bool, + fmt: PixelFormat, + pack: PackRows, + width: u32, + stripe: &'s mut [u8], + row_bytes: usize, + stripe_rows: usize, + /// The image row the stripe's first row holds. + first: u32, + /// The rows the stripe holds. + held: usize, + deliver: &'s mut DeliverRow<'s, E>, + /// Whether any row was handed to `deliver`. + delivered: bool, + error: Option>, +} + +impl StripeSink<'_, E> { + /// Pack a rectangle of whole rows from row `y`, which must be the next + /// row, delivering every stripe it fills. + fn take_rows( + &mut self, + x: u32, + y: u32, + planes: &[j2k_native::ComponentPlane<'_>], + ) -> Result<(), DecodeRowsError> { + let dims = planes + .first() + .map_or((0, 0), j2k_native::ComponentPlane::dimensions); + let next = u32::try_from(self.held) + .ok() + .and_then(|held| self.first.checked_add(held)); + if x != 0 || dims.0 != self.width || Some(y) != next { + return Err(DecodeRowsError::Decode(J2kError::internal_backend( + "streamed rows arrived out of order", + ))); + } + let rows = dims.1 as usize; + let mut row = 0; + while row < rows { + let take = (self.stripe_rows - self.held).min(rows - row); + let out = &mut self.stripe[self.held * self.row_bytes..]; + (self.pack)( + self.color_space, + self.has_alpha, + planes, + dims, + row..row + take, + out, + self.row_bytes, + self.fmt, + ) + .map_err(DecodeRowsError::Decode)?; + self.held += take; + row += take; + if self.held == self.stripe_rows { + self.flush()?; + } + } + Ok(()) + } + + /// Hand the rows the stripe holds over, a row at a time. + fn flush(&mut self) -> Result<(), DecodeRowsError> { + let mut y = self.first; + for row in self.stripe.chunks_exact(self.row_bytes).take(self.held) { + self.delivered = true; + (self.deliver)(y, row).map_err(DecodeRowsError::Sink)?; + y += 1; + } + self.first = y; + self.held = 0; + Ok(()) + } +} + +impl j2k_native::ComponentRowSink for StripeSink<'_, E> { + fn write_rows(&mut self, x: u32, y: u32, planes: &[j2k_native::ComponentPlane<'_>]) { + if self.error.is_some() { + return; + } + if let Err(error) = self.take_rows(x, y, planes) { + self.error = Some(error); + } + } +} diff --git a/crates/j2k/src/view/rows.rs b/crates/j2k/src/view/rows.rs index 752f1038..cbd96593 100644 --- a/crates/j2k/src/view/rows.rs +++ b/crates/j2k/src/view/rows.rs @@ -4,7 +4,10 @@ use super::{J2kDecoder, J2kRowDecodeOptions}; use crate::{ - decode::{decode_prepared_image_region_into, J2kDecodeOutcome}, + decode::{ + decode_image_rows_in_stripes, decode_prepared_image_region_into, rows_stream_whole, + DeliverRow, J2kDecodeOutcome, StreamedRowsError, + }, scratch::J2kScratchPool, J2kError, }; @@ -67,9 +70,35 @@ impl J2kDecoder<'_> { /// Decode rows into a `u8` row sink while bounding host output scratch to /// at most `options.max_rows_per_stripe()` rows. /// + /// Whether the image decodes once is planned from its main and tile + /// headers before anything is decoded: an image whose tiles span its + /// width and whose region decode streams its rows as far as those headers + /// say decodes as one region decode of the whole image, which hands + /// finished rows over a stripe at a time; other images decode each stripe + /// as a region of its own. Coding the plan does not see, such as a + /// tile-level coding style without decomposition levels or HTJ2K + /// code-blocks mixed with classic ones, can keep that region decode from + /// streaming: it then decodes into component planes of the whole image + /// and hands their rows over a stripe at a time once they are complete. + /// + /// # Memory + /// `max_rows_per_stripe` and `max_stripe_bytes` bound the packed output + /// stripe. These limits do not bound decoder working memory or total + /// process memory. Decoder memory depends on the image and parallelism + /// and may include the working set of a whole-image decode, including + /// component planes when native streaming is unavailable. Smaller stripes + /// do not necessarily reduce decoder memory. + /// /// # Errors /// Returns a decode error for unsupported formats or malformed input, and /// forwards sink errors without converting them to successful decodes. + /// An error may leave a prefix of rows delivered. A sink error stops + /// further sink calls and is returned as [`DecodeRowsError::Sink`], even + /// if decoding subsequently fails; the underlying decode may continue + /// before this method returns. A decode failure is retried stripe by + /// stripe only if no row has been handed to the sink. After delivery + /// starts, decode failures return [`DecodeRowsError::Decode`] without + /// replaying rows. pub fn decode_rows_u8_bounded>( &mut self, sink: &mut R, @@ -87,6 +116,24 @@ impl J2kDecoder<'_> { let (stripe_rows, max_stripe_len) = bounded_row_stripe_layout(row_bytes, height, options) .map_err(DecodeRowsError::Decode)?; self.ensure_image().map_err(DecodeRowsError::Decode)?; + let mut pool = J2kScratchPool::new(); + if self.rows_stream_whole() { + let stripe = pool + .packed_bytes(max_stripe_len) + .map_err(|error| DecodeRowsError::Decode(J2kError::Buffer(error)))?; + match self.decode_rows_streamed(fmt, (stripe, row_bytes), &mut |y, row| { + sink.write_row(y, row) + }) { + Ok(()) => { + return Ok(j2k_core::DecodeOutcome::new( + Rect::full(self.info.dimensions), + Vec::new(), + )) + } + Err(StreamedRowsError::Rows(error)) => return Err(error), + Err(StreamedRowsError::BeforeFirstRow) => {} + } + } let (Some(image), native_context) = (self.image.as_ref(), &mut self.native_context) else { return Err(DecodeRowsError::Decode(J2kError::internal_backend( "internal image cache missing", @@ -95,7 +142,6 @@ impl J2kDecoder<'_> { let mut decoder = image .prepare_region_decoder_with_context(native_context) .map_err(|error| DecodeRowsError::Decode(J2kError::from_native_decode_error(error)))?; - let mut pool = J2kScratchPool::new(); let mut y = 0_u32; while y < height { let rows = stripe_rows.min(height - y); @@ -136,9 +182,29 @@ impl J2kDecoder<'_> { /// Decode rows into a `u16` row sink while bounding host output scratch to /// at most `options.max_rows_per_stripe()` rows. /// + /// Images decode once or a stripe at a time as planned for + /// [`Self::decode_rows_u8_bounded`]; images deeper than 24 bits decode a + /// stripe at a time. + /// + /// # Memory + /// `max_rows_per_stripe` and `max_stripe_bytes` bound the packed output + /// stripe, beside which this method uses one row of `u16` conversion + /// scratch. These limits do not bound decoder working memory or total + /// process memory. Decoder memory depends on the image and parallelism + /// and may include the working set of a whole-image decode, including + /// component planes when native streaming is unavailable. Smaller stripes + /// do not necessarily reduce decoder memory. + /// /// # Errors /// Returns a decode error for unsupported formats or malformed input, and /// forwards sink errors without converting them to successful decodes. + /// An error may leave a prefix of rows delivered. A sink error stops + /// further sink calls and is returned as [`DecodeRowsError::Sink`], even + /// if decoding subsequently fails; the underlying decode may continue + /// before this method returns. A decode failure is retried stripe by + /// stripe only if no row has been handed to the sink. After delivery + /// starts, decode failures return [`DecodeRowsError::Decode`] without + /// replaying rows. pub fn decode_rows_u16_bounded>( &mut self, sink: &mut R, @@ -180,6 +246,25 @@ impl J2kDecoder<'_> { return self.decode_rows_u16_exact_reparsed(sink, fmt, layout); } self.ensure_image().map_err(DecodeRowsError::Decode)?; + if self.rows_stream_whole() { + let mut pool = J2kScratchPool::new(); + let (packed, row) = pool + .packed_bytes_and_row_u16(layout.max_stripe_len, layout.samples_per_row) + .map_err(|error| DecodeRowsError::Decode(J2kError::Buffer(error)))?; + match self.decode_rows_streamed(fmt, (packed, row_bytes), &mut |y, packed_row| { + widen_row_u16(packed_row, row); + sink.write_row(y, row) + }) { + Ok(()) => { + return Ok(j2k_core::DecodeOutcome::new( + Rect::full((width, height)), + Vec::new(), + )) + } + Err(StreamedRowsError::Rows(error)) => return Err(error), + Err(StreamedRowsError::BeforeFirstRow) => {} + } + } let (Some(image), native_context) = (self.image.as_ref(), &mut self.native_context) else { return Err(DecodeRowsError::Decode(J2kError::internal_backend( "internal image cache missing", @@ -193,6 +278,34 @@ impl J2kDecoder<'_> { }) } + /// Whether this decoder's image decodes its rows in one streamed region + /// decode (see [`rows_stream_whole`]). + fn rows_stream_whole(&self) -> bool { + self.cached_image() + .is_ok_and(|image| rows_stream_whole(image, self.info.resolution_levels)) + } + + /// Decode the whole image in one streamed region decode, handing its + /// rows to `deliver` a stripe of `stripe` at a time (see + /// [`decode_image_rows_in_stripes`]). + fn decode_rows_streamed( + &mut self, + fmt: PixelFormat, + stripe: (&mut [u8], usize), + deliver: &mut DeliverRow<'_, E>, + ) -> Result<(), StreamedRowsError> { + let (Some(image), native_context) = (self.image.as_ref(), &mut self.native_context) else { + return Err(StreamedRowsError::Rows(DecodeRowsError::Decode( + J2kError::internal_backend("internal image cache missing"), + ))); + }; + let decoded = decode_image_rows_in_stripes(image, native_context, fmt, stripe, deliver); + // As after a region decode, release the decode buffers rather than + // hold them while the decoder is idle. + self.native_context = self.fresh_native_context(); + decoded + } + fn decode_rows_u16_exact_reparsed>( &mut self, sink: &mut R, @@ -248,10 +361,7 @@ fn pump_u16_rows>( .map_err(DecodeRowsError::Decode)?; for row_index in 0..rows { let start = row_index as usize * layout.row_bytes; - let packed_row = &packed[start..start + layout.row_bytes]; - for (dst, src) in row.iter_mut().zip(packed_row.as_chunks::<2>().0) { - *dst = u16::from_le_bytes(*src); - } + widen_row_u16(&packed[start..start + layout.row_bytes], row); sink.write_row(y + row_index, row) .map_err(DecodeRowsError::Sink)?; } @@ -263,6 +373,13 @@ fn pump_u16_rows>( )) } +/// The 16-bit samples of a packed little-endian row. +fn widen_row_u16(packed: &[u8], row: &mut [u16]) { + for (dst, src) in row.iter_mut().zip(packed.as_chunks::<2>().0) { + *dst = u16::from_le_bytes(*src); + } +} + fn row_format_u8(info: &j2k_core::Info) -> Result { match info.components { 1 => Ok(PixelFormat::Gray8), @@ -337,6 +454,191 @@ fn row_samples_for(info: &j2k_core::Info, fmt: PixelFormat) -> Result) -> core::fmt::Result { + write!(f, "row {} refused", self.0) + } + } + + impl core::error::Error for Refused {} + + /// Rows as little-endian bytes and every row index handed over, in + /// order, refusing row `refuse` when it is set. + #[derive(Default)] + struct Collect { + bytes: Vec, + order: Vec, + refuse: Option, + } + + impl Collect { + fn take(&mut self, y: u32) -> Result<(), Refused> { + self.order.push(y); + if self.refuse == Some(y) { + return Err(Refused(y)); + } + Ok(()) + } + } + + impl RowSink for Collect { + type Error = Refused; + + fn write_row(&mut self, y: u32, row: &[u8]) -> Result<(), Refused> { + self.take(y)?; + self.bytes.extend_from_slice(row); + Ok(()) + } + } + + impl RowSink for Collect { + type Error = Refused; + + fn write_row(&mut self, y: u32, row: &[u16]) -> Result<(), Refused> { + self.take(y)?; + self.bytes + .extend(row.iter().flat_map(|sample| sample.to_le_bytes())); + Ok(()) + } + } + + fn rgb_codestream(tile_size: Option<(u32, u32)>, num_decomposition_levels: u8) -> Vec { + let pixels: Vec = (0..64 * 48 * 3_u32) + .map(|index| u8::try_from(index * 7 % 251).expect("a byte")) + .collect(); + let options = EncodeOptions { + tile_size, + num_decomposition_levels, + ..EncodeOptions::default() + }; + j2k_native::encode(&pixels, 64, 48, 3, 8, false, &options).expect("encode") + } + + /// The `decode_into` output in `fmt`. + fn full_decode(bytes: &[u8], fmt: PixelFormat) -> Vec { + let mut decoder = J2kDecoder::new(bytes).expect("decoder"); + let stride = 64 * fmt.bytes_per_pixel(); + let mut full = vec![0; stride * 48]; + decoder + .decode_into(&mut full, stride, fmt) + .expect("full decode"); + full + } + + /// A bounded decode in 7-row stripes into `sink`, with `injected` set + /// for it, and the streamed decodes it started. + fn decode_rows( + bytes: &[u8], + fmt: PixelFormat, + injected: Option, + sink: &mut Collect, + ) -> (Result<(), DecodeRowsError>, usize) { + let streamed = || STREAMED_ROW_DECODES.with(core::cell::Cell::get); + let mut decoder = J2kDecoder::new(bytes).expect("decoder"); + let before = streamed(); + INJECTED_FAILURE.with(|failure| failure.set(injected)); + let options = J2kRowDecodeOptions::new(7); + let result = if fmt == PixelFormat::Rgb16 { + decoder.decode_rows_u16_bounded(sink, options) + } else { + decoder.decode_rows_u8_bounded(sink, options) + }; + assert_eq!(INJECTED_FAILURE.with(core::cell::Cell::get), None); + (result.map(drop), streamed() - before) + } + + #[test] + fn rows_decode_in_one_streamed_decode_only_when_they_arrive_whole_and_in_order() { + for (label, bytes, streams) in [ + ("untiled", rgb_codestream(None, 3), true), + ( + "tiles spanning the width", + rgb_codestream(Some((64, 16)), 3), + true, + ), + ( + "tiles side by side", + rgb_codestream(Some((32, 16)), 3), + false, + ), + ("no decomposition levels", rgb_codestream(None, 0), false), + ] { + for fmt in [PixelFormat::Rgb8, PixelFormat::Rgb16] { + let mut sink = Collect::default(); + let (result, streamed) = decode_rows(&bytes, fmt, None, &mut sink); + result.expect("rows"); + assert_eq!(streamed, usize::from(streams), "{label} {fmt:?}"); + assert_eq!(sink.order, (0..48).collect::>(), "{label} {fmt:?}"); + assert!( + sink.bytes == full_decode(&bytes, fmt), + "{label} {fmt:?}: rows differ from decode_into" + ); + } + } + } + + #[test] + fn a_streamed_decode_failing_before_its_first_row_decodes_a_stripe_at_a_time() { + let bytes = rgb_codestream(None, 3); + for fmt in [PixelFormat::Rgb8, PixelFormat::Rgb16] { + let full = full_decode(&bytes, fmt); + let mut sink = Collect::default(); + let (result, streamed) = + decode_rows(&bytes, fmt, Some(InjectedFailure::BeforeRows), &mut sink); + result.expect("rows a stripe at a time"); + assert_eq!(streamed, 1, "{fmt:?}"); + assert_eq!(sink.order, (0..48).collect::>(), "{fmt:?}"); + assert!(sink.bytes == full, "{fmt:?}: rows differ from decode_into"); + // The stripe-at-a-time decode meets the caller's sink as it is. + let mut sink = Collect { + refuse: Some(10), + ..Collect::default() + }; + let (result, _) = + decode_rows(&bytes, fmt, Some(InjectedFailure::BeforeRows), &mut sink); + assert!( + matches!(result, Err(DecodeRowsError::Sink(Refused(10)))), + "{fmt:?}" + ); + assert_eq!(sink.order, (0..=10).collect::>(), "{fmt:?}"); + } + } + + #[test] + fn a_streamed_decode_failing_after_rows_reached_the_sink_reports_it_without_a_rerun() { + let bytes = rgb_codestream(None, 3); + for fmt in [PixelFormat::Rgb8, PixelFormat::Rgb16] { + let full = full_decode(&bytes, fmt); + let row_bytes = 64 * fmt.bytes_per_pixel(); + let mut sink = Collect::default(); + let (result, streamed) = + decode_rows(&bytes, fmt, Some(InjectedFailure::AfterRows), &mut sink); + assert!(matches!(result, Err(DecodeRowsError::Decode(_))), "{fmt:?}"); + assert_eq!(streamed, 1, "{fmt:?}"); + // Six whole 7-row stripes; the last one, which the failure + // ended, is held back, and nothing is handed over twice. + assert_eq!(sink.order, (0..42).collect::>(), "{fmt:?}"); + assert!(sink.bytes == full[..42 * row_bytes], "{fmt:?}"); + // A sink error comes first. + let mut sink = Collect { + refuse: Some(10), + ..Collect::default() + }; + let (result, _) = decode_rows(&bytes, fmt, Some(InjectedFailure::AfterRows), &mut sink); + assert!( + matches!(result, Err(DecodeRowsError::Sink(Refused(10)))), + "{fmt:?}" + ); + assert_eq!(sink.order, (0..=10).collect::>(), "{fmt:?}"); + } + } #[test] fn bounded_row_stripe_layout_clamps_rows_to_byte_cap() { diff --git a/crates/j2k/tests/bounded_rows.rs b/crates/j2k/tests/bounded_rows.rs new file mode 100644 index 00000000..34716d11 --- /dev/null +++ b/crates/j2k/tests/bounded_rows.rs @@ -0,0 +1,409 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! A bounded row decode of an image whose tiles span its width runs as one +//! region decode of the whole image that streams its rows and hands them +//! over a stripe at a time; other images decode each stripe as a region of +//! its own. Either way the rows must be the bytes `decode_into` writes, in +//! order and once each, for every stripe height, pool and sample depth, and +//! a sink that fails must stop the rows at the row it refused and be +//! reported as the sink's error. + +use core::fmt; + +use j2k::{CpuDecodeParallelism, DecodeRowsError, J2kDecoder, J2kRowDecodeOptions, RowSink}; +use j2k_core::PixelFormat; +use j2k_native::{encode, EncodeOptions}; +use j2k_test_support::wrap_jp2_rgba_codestream; +use rayon::ThreadPoolBuilder; + +const WIDTH: u32 = 203; +const HEIGHT: u32 = 157; + +fn noisy_samples(components: u32, bit_depth: u8) -> Vec { + let mut state = 0x2545_f491_u32; + let mut next = move || { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state + }; + let samples = WIDTH * HEIGHT * components; + if bit_depth <= 8 { + return (0..samples).map(|_| (next() >> 24) as u8).collect(); + } + let max = (1_u32 << bit_depth) - 1; + (0..samples) + .flat_map(|_| { + u16::try_from(next() % (max + 1)) + .expect("sample") + .to_le_bytes() + }) + .collect() +} + +fn codestream(components: u16, bit_depth: u8, options: &EncodeOptions) -> Vec { + encoded(components, bit_depth, false, options) +} + +fn encoded(components: u16, bit_depth: u8, signed: bool, options: &EncodeOptions) -> Vec { + let samples = noisy_samples(u32::from(components), bit_depth); + encode( + &samples, WIDTH, HEIGHT, components, bit_depth, signed, options, + ) + .expect("encode fixture") +} + +/// A sink error carrying the row the sink refused. +#[derive(Debug, PartialEq, Eq)] +struct Refused(u32); + +impl fmt::Display for Refused { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "row {} refused", self.0) + } +} + +impl std::error::Error for Refused {} + +/// The rows a sink took, as little-endian bytes, and every row index it +/// was given, refusing the row `refuse` when it is set. +#[derive(Default)] +struct Rows { + bytes: Vec, + order: Vec, + refuse: Option, +} + +impl Rows { + fn take(&mut self, y: u32) -> Result<(), Refused> { + self.order.push(y); + if self.refuse == Some(y) { + return Err(Refused(y)); + } + Ok(()) + } +} + +impl RowSink for Rows { + type Error = Refused; + + fn write_row(&mut self, y: u32, row: &[u8]) -> Result<(), Refused> { + self.take(y)?; + self.bytes.extend_from_slice(row); + Ok(()) + } +} + +impl RowSink for Rows { + type Error = Refused; + + fn write_row(&mut self, y: u32, row: &[u16]) -> Result<(), Refused> { + self.take(y)?; + self.bytes + .extend(row.iter().flat_map(|sample| sample.to_le_bytes())); + Ok(()) + } +} + +fn fmt_for(components: u16, wide: bool) -> PixelFormat { + match (components, wide) { + (1, false) => PixelFormat::Gray8, + (1, true) => PixelFormat::Gray16, + (4, false) => PixelFormat::Rgba8, + (4, true) => PixelFormat::Rgba16, + (_, false) => PixelFormat::Rgb8, + (_, true) => PixelFormat::Rgb16, + } +} + +fn decode_into(bytes: &[u8], parallelism: CpuDecodeParallelism, fmt: PixelFormat) -> Vec { + let mut decoder = J2kDecoder::new(bytes).expect("decoder"); + decoder.set_cpu_decode_parallelism(parallelism); + let stride = WIDTH as usize * fmt.bytes_per_pixel(); + let mut out = vec![0_u8; stride * HEIGHT as usize]; + decoder + .decode_into(&mut out, stride, fmt) + .expect("full decode"); + out +} + +fn decode_rows( + bytes: &[u8], + parallelism: CpuDecodeParallelism, + wide: bool, + stripe_rows: u32, + sink: &mut Rows, +) -> Result<(), DecodeRowsError> { + let mut decoder = J2kDecoder::new(bytes).expect("decoder"); + decoder.set_cpu_decode_parallelism(parallelism); + decode_rows_with(&mut decoder, wide, stripe_rows, sink) +} + +fn decode_rows_with( + decoder: &mut J2kDecoder<'_>, + wide: bool, + stripe_rows: u32, + sink: &mut Rows, +) -> Result<(), DecodeRowsError> { + let options = J2kRowDecodeOptions::new(stripe_rows); + let outcome = if wide { + decoder.decode_rows_u16_bounded(sink, options) + } else { + decoder.decode_rows_u8_bounded(sink, options) + }?; + assert_eq!(outcome.decoded, j2k_core::Rect::full((WIDTH, HEIGHT))); + Ok(()) +} + +struct Fixture { + name: &'static str, + bytes: Vec, + components: u16, + /// Whether 16-bit rows are compared too. + wide: bool, + /// Whether the rows decode in one streamed region decode, rather than + /// a stripe at a time. + streams: bool, +} + +fn fixtures() -> Vec { + let base = EncodeOptions { + num_decomposition_levels: 4, + ..EncodeOptions::default() + }; + let lossy = EncodeOptions { + reversible: false, + ..base.clone() + }; + let rgba = codestream(4, 8, &base); + vec![ + Fixture { + name: "rgb 9/7", + bytes: codestream(3, 8, &lossy), + components: 3, + wide: true, + streams: true, + }, + Fixture { + name: "rgb 5/3 HT", + bytes: codestream( + 3, + 8, + &EncodeOptions { + use_ht_block_coding: true, + ..base.clone() + }, + ), + components: 3, + wide: true, + streams: true, + }, + Fixture { + name: "gray 12-bit 9/7", + bytes: codestream(1, 12, &lossy), + components: 1, + wide: true, + streams: true, + }, + Fixture { + name: "gray signed 12-bit 5/3", + bytes: encoded(1, 12, true, &base), + components: 1, + wide: true, + streams: true, + }, + // Tiles as wide as the image: one region decode, a tile after another. + Fixture { + name: "rgb tiles spanning the width", + bytes: codestream( + 3, + 8, + &EncodeOptions { + tile_size: Some((WIDTH, 48)), + ..lossy.clone() + }, + ), + components: 3, + wide: true, + streams: true, + }, + // Rows of one tile arrive before the next tile's: a stripe at a time. + Fixture { + name: "rgb tiles side by side", + bytes: codestream( + 3, + 8, + &EncodeOptions { + tile_size: Some((64, 64)), + ..lossy.clone() + }, + ), + components: 3, + wide: true, + streams: false, + }, + // A channel definition keeps region decodes on component planes. + Fixture { + name: "rgba with channel definition", + bytes: wrap_jp2_rgba_codestream(&rgba, WIDTH, HEIGHT, 8), + components: 4, + wide: true, + streams: false, + }, + // No wavelet levels to stream through. + Fixture { + name: "rgb without decomposition levels", + bytes: codestream( + 3, + 8, + &EncodeOptions { + num_decomposition_levels: 0, + ..base.clone() + }, + ), + components: 3, + wide: false, + streams: false, + }, + ] +} + +fn parallelisms() -> [(&'static str, CpuDecodeParallelism, usize); 4] { + [ + ("serial", CpuDecodeParallelism::Serial, 1), + ("auto on 1 thread", CpuDecodeParallelism::Auto, 1), + ("auto on 4 threads", CpuDecodeParallelism::Auto, 4), + ("auto on 11 threads", CpuDecodeParallelism::Auto, 11), + ] +} + +#[test] +fn bounded_rows_match_full_decode_at_every_stripe_height_pool_and_depth() { + let fixtures = fixtures(); + for (pool_name, parallelism, threads) in parallelisms() { + let pool = ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("pool"); + pool.install(|| { + for fixture in &fixtures { + for wide in [false, true].into_iter().filter(|&wide| !wide || fixture.wide) { + let fmt = fmt_for(fixture.components, wide); + let expected = decode_into(&fixture.bytes, parallelism, fmt); + // One-row stripes of a stripe-at-a-time decode take a + // region decode per row: serially only. + let one_row = fixture.streams || parallelism == CpuDecodeParallelism::Serial; + let heights = [1, 7, 64, HEIGHT, 4 * HEIGHT]; + for stripe_rows in heights.into_iter().filter(|&rows| rows > 1 || one_row) { + let mut sink = Rows::default(); + decode_rows(&fixture.bytes, parallelism, wide, stripe_rows, &mut sink) + .unwrap_or_else(|error| { + panic!( + "{} {fmt:?} {stripe_rows} rows, {pool_name}: {error}", + fixture.name + ) + }); + assert_eq!( + sink.order, + (0..HEIGHT).collect::>(), + "{} {fmt:?} {stripe_rows} rows, {pool_name}: rows in order, once", + fixture.name + ); + assert!( + sink.bytes == expected, + "{} {fmt:?} {stripe_rows} rows, {pool_name}: rows differ from decode_into", + fixture.name + ); + } + } + } + }); + } +} + +#[test] +fn a_refused_row_stops_the_rows_and_is_reported_as_the_sinks_error() { + let fixtures = fixtures(); + for (pool_name, parallelism, threads) in parallelisms() { + let pool = ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("pool"); + pool.install(|| { + for fixture in &fixtures { + for wide in [false, true].into_iter().filter(|&wide| !wide || fixture.wide) { + let fmt = fmt_for(fixture.components, wide); + let expected = decode_into(&fixture.bytes, parallelism, fmt); + let row_bytes = WIDTH as usize * fmt.bytes_per_pixel(); + for (stripe_rows, refuse) in [(1, 0), (7, 50), (64, 100), (HEIGHT, HEIGHT - 1)] + { + let mut decoder = J2kDecoder::new(&fixture.bytes).expect("decoder"); + decoder.set_cpu_decode_parallelism(parallelism); + let mut sink = Rows { + refuse: Some(refuse), + ..Rows::default() + }; + let result = decode_rows_with(&mut decoder, wide, stripe_rows, &mut sink); + let context = + format!("{} {fmt:?} {stripe_rows} rows, {pool_name}", fixture.name); + assert!( + matches!(result, Err(DecodeRowsError::Sink(Refused(row))) if row == refuse), + "{context}: {result:?}" + ); + // The rows before the refused one, then the refused + // one, and nothing after it. + assert_eq!( + sink.order, + (0..=refuse).collect::>(), + "{context}: rows handed over" + ); + assert!( + sink.bytes == expected[..refuse as usize * row_bytes], + "{context}: rows before the refused one" + ); + // The decoder decodes every row again afterwards. + if stripe_rows == 7 { + let mut again = Rows::default(); + decode_rows_with(&mut decoder, wide, stripe_rows, &mut again) + .unwrap_or_else(|error| panic!("{context}: again: {error}")); + assert!(again.bytes == expected, "{context}: rows after a refusal"); + } + } + } + } + }); + } +} + +#[test] +fn a_codestream_that_fails_before_its_rows_hands_over_none_and_reports_the_decode_error() { + let mut bytes = codestream( + 3, + 8, + &EncodeOptions { + num_decomposition_levels: 4, + reversible: false, + ..EncodeOptions::default() + }, + ); + // Without its end-of-codestream marker the tile data is rejected. + assert_eq!(bytes.pop(), Some(0xd9)); + assert_eq!(bytes.pop(), Some(0xff)); + for (pool_name, parallelism, threads) in parallelisms() { + let pool = ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("pool"); + pool.install(|| { + for wide in [false, true] { + let mut sink = Rows::default(); + let result = decode_rows(&bytes, parallelism, wide, 7, &mut sink); + assert!( + matches!(result, Err(DecodeRowsError::Decode(_))), + "{pool_name} wide={wide}: {result:?}" + ); + assert!(sink.order.is_empty(), "{pool_name} wide={wide}: no rows"); + } + }); + } +} From 9af98483dfad9b3003f784e0f9e6bcb41559acb8 Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 03:58:57 -0500 Subject: [PATCH 80/94] docs(changelog): note the one-pass bounded row decodes --- CHANGELOG.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08a76095..9ae54912 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,34 @@ Staged workspace version: `0.13.2`. status, detail and batch source index. CUDA keeps its typed source and job attribution. Unsupported statuses, raw runtime failures and combined cleanup or release failures keep their existing classes. +- `decode_rows_u8_bounded` and `decode_rows_u16_bounded` decode an image + whose tiles span its width, and whose headers let its region decode stream + its rows, once, as a region decode of the whole image that streams its + rows into a buffer of at most the requested stripe rows and hands them + over a stripe at a time. They decoded every stripe as a region decode of + its own, which decoded each code-block again for every stripe its wavelet + support reaches: at the default 64-row stripes 6.0 to 7.4 times the + code-block samples of one full decode on 2048- to 8287-row images. On an + 18-core ARM64 machine the default stripes of a 5661 x 8287 RGB image take + 1.34 s instead of 12.9 s on one thread and 0.11 s instead of 1.52 s on 18. + The decode's peak heap is then that of the streamed full-image decode + (22.5 MiB instead of 10.3 on one thread, 69 instead of 12.8 on 18); the + process's peak resident memory beside its input and output falls on such + classic images (25.6 MiB instead of 31.8, 72.8 instead of 77.4) and rises + on HTJ2K images on a pool (a 4096 x 2048 image on 18 threads: 76.9 MiB + instead of 42.6). Rows, bytes, the stripe bound and errors are unchanged: + a decode that fails before its first row reaches the sink decodes a stripe + at a time instead, and a sink that refuses a row still stops the rows + there, though a one-pass decode now goes on until it completes or fails. + Whether an image decodes once is planned from its main and tile headers: + images whose tiles sit side by side, whose headers keep region decodes on + component planes, or whose main header codes no decomposition level, and + 16-bit rows of images deeper than 24 bits, still decode a stripe at a + time, and an image whose tiles turn out not to stream (a tile-level coding + style without decomposition levels, or HTJ2K code-blocks mixed with + classic ones) decodes into component planes of the whole image and hands + their rows over a stripe at a time. The stripe bound covers the packed + output stripe, not the decoder's working memory. - Classic code-blocks of several codeword segments, with selective bypass or per-pass termination (as archival images and multi-layer encodes of this crate often have), decode on the worker pool's tile route and pool From b45b8a289c57af77c699996a385ae054141a606a Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 01:24:27 -0500 Subject: [PATCH 81/94] perf(j2k-native): hold two finest bands for classic tiles read per component without deep rings A classic tile of several components that streams its rows from pool bands with a reader per component and full-width reads holds three bands in each finest-level ring and two in the others, plus one at every level with the deep rings of 12 threads and more at full resolution. Without the deep rings, as on 10 and 11 threads or at a reduced resolution, a three-component tile leaves 18 finest-level bands ahead of its readers. That is at least the pool's threads, so plan_parts splits no band, and a thread that finds no whole band to start waits. Such tiles now hold two bands in each finest-level ring when the pool has more threads than the tile has finest-level sub-bands. The nine bands ahead are then fewer than the threads, so they split into parts that every thread takes. Deep rings, one-reader tiles, HTJ2K tiles, single-component tiles and narrower regions keep their rings: on 6 and 8 threads two finest-level bands drain the pool between band rows (25-40% slower), and a single-component tile on 18 threads leaves too few bands to split (27% slower). Peak heap (counting allocator), output identical: - 5661 x 8287 RGB on 10 threads: 55.3 -> 49.2 MiB; at half resolution on 12 threads 32.2 -> 29.1 MiB, at a quarter 17.8 -> 16.3 MiB. - 18000 x 12000 RGB on 10 threads: 168.0 -> 148.4 MiB; at half resolution on 10 and 12 threads 89.1 -> 79.2 MiB. - 6061 x 4252 RGB on 10 threads: 51.7 -> 45.1 MiB. - 16384 x 16384 RGB on 10 threads: 152.3 -> 134.4 MiB. Other decodes hold the same bytes as before. Decode times, 7 interleaved rounds against two builds of the base: on a 10-core machine at 10 threads and the default pool, the full-resolution RGB images take 0.5-4% less time (5661 x 8287: 1.5-3%; 16384 x 16384: 4%; 18000 x 12000: 0.5%) and the 18000 x 12000 one at half resolution 6.5% less; on an 18-core machine at 10 threads the full-resolution ones take 5-7% less. Decodes the rule leaves alone run the same code. The row-sink test decodes three-component tiles on 9 to 12 threads and four-component ones on 12 and 13, at full and half resolution, and checks the deepest ring each level holds (a test-only counter of the planned rings) and that the bands split into parts exactly where the finest rings are shallow. With three finest-level bands it fails on 10 threads, and without the thread condition on four components at 12 threads. The band executor's counting-allocator test gains a wider tile, which takes the shallow rings on 10 and 11 threads and splits its bands into parts by itself, and checks the executor allocates within its charge there too; leaving the slot records of such a tile out of the charge fails it. --- crates/j2k-native/src/j2c/build/allocation.rs | 10 +- crates/j2k-native/src/j2c/decode.rs | 7 + .../src/j2c/decode/parallel_bands.rs | 177 +++++++++++++++--- crates/j2k-native/src/j2c/decode/state.rs | 6 + crates/j2k-native/src/tests/row_sink.rs | 97 ++++++++++ 5 files changed, 270 insertions(+), 27 deletions(-) diff --git a/crates/j2k-native/src/j2c/build/allocation.rs b/crates/j2k-native/src/j2c/build/allocation.rs index 06bcb057..0a0ab7fc 100644 --- a/crates/j2k-native/src/j2c/build/allocation.rs +++ b/crates/j2k-native/src/j2c/build/allocation.rs @@ -398,6 +398,11 @@ pub(crate) const OTHER_LEVEL_BAND_SLOTS: u32 = 2; /// reconstructed rows. const DEEP_RING_EXTRA_SLOTS: u32 = 1; +/// Band slots each sub-band of the finest active level holds with +/// `shallow_band_rings`: one fewer, as the pool's threads split their bands +/// into parts instead of waiting for whole bands further ahead. +const SHALLOW_FINEST_LEVEL_BAND_SLOTS: u32 = 2; + /// Give each sub-band of a region decode the coefficient range for its /// stored rectangle: the required code-blocks, clipped to the IDWT windows /// unless `clip_to_idwt_windows` is false. With banded Tier-1, a range holds @@ -459,7 +464,9 @@ fn lay_out_region_coefficients( .sub_bands .contains(&sub_band_idx) }); - let slots = if finest { + let slots = if finest && storage.shallow_band_rings { + SHALLOW_FINEST_LEVEL_BAND_SLOTS + } else if finest { FINEST_LEVEL_BAND_SLOTS } else { OTHER_LEVEL_BAND_SLOTS @@ -516,6 +523,7 @@ pub(crate) fn lay_out_whole_region_coefficients( storage.banded_tier1 = false; storage.parallel_bands = false; storage.deep_band_rings = false; + storage.shallow_band_rings = false; let coefficient_count = lay_out_region_coefficients(storage, true, 0)?; let mut budget = reuse::ReallocationBudget::from_live_bytes(storage.structural_workspace_bytes)?; diff --git a/crates/j2k-native/src/j2c/decode.rs b/crates/j2k-native/src/j2c/decode.rs index 2b4d4a70..76c9fadb 100644 --- a/crates/j2k-native/src/j2c/decode.rs +++ b/crates/j2k-native/src/j2c/decode.rs @@ -767,6 +767,12 @@ pub(crate) struct DecodeDebugCounters { /// reconstructed them on. #[cfg(all(test, feature = "parallel"))] pub(crate) pool_band_readers: usize, + /// The most band slots a ring of pool bands held that did not hold its + /// whole window, at the finest active level and at the others. + #[cfg(all(test, feature = "parallel"))] + pub(crate) pool_band_finest_slots: usize, + #[cfg(all(test, feature = "parallel"))] + pub(crate) pool_band_other_slots: usize, /// The most line-buffer samples a tile streaming its rows from pool /// bands held beyond the IDWT samples its ROI plan charged. #[cfg(all(test, feature = "parallel"))] @@ -990,6 +996,7 @@ fn prepare_region_storage( storage.one_synthesis_reader = storage.parallel_bands && storage.lockstep_lines && parallel_bands::one_reader(tile, storage); + storage.shallow_band_rings = parallel_bands::shallow_rings(tile, storage); } build::allocate_deferred_coefficients( storage, diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index 64756b24..772f7816 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -247,6 +247,8 @@ fn decode_route( return Ok(None); } let (rings, mut bands, active_levels) = plan_rings(route, storage, header, component_idx)?; + #[cfg(test)] + count_ring_slots(tile_ctx, &rings); if route.coding() == CodeBlockCoding::Classic { split_bands( tile_ctx, @@ -454,6 +456,8 @@ pub(super) fn decode_tile_rows<'a>( 1 }; let workers = route_workers.min(bank.saturating_sub(readers)); + #[cfg(test)] + count_ring_slots(tile_ctx, &rings); if coding == CodeBlockCoding::Classic { split_bands( tile_ctx, @@ -677,21 +681,54 @@ pub(super) fn deep_rings(tile: &Tile<'_>) -> bool { } /// Whether a tile reads whole rows at full resolution: no resolution level -/// skipped, and every sub-band window the IDWT reads spans its sub-band. Only -/// such tiles hold deep band rings: a reduced or narrower read decodes as -/// fast without the extra band in every ring. +/// skipped, and full-width windows (see [`reads_full_width`]). Only such +/// tiles hold deep band rings: a reduced or narrower read decodes as fast +/// without the extra band in every ring. pub(super) fn reads_whole_rows(header: &Header<'_>, storage: &DecompositionStorage<'_>) -> bool { - header.skipped_resolution_levels == 0 - && storage.roi_plan.as_ref().is_some_and(|plan| { - storage - .sub_bands - .iter() - .enumerate() - .all(|(index, sub_band)| { - plan.sub_band_window(index) - .is_none_or(|window| window.width() == sub_band.rect.width()) - }) + header.skipped_resolution_levels == 0 && reads_full_width(storage) +} + +/// Whether every sub-band window a tile's IDWT reads spans its sub-band. +fn reads_full_width(storage: &DecompositionStorage<'_>) -> bool { + storage.roi_plan.as_ref().is_some_and(|plan| { + storage + .sub_bands + .iter() + .enumerate() + .all(|(index, sub_band)| { + plan.sub_band_window(index) + .is_none_or(|window| window.width() == sub_band.rect.width()) + }) + }) +} + +/// Whether a tile streaming its rows from pool bands holds one band fewer in +/// each ring of its finest level (see +/// `DecompositionStorage::shallow_band_rings`): a classic tile of several +/// components that reconstructs each on its own reader without deep band +/// rings, as on 10 and 11 threads or at a reduced resolution, reading +/// full-width windows, on a pool with more threads than the tile has +/// finest-level sub-bands. Its readers wait on classic Tier-1. With two slots +/// the bands ahead of them are then fewer than the pool's threads, so the +/// bands split into parts (see [`plan_parts`]) that every thread takes, which +/// keeps the pool as busy as a band more in every ring did. Narrower regions +/// keep their rings. +pub(super) fn shallow_rings(tile: &Tile<'_>, storage: &DecompositionStorage<'_>) -> bool { + let components = tile.component_infos.len(); + storage.parallel_bands + && storage.lockstep_lines + && !storage.deep_band_rings + && !storage.one_synthesis_reader + && components > 1 + && 3 * components < rayon::current_num_threads() + && tile.component_infos.iter().all(|component| { + !component + .coding_style + .parameters + .code_block_style + .uses_high_throughput_block_coding() }) + && reads_full_width(storage) } /// Whether a tile streaming its rows from pool bands reconstructs them @@ -1184,6 +1221,21 @@ fn plan_rings( Ok((rings, bands, active_levels)) } +/// Count the deepest of `rings` that hold no whole window, at the finest +/// level and at the others, in the decode's debug counters. +#[cfg(test)] +fn count_ring_slots(tile_ctx: &mut TileDecodeContext, rings: &[Ring]) { + let counters = &mut tile_ctx.debug_counters; + for ring in rings.iter().filter(|ring| !ring.whole_window) { + let deepest = if ring.finest { + &mut counters.pool_band_finest_slots + } else { + &mut counters.pool_band_other_slots + }; + *deepest = (*deepest).max(ring.slots); + } +} + /// Most parts a band splits into. const MAX_BAND_PARTS: usize = 8; @@ -2735,6 +2787,10 @@ mod tests { parted: usize, bands: usize, deep: bool, + shallow: bool, + /// The deepest finest-level and coarser-level rings that hold no + /// whole window. + slots: (usize, usize), } /// Read every row of the rings that `ring_readers` gives to `readers`, @@ -2972,9 +3028,25 @@ mod tests { parted: bands.iter().filter(|band| band.parts > 1).count(), bands: bands.len(), deep: storage.deep_band_rings, + shallow: storage.shallow_band_rings, + slots: deepest_rings(&rings), } } + /// The most slots of a finest-level ring, and of a coarser-level one, + /// that holds no whole window. + fn deepest_rings(rings: &[Ring]) -> (usize, usize) { + let deepest = |finest: bool| { + rings + .iter() + .filter(|ring| ring.finest == finest && !ring.whole_window) + .map(|ring| ring.slots) + .max() + .unwrap_or(0) + }; + (deepest(true), deepest(false)) + } + /// Whether this process is the one that measures the test `name`: the /// allocator's high-water mark is the process's, so when it is not, /// this runs the test alone in a process of its own and checks that it @@ -3109,11 +3181,27 @@ mod tests { } } + /// Pools of `threads` threads each, every thread of which has run a + /// job, so none allocates its first job's state inside a measurement. + fn warmed_pools(threads: &[usize]) -> Vec { + threads + .iter() + .map(|&threads| { + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("build pool"); + pool.broadcast(|_| ()); + pool + }) + .collect() + } + /// Under the counting allocator, the band executor holds at most what /// the decode charges for it, planning its routes, rings and bands and /// while they decode: for classic and HTJ2K tiles, whole and in a - /// region, with bands in parts, on pools of 2, 4 and 12 threads and - /// from outside a pool, with one reader and with one per component, + /// region, with bands in parts, on pools of 2, 4, 10, 11 and 12 threads + /// and from outside a pool, with one reader and with one per component, /// with shallow rings and deep ones. #[test] fn band_executor_allocates_within_its_charge() { @@ -3128,7 +3216,9 @@ mod tests { .expect("the measuring process's first use of the global pool"); let classic = include_bytes!("../../../fixtures/multi_segment/bypass_termall_rgb_53.j2k"); let ht = ht_fixture(); + let wide = shallow_fixture(); let (mut readers, mut parted, mut clipped, mut deep) = (Vec::new(), 0, false, false); + let mut shallow = Vec::new(); let mut check = |case: &str, measured: Measured, whole: &Measured| { assert!( measured.routing.0 <= measured.routing_charge, @@ -3144,6 +3234,10 @@ mod tests { measured.running <= measured.running_charge, "{case}: {measured:?}" ); + if measured.shallow { + assert_eq!(measured.slots.0, 2, "{case}: shallow finest rings"); + shallow.push((case.to_owned(), measured.slots, measured.parted)); + } readers.push(measured.readers); parted += measured.parted; clipped |= measured.bands < whole.bands; @@ -3153,17 +3247,7 @@ mod tests { // ends, so no pool's teardown frees bytes inside another's window. // Every thread of each runs a job before any measurement, and every // measurement waits until they idle (see `settle`). - let pools: Vec<_> = [2, 4, 12] - .into_iter() - .map(|threads| { - let pool = rayon::ThreadPoolBuilder::new() - .num_threads(threads) - .build() - .expect("build pool"); - pool.broadcast(|_| ()); - pool - }) - .collect(); + let pools = warmed_pools(&[2, 4, 10, 11, 12]); rayon::broadcast(|_| ()); for pool in &pools { let threads = pool.current_num_threads(); @@ -3171,6 +3255,7 @@ mod tests { for (label, bytes, splits) in [ ("classic", &classic[..], &[None, Some(4)][..]), ("HTJ2K", &ht[..], &[None][..]), + ("wide classic", &wide[..], &[None, Some(4)][..]), ] { let whole = measure_executor(bytes, None, None); for &roi in &[None, Some((48, 48, 16, 16))] { @@ -3196,6 +3281,46 @@ mod tests { parted > 0 && clipped && deep, "parts {parted}, region {clipped}, deep {deep}" ); + // The wide tile at full resolution on 10 and 11 threads holds the + // shallow finest rings, and its bands split into parts by themselves. + assert!( + shallow + .iter() + .any(|(case, slots, parted)| case.contains("parts None") + && slots.1 == 2 + && *parted > 0), + "{shallow:?}" + ); + } + + /// A classic RGB tile of noise wide and tall enough for the shallow + /// finest rings: a reader per component on 10 threads (the unit tests' + /// sample gate), finest-level bands of several blocks, and coarser + /// levels with more bands than slots. + fn shallow_fixture() -> Vec { + let mut state = 0x9e37_79b9_u32; + let pixels: Vec = (0..384 * 192 * 3) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state.to_le_bytes()[3] + }) + .collect(); + encode( + &pixels, + 384, + 192, + 3, + 8, + false, + &EncodeOptions { + num_decomposition_levels: 4, + code_block_height_exp: 2, + ..EncodeOptions::default() + }, + ) + .expect("encode shallow ring fixture") } /// A three-component 64 x 64 HTJ2K image in 16 x 16 code-blocks. diff --git a/crates/j2k-native/src/j2c/decode/state.rs b/crates/j2k-native/src/j2c/decode/state.rs index e186c29c..5f16d540 100644 --- a/crates/j2k-native/src/j2c/decode/state.rs +++ b/crates/j2k-native/src/j2c/decode/state.rs @@ -126,6 +126,11 @@ pub(crate) struct DecompositionStorage<'a> { /// resolution from a reader per component (see /// [`parallel_bands::deep_rings`] and [`parallel_bands::reads_whole_rows`]). pub(crate) deep_band_rings: bool, + /// Whether the finest level's band rings of `parallel_bands` hold one + /// band fewer, for a classic tile that streams full-width rows from a + /// reader per component without deep rings (see + /// [`parallel_bands::shallow_rings`]). + pub(crate) shallow_band_rings: bool, /// Whether a tile streaming its rows from `parallel_bands` reconstructs /// them through one reader whatever its pool's workers (see /// [`parallel_bands::one_reader`]), which holds one short batch of rows. @@ -172,6 +177,7 @@ impl DecompositionStorage<'_> { self.banded_tier1 = false; self.parallel_bands = false; self.deep_band_rings = false; + self.shallow_band_rings = false; self.one_synthesis_reader = false; self.lockstep_lines = false; self.roi_idwt_charge_elements = 0; diff --git a/crates/j2k-native/src/tests/row_sink.rs b/crates/j2k-native/src/tests/row_sink.rs index 7dbf6c2b..1e8a8ce9 100644 --- a/crates/j2k-native/src/tests/row_sink.rs +++ b/crates/j2k-native/src/tests/row_sink.rs @@ -94,6 +94,10 @@ struct Decoded { expect(dead_code, reason = "only the pool band tests compare the counters") )] parted_bands: usize, + /// The deepest finest-level and coarser-level rings of the pool bands + /// that held no whole window. + #[cfg(feature = "parallel")] + ring_slots: (usize, usize), } fn decode_planes(image: &Image<'_>, roi: (u32, u32, u32, u32)) -> Decoded { @@ -137,6 +141,11 @@ fn decode_planes(image: &Image<'_>, roi: (u32, u32, u32, u32)) -> Decoded { decoded_blocks: counters.decoded_code_blocks, skipped_blocks: counters.skipped_code_blocks, parted_bands: counters.pool_band_parts, + #[cfg(feature = "parallel")] + ring_slots: ( + counters.pool_band_finest_slots, + counters.pool_band_other_slots, + ), } } @@ -184,6 +193,11 @@ fn decode_rows_with( decoded_blocks: counters.decoded_code_blocks, skipped_blocks: counters.skipped_code_blocks, parted_bands: counters.pool_band_parts, + #[cfg(feature = "parallel")] + ring_slots: ( + counters.pool_band_finest_slots, + counters.pool_band_other_slots, + ), } } @@ -969,6 +983,89 @@ fn classic_streamed_rows_charge_the_batches_of_their_readers() { } } +/// The fixture's noise repeated `across` times side by side and `down` +/// times one under the other, with `components` components, encoded with +/// four levels and 16-row code-blocks: every component has more rows of +/// blocks than the pool threads, so its bands decode on the pool. +#[cfg(feature = "parallel")] +fn pool_band_fixture(components: u8, across: usize, down: usize) -> Vec { + let row = usize::from(components) * WIDTH as usize; + let rows: Vec = noisy_pixels(components) + .chunks_exact(row) + .flat_map(|samples| core::iter::repeat_n(samples, across).flatten()) + .copied() + .collect(); + let width = WIDTH * u32::try_from(across).expect("fixture width"); + let height = HEIGHT * u32::try_from(down).expect("fixture height"); + encode( + &rows.repeat(down), + width, + height, + u16::from(components), + 8, + false, + &EncodeOptions { + num_decomposition_levels: 4, + code_block_height_exp: 2, + ..EncodeOptions::default() + }, + ) + .expect("encode pool band fixture") +} + +/// A classic tile that reconstructs each component on its own reader without +/// deep band rings holds two bands in each finest ring when its pool has +/// more threads than the tile has finest rings, and its bands then split +/// into parts so every thread has one: three components on 10 and 11 +/// threads and at half resolution from 12, four components at half +/// resolution on 13 threads but not on 12. One reader keeps three bands, +/// and the deep rings of full resolution from 12 threads four, both in +/// whole bands; the coarser levels keep theirs. The rows match the serial +/// planes on every pool. +#[cfg(feature = "parallel")] +#[test] +fn shallow_finest_rings_split_their_bands_into_parts() { + // Three fixture widths side by side give finest-level bands of several + // code-blocks at both resolutions, each above the unit tests' reader + // size gate, and two heights give the coarser levels rings that hold no + // whole window. + for (components, threads) in [(3_u8, &[9_usize, 10, 11, 12][..]), (4, &[12, 13][..])] { + let bytes = pool_band_fixture(components, 3, 2); + for reduction in [0_u8, 1] { + let image = Image::new_with_reduction(&bytes, &DecodeSettings::default(), reduction) + .expect("parse shallow ring fixture"); + let roi = (0, 0, image.width(), image.height()); + let planes = decode_planes(&image, roi); + for &threads in threads { + let label = + format!("components={components} threads={threads} reduction={reduction}"); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("build pool"); + let rows = pool.install(|| decode_rows(&image, roi, CpuDecodeParallelism::Auto)); + assert_eq!(rows.planes, planes.planes, "{label}"); + assert_eq!( + rows.pool_components, + usize::from(components), + "{label}: pool bands" + ); + let deep = threads >= 12 && reduction == 0; + let shallow = threads >= 10 && !deep && 3 * usize::from(components) < threads; + let slots = if deep { + (4, 3) + } else if shallow { + (2, 2) + } else { + (3, 2) + }; + assert_eq!(rows.ring_slots, slots, "{label}: ring slots"); + assert_eq!(rows.parted_bands > 0, shallow, "{label}: parted bands"); + } + } + } +} + /// A streamed decode that holds the image's packet directory and runs out of /// budget gives the directory up and reruns only while no row has reached the /// sink, since a rerun after that would send those rows again. This decode From 7469685093bdf1dfc4d8c883b83d086556afe830 Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 01:24:27 -0500 Subject: [PATCH 82/94] docs(changelog): note the two finest-level bands of shallow classic rings --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ae54912..1aca2413 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,17 @@ Staged workspace version: `0.13.2`. status, detail and batch source index. CUDA keeps its typed source and job attribution. Unsupported statuses, raw runtime failures and combined cleanup or release failures keep their existing classes. +- Classic tiles of several components that stream their rows from the + worker pool's bands with a reader per component and full-width reads, + but without the deeper band rings of 12 threads and more, as on 10 and + 11 threads or at a reduced resolution, hold two bands in each + finest-level ring in place of three when the pool has more threads than + three per component. The bands ahead of the readers are then fewer than + the pool's threads, so they split into parts that every thread takes. A + 5661 x 8287 RGB image on 10 threads peaks at 49.2 MiB in place of 55.3 + MiB and an 18000 x 12000 one at 148 MiB in place of 168 MiB, and the + latter at half resolution on 12 threads at 79 MiB in place of 89 MiB, + with the same or slightly shorter decode times. Output is unchanged. - `decode_rows_u8_bounded` and `decode_rows_u16_bounded` decode an image whose tiles span its width, and whose headers let its region decode stream its rows, once, as a region decode of the whole image that streams its From fa479010239841acfd94f522eac9bbe5bb01372a Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 02:02:22 -0500 Subject: [PATCH 83/94] perf(j2k): hold two bands per ring for images that share a batch's pool A batch with an explicit worker count keeps up to that many images in flight on one stream pool of as many threads. Each image's pool bands hold the rings of a decode that has the pool to itself: three bands at the finest level and two at the others, plus one at every level with the deep rings of 12 threads and more. The extra depth keeps a lone decode's threads busy while its readers cross a band row; when the batch plans as many images at once as the pool has threads, each thread starts with an image of its own and an idle one takes the bands of other images still in flight instead. DecoderContext gains a doc-hidden set_shared_pool (one line in the rustdoc-hidden API snapshot). The batch sets it on the context of every chunk it runs on its stream pool when the workers it plans at once are at least the pool's threads (shares_stream_pool): eight images on 2, 3, 4 or 8 workers, three on 2, but not eight on 6 (four at a time) or two on 18. An image encoded in more than one and a half times the group's mean bytes keeps the usual rings (shared_ring_bytes): Tier-1 work follows the coded bytes, so it is the likeliest to decode last with the pool to itself, where the shallow rings cost a lone decode 25-40%. The tile context carries the mark; from_workspace and into_workspace clear it, so a recovered workspace decodes with the usual rings. A decode with the mark on a pool of fewer than 12 threads holds two bands in every ring of its pool bands (shared_rings) and keeps its bands whole; the row batches are unchanged. Two-band rings leave fewer bands ahead than a 10-thread pool has threads, and splitting them into parts for the pool's other threads, which the other images keep busy, made ten images on 10 workers 1.3-1.6% slower on both machines. From 12 threads a lone tile holds the deep rings, and two bands a ring in their place made eighteen images on 18 workers 1.6-1.9% slower, with whole bands too, so the mark leaves those decodes alone. One band at the coarser levels saved more but made 4-worker batches 3-6% slower on a 10-core machine, as a reader then decodes those bands alone. Peak heap of the batch's first iteration (counting allocator, including the decoded output), output identical: - eight 2048 x 2048 RGB images on 2 / 4 / 8 workers: 121.4 / 147.3 / 200.6 -> 116.9 / 138.3 / 182.6 MiB; - ten 2048 x 2048 RGB images on 10 workers: 252.0 -> 229.5 MiB; - eight 2896 x 2896 gray images on 8 workers: 117.5 -> 108.9 MiB; - eight 2048 x 2048 HTJ2K RGB images on 4 workers: 239.8 -> 230.8 MiB; - eighteen on 18 workers: unchanged (736.3 MiB). Batch times, 7 interleaved rounds against two builds of the base on an 18-core and a 10-core machine, stay within the 1% noise on every one of these batches (ten on 10 workers 1.001 / 0.997, eighteen on 18 1.002). One noise image with 3.7 times the coded bytes among crops of the same shape, eight on 8 workers, took 2.2-2.5% longer with every image marked and takes the same time with the size cutoff (1.000, 0.999). Single-image decodes are unchanged. The row-sink test decodes a three-component tile on 4, 10 and 12 threads with and without the mark and checks the deepest ring of each level and the bands in parts: two bands a ring with the mark below 12 threads, and whole bands where on 10 threads a lone decode's two-band rings split them; on 12 threads the marked decode holds the lone decode's deep rings. The rows match the serial planes and a marked decode with fewer bands a ring peaks lower. A context recovered from the workspace of a marked decode holds the unmarked rings. Batch tests check which worker and image counts share the pool and which images the size cutoff leaves out. In the band executor's counting-allocator test, every case also decodes with the mark: marked decodes hold two bands in every ring below 12 threads and never the deep rings, within their charge. --- crates/j2k-native/src/j2c/build/allocation.rs | 34 +++++--- crates/j2k-native/src/j2c/decode.rs | 1 + .../src/j2c/decode/parallel_bands.rs | 66 +++++++++++++--- crates/j2k-native/src/j2c/decode/state.rs | 9 +++ crates/j2k-native/src/j2c/decode/workspace.rs | 17 +++- crates/j2k-native/src/tests/row_sink.rs | 77 +++++++++++++++++++ crates/j2k/src/batch/scheduler.rs | 2 +- crates/j2k/src/batch/scheduler/retained.rs | 9 +++ crates/j2k/src/owned_batch.rs | 2 +- crates/j2k/src/owned_batch/cpu_execute.rs | 43 +++++++++-- .../prepare/cpu_preflight_tests.rs | 48 ++++++++++++ ...able-api-1.0.implementation-public-api.txt | 1 + 12 files changed, 282 insertions(+), 27 deletions(-) diff --git a/crates/j2k-native/src/j2c/build/allocation.rs b/crates/j2k-native/src/j2c/build/allocation.rs index 0a0ab7fc..378dedf9 100644 --- a/crates/j2k-native/src/j2c/build/allocation.rs +++ b/crates/j2k-native/src/j2c/build/allocation.rs @@ -403,6 +403,25 @@ const DEEP_RING_EXTRA_SLOTS: u32 = 1; /// into parts instead of waiting for whole bands further ahead. const SHALLOW_FINEST_LEVEL_BAND_SLOTS: u32 = 2; +/// The band slots of each sub-band's ring with parallel bands, at the +/// finest active level and at the others. With `shared_pool_band_rings` the +/// finest level holds the shallow rings (and no ring the deep rings' extra +/// band, which needs a larger pool): pool threads that find no band of the +/// image to start take another image's bands instead of bands further ahead. +fn pool_band_slots(storage: &DecompositionStorage<'_>) -> (u32, u32) { + let finest = if storage.shallow_band_rings || storage.shared_pool_band_rings { + SHALLOW_FINEST_LEVEL_BAND_SLOTS + } else { + FINEST_LEVEL_BAND_SLOTS + }; + let extra = if storage.deep_band_rings { + DEEP_RING_EXTRA_SLOTS + } else { + 0 + }; + (finest + extra, OTHER_LEVEL_BAND_SLOTS + extra) +} + /// Give each sub-band of a region decode the coefficient range for its /// stored rectangle: the required code-blocks, clipped to the IDWT windows /// unless `clip_to_idwt_windows` is false. With banded Tier-1, a range holds @@ -417,6 +436,7 @@ fn lay_out_region_coefficients( return Err(DecodingError::CodeBlockDecodeFailure.into()); }; let mut coefficient_count = 0usize; + let (finest_slots, other_slots) = pool_band_slots(storage); // Build stores each component's sub-bands together, starting with its // LL band, so a component starts where its LL band does. let per_component = storage.coefficient_layout == CoefficientLayout::Component; @@ -464,17 +484,10 @@ fn lay_out_region_coefficients( .sub_bands .contains(&sub_band_idx) }); - let slots = if finest && storage.shallow_band_rings { - SHALLOW_FINEST_LEVEL_BAND_SLOTS - } else if finest { - FINEST_LEVEL_BAND_SLOTS - } else { - OTHER_LEVEL_BAND_SLOTS - }; - if storage.deep_band_rings { - slots + DEEP_RING_EXTRA_SLOTS + if finest { + finest_slots } else { - slots + other_slots } } else { 1 @@ -524,6 +537,7 @@ pub(crate) fn lay_out_whole_region_coefficients( storage.parallel_bands = false; storage.deep_band_rings = false; storage.shallow_band_rings = false; + storage.shared_pool_band_rings = false; let coefficient_count = lay_out_region_coefficients(storage, true, 0)?; let mut budget = reuse::ReallocationBudget::from_live_bytes(storage.structural_workspace_bytes)?; diff --git a/crates/j2k-native/src/j2c/decode.rs b/crates/j2k-native/src/j2c/decode.rs index 76c9fadb..0f55e222 100644 --- a/crates/j2k-native/src/j2c/decode.rs +++ b/crates/j2k-native/src/j2c/decode.rs @@ -997,6 +997,7 @@ fn prepare_region_storage( && storage.lockstep_lines && parallel_bands::one_reader(tile, storage); storage.shallow_band_rings = parallel_bands::shallow_rings(tile, storage); + storage.shared_pool_band_rings = parallel_bands::shared_rings(tile_ctx, storage); } build::allocate_deferred_coefficients( storage, diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index 772f7816..2b969e1e 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -252,6 +252,7 @@ fn decode_route( if route.coding() == CodeBlockCoding::Classic { split_bands( tile_ctx, + storage.shared_pool_band_rings, &rings, &mut bands, route.workers() + 1, @@ -461,6 +462,7 @@ pub(super) fn decode_tile_rows<'a>( if coding == CodeBlockCoding::Classic { split_bands( tile_ctx, + storage.shared_pool_band_rings, &rings, &mut bands, workers + readers, @@ -731,6 +733,21 @@ pub(super) fn shallow_rings(tile: &Tile<'_>, storage: &DecompositionStorage<'_>) && reads_full_width(storage) } +/// Whether a tile streaming its rows from pool bands holds the shared rings +/// (see `DecompositionStorage::shared_pool_band_rings`): its decode shares +/// the pool with other images (see `TileDecodeContext::shared_pool`), on +/// fewer than [`MIN_CLASSIC_SYNTHESIS_THREADS`] threads. From there a lone +/// classic tile holds deep rings, and two bands a ring left a batch filling +/// an 18-thread pool 1-2% slower. +pub(super) fn shared_rings( + tile_ctx: &TileDecodeContext, + storage: &DecompositionStorage<'_>, +) -> bool { + storage.parallel_bands + && tile_ctx.shared_pool + && rayon::current_num_threads() < MIN_CLASSIC_SYNTHESIS_THREADS +} + /// Whether a tile streaming its rows from pool bands reconstructs them /// through one reader whatever workers its routes plan (see /// [`parallel_synthesis`]): it has one component, or classic code-blocks on @@ -1246,9 +1263,13 @@ const MAX_BAND_PARTS: usize = 8; const BAND_PARTS_PER_THREAD: usize = 2; /// Split a classic tile's bands into parts for `threads` threads (see -/// [`plan_parts`]), and count the bands that split. +/// [`plan_parts`]), and count the bands that split. A tile on the shared +/// rings (see `DecompositionStorage::shared_pool_band_rings`) keeps its bands +/// whole: the other images' bands keep the pool's threads busy, and the parts +/// left a batch filling a 10-thread pool 1% slower. fn split_bands( tile_ctx: &mut TileDecodeContext, + shared: bool, rings: &[Ring], bands: &mut [Band], threads: usize, @@ -1258,6 +1279,9 @@ fn split_bands( let forced = tile_ctx.band_parts; #[cfg(not(test))] let forced = None; + if shared && forced.is_none() { + return Ok(()); + } plan_parts(rings, bands, threads, forced, split)?; tile_ctx.debug_counters.pool_band_parts += bands.iter().filter(|band| band.parts > 1).count(); Ok(()) @@ -2580,6 +2604,7 @@ mod tests { let workers = route_workers.min(tile_ctx.classic_task_workspaces.len() - readers); split_bands( tile_ctx, + storage.shared_pool_band_rings, &rings, &mut bands, workers + readers, @@ -2788,6 +2813,7 @@ mod tests { bands: usize, deep: bool, shallow: bool, + shared: bool, /// The deepest finest-level and coarser-level rings that hold no /// whole window. slots: (usize, usize), @@ -2843,10 +2869,12 @@ mod tests { bytes: &[u8], roi: Option<(u32, u32, u32, u32)>, parts: Option, + shared: bool, ) -> Measured { let image = Image::new(bytes, &DecodeSettings::default()).expect("parse fixture"); let roi = roi.unwrap_or((0, 0, image.width(), image.height())); let mut context = DecoderContext::default(); + context.set_shared_pool(shared); context.set_output_region(Some(roi)); context.tile_decode_context.line_region_idwt = true; context.tile_decode_context.stream_rows = true; @@ -2942,6 +2970,7 @@ mod tests { if coding == CodeBlockCoding::Classic { split_bands( tile_ctx, + storage.shared_pool_band_rings, &rings, &mut bands, workers + readers, @@ -3029,6 +3058,7 @@ mod tests { bands: bands.len(), deep: storage.deep_band_rings, shallow: storage.shallow_band_rings, + shared: storage.shared_pool_band_rings, slots: deepest_rings(&rings), } } @@ -3202,7 +3232,8 @@ mod tests { /// while they decode: for classic and HTJ2K tiles, whole and in a /// region, with bands in parts, on pools of 2, 4, 10, 11 and 12 threads /// and from outside a pool, with one reader and with one per component, - /// with shallow rings and deep ones. + /// with shallow rings and deep ones, with the shared-pool mark and + /// without. #[test] fn band_executor_allocates_within_its_charge() { if !measures_alone("band_executor_allocates_within_its_charge") { @@ -3218,7 +3249,7 @@ mod tests { let ht = ht_fixture(); let wide = shallow_fixture(); let (mut readers, mut parted, mut clipped, mut deep) = (Vec::new(), 0, false, false); - let mut shallow = Vec::new(); + let (mut shallow, mut shared) = (Vec::new(), Vec::new()); let mut check = |case: &str, measured: Measured, whole: &Measured| { assert!( measured.routing.0 <= measured.routing_charge, @@ -3234,7 +3265,13 @@ mod tests { measured.running <= measured.running_charge, "{case}: {measured:?}" ); - if measured.shallow { + if measured.shared { + assert!( + measured.slots.0 <= 2 && measured.slots.1 <= 2, + "{case}: shared rings" + ); + shared.push((case.to_owned(), measured.slots, measured.deep)); + } else if measured.shallow { assert_eq!(measured.slots.0, 2, "{case}: shallow finest rings"); shallow.push((case.to_owned(), measured.slots, measured.parted)); } @@ -3257,11 +3294,15 @@ mod tests { ("HTJ2K", &ht[..], &[None][..]), ("wide classic", &wide[..], &[None, Some(4)][..]), ] { - let whole = measure_executor(bytes, None, None); + let whole = measure_executor(bytes, None, None, false); for &roi in &[None, Some((48, 48, 16, 16))] { for &parts in splits { - let case = format!("{label} {roi:?} parts {parts:?} on {threads}"); - check(&case, measure_executor(bytes, roi, parts), &whole); + for shared in [false, true] { + let case = format!( + "{label} {roi:?} parts {parts:?} shared {shared} on {threads}" + ); + check(&case, measure_executor(bytes, roi, parts, shared), &whole); + } } } } @@ -3269,10 +3310,10 @@ mod tests { } // Called from outside a pool, the executor sends its workers through // the global pool's shared queue. - let whole = measure_executor(classic, None, None); + let whole = measure_executor(classic, None, None, false); check( "classic outside the pool", - measure_executor(classic, None, Some(4)), + measure_executor(classic, None, Some(4), false), &whole, ); drop(pools); @@ -3291,6 +3332,13 @@ mod tests { && *parted > 0), "{shallow:?}" ); + // Marked decodes hold two bands in every ring below 12 threads, and + // never with the deep rings, which hold from 12. + assert!( + shared.iter().any(|(_, slots, _)| *slots == (2, 2)) + && shared.iter().all(|(_, _, deep_rings)| !deep_rings), + "{shared:?}" + ); } /// A classic RGB tile of noise wide and tall enough for the shallow diff --git a/crates/j2k-native/src/j2c/decode/state.rs b/crates/j2k-native/src/j2c/decode/state.rs index 5f16d540..8e66d680 100644 --- a/crates/j2k-native/src/j2c/decode/state.rs +++ b/crates/j2k-native/src/j2c/decode/state.rs @@ -131,6 +131,11 @@ pub(crate) struct DecompositionStorage<'a> { /// reader per component without deep rings (see /// [`parallel_bands::shallow_rings`]). pub(crate) shallow_band_rings: bool, + /// Whether the band rings of `parallel_bands` hold two bands at every + /// level and their bands stay whole, for a decode whose pool other images + /// share, on a pool too small for deep rings (see + /// [`parallel_bands::shared_rings`]). + pub(crate) shared_pool_band_rings: bool, /// Whether a tile streaming its rows from `parallel_bands` reconstructs /// them through one reader whatever its pool's workers (see /// [`parallel_bands::one_reader`]), which holds one short batch of rows. @@ -178,6 +183,7 @@ impl DecompositionStorage<'_> { self.parallel_bands = false; self.deep_band_rings = false; self.shallow_band_rings = false; + self.shared_pool_band_rings = false; self.one_synthesis_reader = false; self.lockstep_lines = false; self.roi_idwt_charge_elements = 0; @@ -321,6 +327,9 @@ pub(crate) struct TileDecodeContext { /// Whether the inverse MCT is left to the caller's RGB8 pack, set /// around a decode like `round_irreversible_output`. pub(crate) defer_mct: bool, + /// Whether this context's decodes share their pool with other images + /// decoding at the same time (see `DecoderContext::set_shared_pool`). + pub(crate) shared_pool: bool, /// The inverse MCT this decode left to the caller. pub(crate) deferred_mct: Option, /// The last IDWT level this decode leaves to the RGB8 pack with the diff --git a/crates/j2k-native/src/j2c/decode/workspace.rs b/crates/j2k-native/src/j2c/decode/workspace.rs index 7d8e4916..79104790 100644 --- a/crates/j2k-native/src/j2c/decode/workspace.rs +++ b/crates/j2k-native/src/j2c/decode/workspace.rs @@ -207,7 +207,8 @@ impl Default for DecoderContext<'_> { impl DecoderContext<'_> { /// Create a borrowing decoder context from a lifetime-free reusable workspace. #[must_use] - pub fn from_workspace(workspace: DecoderWorkspace) -> Self { + pub fn from_workspace(mut workspace: DecoderWorkspace) -> Self { + workspace.tile_decode_context.shared_pool = false; Self { tile_decode_context: workspace.tile_decode_context, storage: DecompositionStorage::default(), @@ -223,6 +224,7 @@ impl DecoderContext<'_> { #[must_use] pub fn into_workspace(mut self) -> DecoderWorkspace { self.storage.release_all_allocations(); + self.tile_decode_context.shared_pool = false; DecoderWorkspace { tile_decode_context: self.tile_decode_context, cpu_decode_parallelism: self.cpu_decode_parallelism, @@ -316,6 +318,19 @@ impl DecoderContext<'_> { self.cpu_decode_parallelism = parallelism; } + /// Mark this context's decodes as sharing their worker pool with other + /// images that decode on it at the same time, as the images of a batch do + /// that plans as many at once as its pool has threads. + /// Pool threads with no band of one image to decode take another image's + /// bands, so on a pool of fewer than 12 threads each image's band rings + /// hold two bands at every level and its bands stay whole; larger pools + /// keep the usual rings. Off by default; recovering the context's + /// workspace turns it off. + #[doc(hidden)] + pub fn set_shared_pool(&mut self, shared: bool) { + self.tile_decode_context.shared_pool = shared; + } + /// Turn the row-streamed region IDWT on or off; it is on by default and /// lowers region-decode memory without changing output. #[cfg(test)] diff --git a/crates/j2k-native/src/tests/row_sink.rs b/crates/j2k-native/src/tests/row_sink.rs index 1e8a8ce9..af2c6050 100644 --- a/crates/j2k-native/src/tests/row_sink.rs +++ b/crates/j2k-native/src/tests/row_sink.rs @@ -1066,6 +1066,83 @@ fn shallow_finest_rings_split_their_bands_into_parts() { } } +/// A decode that shares its pool with other images in flight, as the images +/// of an explicit-worker batch that fill its pool do, holds two bands in +/// every ring below 12 threads, in place of three at the finest level, and +/// keeps its bands whole, where on 10 threads a lone decode's two-band rings +/// split them; with fewer bands per ring it peaks lower than a lone decode. +/// From 12 threads it holds the lone decode's deep rings. Its rows match the serial planes. A context +/// recovered from the workspace of a marked decode decodes as a fresh one +/// does. +#[cfg(feature = "parallel")] +#[test] +fn shared_pool_rows_hold_two_bands_per_ring() { + // Two fixture heights give the coarser levels rings that hold no whole + // window, deep ones too. + let bytes = pool_band_fixture(3, 1, 2); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("parse shared pool fixture"); + let roi = (0, 0, image.width(), image.height()); + let planes = decode_planes(&image, roi); + for (threads, lone_slots, shared_slots) in [ + (4, (3, 2), (2, 2)), + (10, (2, 2), (2, 2)), + (12, (4, 3), (4, 3)), + ] { + let label = format!("threads={threads}"); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("build pool"); + let (lone, shared, recovered) = pool.install(|| { + let lone = decode_rows(&image, roi, CpuDecodeParallelism::Auto); + let shared = decode_rows_with(&image, roi, CpuDecodeParallelism::Auto, |context| { + context.set_shared_pool(true); + }); + let mut marked = DecoderContext::default(); + marked.set_shared_pool(true); + let mut sink = CollectingSink::new(roi.2, roi.3); + image + .decode_region_rows_for_integer_output_with_context(roi, &mut marked, &mut sink) + .expect("decode marked rows"); + let workspace = marked.into_workspace(); + let recovered = decode_rows_with(&image, roi, CpuDecodeParallelism::Auto, |context| { + *context = DecoderContext::from_workspace(workspace); + }); + (lone, shared, recovered) + }); + for (decoded, name) in [ + (&lone, "lone"), + (&shared, "shared"), + (&recovered, "recovered"), + ] { + assert_eq!(decoded.planes, planes.planes, "{label} {name}"); + assert_eq!(decoded.pool_components, 3, "{label} {name}: pool bands"); + } + assert_eq!(lone.ring_slots, lone_slots, "{label}: lone ring slots"); + assert_eq!( + shared.ring_slots, shared_slots, + "{label}: shared ring slots" + ); + assert_eq!( + recovered.ring_slots, lone_slots, + "{label}: recovered ring slots" + ); + assert_eq!( + (lone.parted_bands > 0, shared.parted_bands), + (threads == 10, 0), + "{label}: bands in parts" + ); + if shared_slots != lone_slots { + assert!( + shared.peak_bytes < lone.peak_bytes, + "{label}: shared {} lone {}", + shared.peak_bytes, + lone.peak_bytes + ); + } + } +} + /// A streamed decode that holds the image's packet directory and runs out of /// budget gives the directory up and reruns only while no row has reached the /// sink, since a rerun after that would send those rows again. This decode diff --git a/crates/j2k/src/batch/scheduler.rs b/crates/j2k/src/batch/scheduler.rs index 09c239f6..b6984440 100644 --- a/crates/j2k/src/batch/scheduler.rs +++ b/crates/j2k/src/batch/scheduler.rs @@ -25,7 +25,7 @@ use crate::CpuDecodeParallelism; mod retained; pub(crate) use self::retained::{ - decode_retained_tiles, retained_inner_parallelism, run_retained_chunks, + decode_retained_tiles, retained_inner_parallelism, retained_worker_count, run_retained_chunks, }; pub(super) type ScopedWorkerHandle<'scope> = ( diff --git a/crates/j2k/src/batch/scheduler/retained.rs b/crates/j2k/src/batch/scheduler/retained.rs index 427157ba..8c2d7b03 100644 --- a/crates/j2k/src/batch/scheduler/retained.rs +++ b/crates/j2k/src/batch/scheduler/retained.rs @@ -26,6 +26,15 @@ pub(crate) fn retained_inner_parallelism( Ok(plan_batch(job_count, options)?.inner_parallelism) } +/// Workers that [`run_retained_chunks`] runs at once for a batch of +/// `job_count` jobs. +pub(crate) fn retained_worker_count( + job_count: usize, + options: TileBatchOptions, +) -> Result { + Ok(plan_batch(job_count, options)?.worker_count) +} + /// Run disjoint jobs without destroying worker-owned scratch after each call. pub(crate) fn run_retained_chunks( workers: &mut [BatchWorker], diff --git a/crates/j2k/src/owned_batch.rs b/crates/j2k/src/owned_batch.rs index 4dab7ec1..2e875afe 100644 --- a/crates/j2k/src/owned_batch.rs +++ b/crates/j2k/src/owned_batch.rs @@ -14,7 +14,7 @@ use crate::{ backend, batch::{ allocation::{generic_batch_worker_limit, J2K_BATCH_METADATA_ALLOWANCE_BYTES}, - scheduler::{retained_inner_parallelism, run_retained_chunks}, + scheduler::{retained_inner_parallelism, retained_worker_count, run_retained_chunks}, worker::BatchWorker, }, decode::decode_warnings_for_recovery, diff --git a/crates/j2k/src/owned_batch/cpu_execute.rs b/crates/j2k/src/owned_batch/cpu_execute.rs index 6954a560..7b87c796 100644 --- a/crates/j2k/src/owned_batch/cpu_execute.rs +++ b/crates/j2k/src/owned_batch/cpu_execute.rs @@ -9,11 +9,12 @@ use super::cpu_materialize::{ }; use super::cpu_staged_execute::run_staged_typed_group; use super::{ - decode_warnings_for_recovery, retained_inner_parallelism, run_retained_chunks, Arc, - BatchCodecRoute, BatchColor, BatchDecodeOptions, BatchErrorStage, BatchGroupInfo, - BatchInfrastructureError, BatchItemError, BatchWorker, CpuBatchGroup, CpuBatchSamples, - CpuDecodeParallelism, CpuGroupFastWorkspace, CpuStagedWorkspace, IndexedBatchError, J2kError, - NativeSampleType, NonZeroUsize, PreparedBatchGroup, PreparedImage, TileBatchOptions, Vec, + decode_warnings_for_recovery, retained_inner_parallelism, retained_worker_count, + run_retained_chunks, Arc, BatchCodecRoute, BatchColor, BatchDecodeOptions, BatchErrorStage, + BatchGroupInfo, BatchInfrastructureError, BatchItemError, BatchWorker, CpuBatchGroup, + CpuBatchSamples, CpuDecodeParallelism, CpuGroupFastWorkspace, CpuStagedWorkspace, + IndexedBatchError, J2kError, NativeSampleType, NonZeroUsize, PreparedBatchGroup, PreparedImage, + TileBatchOptions, Vec, }; use crate::batch::admission::{hold_at_least, BatchAllocationBudget}; use crate::batch::allocation::try_vec_with_capacity; @@ -209,6 +210,32 @@ pub(super) fn image_policy( ) } +/// Whether the images of a group of `job_count` that decode on a stream pool +/// of `pool_threads` share it (see `DecoderContext::set_shared_pool`): the +/// batch plans at least as many workers at once as the pool has threads, so +/// each thread starts with an image of its own and an idle one takes the +/// bands of other images still in flight. The images that finish last keep +/// those rings with fewer images beside them. With fewer workers planned +/// than threads, each image holds the rings of a decode alone on the pool. +pub(super) fn shares_stream_pool( + job_count: usize, + tile_options: TileBatchOptions, + pool_threads: usize, +) -> Result { + Ok(retained_worker_count(job_count, tile_options)? >= pool_threads) +} + +/// The encoded bytes up to which an image of a group that shares its stream +/// pool holds the shared rings, from the encoded lengths of the group's +/// images: one and a half times their mean. Tier-1 work follows the coded +/// bytes, so a larger image is the likeliest to decode last with few other +/// images beside it, and keeps the rings of a decode alone on the pool. +pub(super) fn shared_ring_bytes(lengths: impl ExactSizeIterator) -> usize { + let count = lengths.len().max(1); + let mean = lengths.fold(0_usize, usize::saturating_add) / count; + mean.saturating_add(mean / 2) +} + /// The threads a group's streamed images decode on when the caller set the /// worker count, created on first use and kept by the session. /// @@ -312,6 +339,11 @@ fn run_typed_group( convert, )?; } else { + let shared_pool = match stream_pool { + Some(pool) => shares_stream_pool(jobs.len(), tile_options, pool.current_num_threads())?, + None => false, + }; + let shared_bytes = shared_ring_bytes(group.images.iter().map(|image| image.bytes().len())); run_retained_chunks( workers, &mut jobs, @@ -327,6 +359,7 @@ fn run_typed_group( let mut context = j2k_native::DecoderContext::from_workspace(workspace); let mut claim = None; for (job, slot) in jobs.iter_mut().zip(results) { + context.set_shared_pool(shared_pool && job.image.bytes().len() <= shared_bytes); hold_at_least( admission, &mut claim, diff --git a/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs b/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs index 202a8f8d..a0aef92e 100644 --- a/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs +++ b/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs @@ -668,6 +668,54 @@ fn explicit_worker_counts_set_the_policy_of_images_decoded_one_at_a_time() { assert_eq!(image_policy(3, true, None), (false, None)); } +#[test] +fn batch_images_share_the_stream_pool_only_when_they_fill_it() { + use super::super::cpu_execute::shares_stream_pool; + let limit = crate::batch::allocation::generic_batch_worker_limit(); + // The session's workers, and so its stream pool, stop at the limit. + let cases = [ + // Eight images on 2, 3, 4 or 8 workers, and three on 2, keep an image + // in flight for every thread of the pool. + (8, 2, true), + (8, 3, true), + (8, 4, true), + (8, 8, true), + (3, 2, true), + // Eight on 6 run four at a time, five on 4 three, two on 18 two: + // fewer images in flight than threads. + (8, 6, false), + (5, 4, false), + (2, 18, false), + ]; + for (images, workers, shared) in cases.into_iter().filter(|case| case.1 <= limit) { + let options = crate::TileBatchOptions::new(NonZeroUsize::new(workers)); + assert_eq!( + shares_stream_pool(images, options, workers).expect("plan"), + shared, + "{images} on {workers}" + ); + } +} + +#[test] +fn images_far_above_their_group_mean_keep_the_lone_rings() { + use super::super::cpu_execute::shared_ring_bytes; + // Seven crops of about the same coded size share the rings; one noise + // image of the same shape with 3.7 times their bytes keeps its own. + let crops = [ + 3_585_095, 3_622_220, 3_720_453, 3_745_469, 3_787_355, 3_807_463, 3_648_167, + ]; + let mut group = crops.to_vec(); + group.push(13_840_854); + let limit = shared_ring_bytes(group.into_iter()); + assert!(crops.iter().all(|&bytes| bytes <= limit), "{limit}"); + assert!(13_840_854 > limit, "{limit}"); + // A uniform group shares them all, half again above the mean at most. + assert_eq!(shared_ring_bytes([100, 100, 100].into_iter()), 150); + assert_eq!(shared_ring_bytes([90, 100, 110, 160].into_iter()), 172); + assert_eq!(shared_ring_bytes(core::iter::empty()), 0); +} + #[test] fn plans_beyond_the_flattening_cap_decode_one_image_at_a_time() { use super::super::cpu_fast::{ diff --git a/xtask/api/stable-api-1.0.implementation-public-api.txt b/xtask/api/stable-api-1.0.implementation-public-api.txt index 8fdefc3a..c32ff36b 100644 --- a/xtask/api/stable-api-1.0.implementation-public-api.txt +++ b/xtask/api/stable-api-1.0.implementation-public-api.txt @@ -2470,6 +2470,7 @@ pub fn j2k_native::DecodedComponents<'a>::into_parts(self) -> j2k_native::Decode pub fn j2k_native::DecodedComponents<'a>::live_bytes(&self) -> usize pub fn j2k_native::DecodedNativeComponents::allocated_bytes(&self) -> core::option::Option pub fn j2k_native::DecodedNativeComponents::into_parts(self) -> j2k_native::DecodedNativeComponentsParts +pub fn j2k_native::DecoderContext<'_>::set_shared_pool(&mut self, bool) pub fn j2k_native::HtCodeBlockDecodeProfile::new() -> Self pub fn j2k_native::HtCodeBlockDecodeWorkspace::coefficient_capacity(&self) -> usize pub fn j2k_native::HtCodeBlockDecoder::decode_code_block(&mut self, j2k_native::HtCodeBlockDecodeJob<'_>, &mut [f32]) -> j2k_native::Result<()> From 4749ed44d0a29abda59f66c5fe6ec9d1bc900cb7 Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 02:02:22 -0500 Subject: [PATCH 84/94] docs(changelog): note the two-band rings of images that share a batch's pool --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1aca2413..9000671b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,19 @@ Staged workspace version: `0.13.2`. status, detail and batch source index. CUDA keeps its typed source and job attribution. Unsupported statuses, raw runtime failures and combined cleanup or release failures keep their existing classes. +- Images that a batch with an explicit worker count decodes together on + its stream pool, when it plans as many images at once as the pool has + threads and the pool has fewer than 12, hold two bands in every ring of + the pool's bands in place of three at the finest level, and keep their + bands whole: a pool thread with no band of one image to decode takes + another image's bands. Eight 2048 x 2048 RGB images peak at 116.9, 138.3 + and 182.6 MiB in place of 121.4, 147.3 and 200.6 MiB on 2, 4 and 8 + workers, and ten on 10 workers at 229.5 in place of 252.0 MiB, in the + same batch time. From 12 threads, where a lone tile holds deeper rings, + the images keep the usual rings, as do an image encoded in more than one + and a half times its group's mean bytes, the likeliest to decode last + with the pool to itself, groups planned with fewer images at once than + pool threads, and single-image decodes. Output is unchanged. - Classic tiles of several components that stream their rows from the worker pool's bands with a reader per component and full-width reads, but without the deeper band rings of 12 threads and more, as on 10 and From e1f41dd6ece9d1169b8a4195d6083fbe8f769b80 Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 06:15:32 -0500 Subject: [PATCH 85/94] fix(j2k-native): keep a tile's reader batches within what the tile is charged for A tile streaming from pool bands through a reader per component sizes its batch of rows by the output region's width, while the ROI plan charges the batch by the tile's final window and the buffer holds rows as wide as the tile. One rule sizes both, and it halves a classic tile's single batch only while that leaves 128 rows, so the two disagree when the region and the tile fall on either side of 8,192 samples: in an 8,200-wide image of 8,192-wide tiles the first tile was charged 128 rows of each component and held 255, twice the charge (6.4 M against 3.4 M samples), past the budget the decode had checked. The batch keeps the rows the region's width gives, capped by the rows the tile's width is charged for. Where a tile held no more than its charge nothing changes; the case above now holds 128 rows. Output is unchanged. The new test decodes that image on 12 threads, a reader per component, and checks the batch's samples and that the line buffer stays within its charge; it fails without this change. On an 18-core ARM64 machine, 7 interleaved rounds, an 8200 x 2048 image in 8192-wide tiles decodes in 0.979 of the time on 12 threads and 0.983 on 18 (a second copy of the old build: 0.997 and 1.001), peak heap 72.6 -> 60.7 MiB; 8192 x 2048 in 4096-wide tiles and 4096 x 4096 in 1024-wide ones are unchanged (1.003 / 1.001 and 0.999 / 1.002, same heap), as on a 10-core machine on 10 threads. --- crates/j2k-native/src/j2c/decode.rs | 4 ++ .../src/j2c/decode/parallel_bands.rs | 19 ++++-- crates/j2k-native/src/tests/row_sink.rs | 68 +++++++++++++++++++ 3 files changed, 84 insertions(+), 7 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode.rs b/crates/j2k-native/src/j2c/decode.rs index 0f55e222..d4c87655 100644 --- a/crates/j2k-native/src/j2c/decode.rs +++ b/crates/j2k-native/src/j2c/decode.rs @@ -777,6 +777,10 @@ pub(crate) struct DecodeDebugCounters { /// bands held beyond the IDWT samples its ROI plan charged. #[cfg(all(test, feature = "parallel"))] pub(crate) uncharged_line_samples: usize, + /// The most samples of row batches a tile streaming from pool bands held + /// beside its IDWT rings. + #[cfg(all(test, feature = "parallel"))] + pub(crate) line_batch_samples: usize, /// Bytes of the pool band plans recorded in the workspace peak, summed /// over components and tiles. #[cfg(all(test, feature = "parallel"))] diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index 2b969e1e..3dd9e260 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -495,19 +495,23 @@ pub(super) fn decode_tile_rows<'a>( { tile_ctx.debug_counters.thread_scratch_charged += live.live_bytes() - unscratched; } - // Rows reconstruct in batches, charged by the region's width (see + // Rows reconstruct in batches as many as fit rows of the region's width, + // but never more than the tile's width is charged for (see // `roi_idwt_peak_elements`): two of them with a reader per component of // an HTJ2K tile, so one finishes while the next reconstructs; a classic // tile's readers wait on Tier-1 instead, and hold one batch of half the // rows, with deep band rings when they read whole rows. One reader holds // one short batch, which it finishes before it reconstructs the next. let pipelined = readers > 1 && coding == CodeBlockCoding::HighThroughput; - let batch = streamed_batch_rows( - region.width() as usize, - copy.height() as usize, - readers > 1 && !pipelined, - readers == 1, - ); + let batch_rows = |width: u32| { + streamed_batch_rows( + width as usize, + copy.height() as usize, + readers > 1 && !pipelined, + readers == 1, + ) + }; + let batch = batch_rows(region.width()).min(batch_rows(copy.width())); let scratch_len = (copy.width() as usize) .checked_mul(components) .and_then(|samples| samples.checked_mul(batch)) @@ -522,6 +526,7 @@ pub(super) fn decode_tile_rows<'a>( .fold(scratch_len, |held, len| held.saturating_add(*len)); let counters = &mut tile_ctx.debug_counters; counters.pool_band_readers = counters.pool_band_readers.max(readers); + counters.line_batch_samples = counters.line_batch_samples.max(scratch_len); counters.uncharged_line_samples = counters .uncharged_line_samples .max(held.saturating_sub(storage.roi_idwt_charge_elements)); diff --git a/crates/j2k-native/src/tests/row_sink.rs b/crates/j2k-native/src/tests/row_sink.rs index af2c6050..cb5a8dbb 100644 --- a/crates/j2k-native/src/tests/row_sink.rs +++ b/crates/j2k-native/src/tests/row_sink.rs @@ -1220,6 +1220,74 @@ fn streamed_rows_evict_the_packet_directory_before_the_first_row() { ); } +/// A classic tile whose components reconstruct on readers of their own +/// holds one batch of rows, which the ROI plan charges by the tile's width. +/// In an 8,200-wide image of 8,192-wide tiles, a batch sized by the image's +/// width keeps its 255 rows whole (half would be under 128), where the +/// tile's 8,192-wide rows are charged 128 (half of 256): the batch must keep +/// to the tile's charge, or the first tile holds twice it. +#[cfg(feature = "parallel")] +#[test] +fn reader_batches_of_tiles_narrower_than_the_image_stay_within_their_charge() { + let (width, height) = (8200_u32, 1024_u32); + // Smooth content keeps the large fixture's Tier-1 cheap. + let pixels: Vec = (0..height) + .flat_map(|y| { + (0..width).flat_map(move |x| { + [ + x.to_le_bytes()[0], + y.to_le_bytes()[0], + (x + y).to_le_bytes()[0], + ] + }) + }) + .collect(); + let bytes = encode( + &pixels, + width, + height, + 3, + 8, + false, + &EncodeOptions { + num_decomposition_levels: 2, + // 16-row blocks give each component more rows of blocks than + // the pool threads, so its bands decode on the pool. + code_block_height_exp: 2, + tile_size: Some((8192, height)), + ..EncodeOptions::default() + }, + ) + .expect("encode wide tile fixture"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("parse wide tile fixture"); + let roi = (0, 0, width, height); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(12) + .build() + .expect("build pool"); + let counters = pool.install(|| { + let mut context = DecoderContext::default(); + context.set_cpu_decode_parallelism(CpuDecodeParallelism::Auto); + let mut sink = CollectingSink::new(roi.2, roi.3); + image + .decode_region_rows_for_integer_output_with_context(roi, &mut context, &mut sink) + .expect("decode region rows"); + assert!(context.tile_decode_context.stream_rows); + assert!(sink.arrivals.iter().all(|&count| count == 1)); + context.tile_decode_context.debug_counters + }); + assert_eq!(counters.pool_band_readers, 3, "a reader per component"); + assert_eq!( + counters.line_batch_samples, + 3 * 8192 * 128, + "one batch of 128 rows of each component of the wide tile" + ); + assert_eq!( + counters.uncharged_line_samples, 0, + "line buffer beyond the charge" + ); +} + /// Stitched windows of a reduced resolution, as a tiled viewer or an IIIF /// server requests them, hold few rows of code-blocks per component: fewer /// than a pool of 18 threads has workers. Such a window's streamed rows must From 98fbf23ef219cfdc8328ab8eedd232b65734048d Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 04:58:31 -0500 Subject: [PATCH 86/94] fix(j2k-native): size the clipped-block scratch with the serial Tier-1 workspace An HTJ2K code-block that decodes straight to samples writes every row and column of the block, so one that the sub-band stores only part of decodes into scratch first. The serial sub-band decode allocated that scratch on first use, against the budget. A streamed region decode reaches the blocks its region clips at the bottom only after its first rows went to the sink, so with a budget that fit everything else it failed there, with a capacity error, after rows went out. The retries that rerun only a decode that has handed nothing over (without the retained scratch, or without the packet directory) were then closed to it. The serial Tier-1 workspace now plans the largest clipped block's scratch with the rest of the workspace and sizes it before the first band; it is kept beside the HTJ2K context and counted with it. A decode short of budget now fails before its first row. Output is unchanged. A tile with clipped HTJ2K blocks holds one more block of samples (at most 16 KiB) through its decode. The new test decodes classic and HTJ2K regions, two of them clipped at the bottom, from budgets at and above the largest one that fits: each either decodes every sample once or fails with a capacity error before its first row. It fails without this change. --- crates/j2k-native/src/j2c/decode/state.rs | 11 +++ crates/j2k-native/src/j2c/decode/subband.rs | 2 +- .../j2k-native/src/j2c/decode/subband/ht.rs | 6 +- crates/j2k-native/src/j2c/decode/tier1.rs | 33 ++++++- crates/j2k-native/src/tests/row_sink.rs | 96 +++++++++++++++++++ 5 files changed, 138 insertions(+), 10 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode/state.rs b/crates/j2k-native/src/j2c/decode/state.rs index 8e66d680..6e2e333b 100644 --- a/crates/j2k-native/src/j2c/decode/state.rs +++ b/crates/j2k-native/src/j2c/decode/state.rs @@ -253,6 +253,10 @@ pub(crate) struct TileDecodeContext { pub(crate) bit_plane_decode_buffers: BitPlaneDecodeBuffers, /// A reusable context for decoding HTJ2K code blocks. pub(crate) ht_block_decode_context: HtBlockDecodeContext, + /// Scratch an HTJ2K code-block that the sub-band stores only part of + /// decodes into before its stored part is copied out, sized with the + /// serial Tier-1 workspace. + pub(crate) ht_clipped_block_scratch: Vec, #[cfg(feature = "parallel")] pub(in crate::j2c::decode) ht_task_workspaces: Vec, /// One classic workspace per worker of the tile-wide route. @@ -420,6 +424,7 @@ impl TileDecodeContext { self.bit_plane_decode_context = BitPlaneDecodeContext::default(); self.bit_plane_decode_buffers = BitPlaneDecodeBuffers::default(); self.ht_block_decode_context = HtBlockDecodeContext::default(); + self.ht_clipped_block_scratch = Vec::new(); #[cfg(feature = "parallel")] { self.ht_task_workspaces = Vec::new(); @@ -432,9 +437,15 @@ impl TileDecodeContext { let classic_bytes = self.bit_plane_decode_context.allocated_bytes()?; let buffer_bytes = self.bit_plane_decode_buffers.allocated_bytes()?; let ht_bytes = self.ht_block_decode_context.allocated_bytes()?; + let clipped_bytes = self + .ht_clipped_block_scratch + .capacity() + .checked_mul(size_of::()) + .ok_or(ValidationError::ImageTooLarge)?; classic_bytes .checked_add(buffer_bytes) .and_then(|bytes| bytes.checked_add(ht_bytes)) + .and_then(|bytes| bytes.checked_add(clipped_bytes)) .ok_or(ValidationError::ImageTooLarge.into()) } diff --git a/crates/j2k-native/src/j2c/decode/subband.rs b/crates/j2k-native/src/j2c/decode/subband.rs index 7363434d..5c37ca1d 100644 --- a/crates/j2k-native/src/j2c/decode/subband.rs +++ b/crates/j2k-native/src/j2c/decode/subband.rs @@ -234,7 +234,7 @@ pub(super) fn decode_sub_band_bitplanes( } /// Where `code_block` lands in `sub_band`'s stored rectangle. -fn stored_placement(code_block: &CodeBlock, sub_band: &SubBand) -> BlockPlacement { +pub(super) fn stored_placement(code_block: &CodeBlock, sub_band: &SubBand) -> BlockPlacement { BlockPlacement::new(code_block.rect, sub_band.storage_rect) } diff --git a/crates/j2k-native/src/j2c/decode/subband/ht.rs b/crates/j2k-native/src/j2c/decode/subband/ht.rs index c652f367..dbf1ac73 100644 --- a/crates/j2k-native/src/j2c/decode/subband/ht.rs +++ b/crates/j2k-native/src/j2c/decode/subband/ht.rs @@ -289,8 +289,6 @@ pub(super) fn decode_sub_band_ht_blocks( } } - #[cfg(feature = "simd")] - let mut clipped_block_scratch = Vec::new(); for precinct in sub_band .precincts .clone() @@ -334,7 +332,7 @@ pub(super) fn decode_sub_band_ht_blocks( let mut budget = DecodeAllocationBudget::for_storage(storage)?; ( super::placement::zeroed_block_scratch( - &mut clipped_block_scratch, + &mut tile_ctx.ht_clipped_block_scratch, code_block.rect.width(), code_block.rect.height(), &mut budget, @@ -369,7 +367,7 @@ pub(super) fn decode_sub_band_ht_blocks( } if !whole { placement.copy_from_block( - &clipped_block_scratch, + &tile_ctx.ht_clipped_block_scratch, code_block.rect.width(), &mut storage.coefficients[sub_band.coefficients.clone()], stored_stride as usize, diff --git a/crates/j2k-native/src/j2c/decode/tier1.rs b/crates/j2k-native/src/j2c/decode/tier1.rs index 77a94322..ec388de7 100644 --- a/crates/j2k-native/src/j2c/decode/tier1.rs +++ b/crates/j2k-native/src/j2c/decode/tier1.rs @@ -2,15 +2,16 @@ //! One active-tile budget for serial Tier-1 contexts and copied segment scratch. -use super::subband::code_block_required_by_index; +use super::subband::{code_block_required_by_index, stored_placement}; use super::{ - CodeBlock, DecodeAllocationBudget, DecompositionStorage, Header, Result, Tile, + CodeBlock, DecodeAllocationBudget, DecompositionStorage, Header, Result, SubBand, Tile, TileDecodeContext, }; use crate::error::{DecodingError, ValidationError}; use crate::j2c::bitplane::classic_decode_workspace_bytes; use crate::j2c::build::CodeBlockCoding; use crate::j2c::ht_block_decode::ht_decode_workspace_bytes; +use crate::try_resize_decode_elements; use core::mem::size_of; use core::ops::Range; @@ -27,6 +28,7 @@ struct Tier1Requirements { ht_width: u32, ht_height: u32, ht_data_bytes: usize, + ht_clipped_samples: usize, has_classic: bool, has_ht: bool, } @@ -75,11 +77,24 @@ impl Tier1Requirements { fn observe_ht( &mut self, code_block: &CodeBlock, + sub_band: &SubBand, storage: &DecompositionStorage<'_>, ) -> Result<()> { self.has_ht = true; - self.ht_width = self.ht_width.max(code_block.rect.width()); - self.ht_height = self.ht_height.max(code_block.rect.height()); + let (width, height) = (code_block.rect.width(), code_block.rect.height()); + self.ht_width = self.ht_width.max(width); + self.ht_height = self.ht_height.max(height); + // A block decoding straight to samples writes all of them, so one + // the sub-band stores only part of needs scratch. + if cfg!(feature = "simd") + && code_block.number_of_coding_passes != 0 + && !stored_placement(code_block, sub_band).is_whole(width, height) + { + let samples = (width as usize) + .checked_mul(height as usize) + .ok_or(ValidationError::ImageTooLarge)?; + self.ht_clipped_samples = self.ht_clipped_samples.max(samples); + } if code_block.number_of_coding_passes != 0 { let (cleanup_bytes, refinement_bytes) = crate::j2c::ht_block_decode::selected_code_block_segment_lengths( @@ -112,6 +127,7 @@ impl Tier1Requirements { .checked_add(ht_decode_workspace_bytes(self.ht_width, self.ht_height)?) .ok_or(ValidationError::ImageTooLarge)?; include_elements::(&mut bytes, self.ht_data_bytes)?; + include_elements::(&mut bytes, self.ht_clipped_samples)?; } Ok(bytes) } @@ -146,6 +162,13 @@ pub(super) fn prepare_tier1_workspace( requirements.ht_height, requirements.ht_data_bytes, )?; + // Sized now, so a band decoding after rows went out never grows + // it past the budget. + let clipped = &mut tile_ctx.ht_clipped_block_scratch; + if clipped.len() < requirements.ht_clipped_samples { + clipped.clear(); + try_resize_decode_elements(clipped, requirements.ht_clipped_samples, 0.0)?; + } } let actual_bytes = tile_ctx.serial_tier1_capacity_bytes()?; @@ -243,7 +266,7 @@ fn observe_sub_band( requirements.observe_classic(code_block, storage)?; } Some(CodeBlockCoding::HighThroughput) => { - requirements.observe_ht(code_block, storage)?; + requirements.observe_ht(code_block, sub_band, storage)?; } None if code_block.number_of_coding_passes == 0 => {} None => return Err(DecodingError::CodeBlockDecodeFailure.into()), diff --git a/crates/j2k-native/src/tests/row_sink.rs b/crates/j2k-native/src/tests/row_sink.rs index cb5a8dbb..0a07ff72 100644 --- a/crates/j2k-native/src/tests/row_sink.rs +++ b/crates/j2k-native/src/tests/row_sink.rs @@ -1288,6 +1288,102 @@ fn reader_batches_of_tiles_narrower_than_the_image_stay_within_their_charge() { ); } +/// A serial region decode that runs out of budget must do so before its +/// first row, whatever the budget, so that the retries without retained +/// scratch or the packet directory can still rerun it. A region that clips +/// the code-blocks of its last rows decodes them only after its first rows +/// went out, so the scratch an HTJ2K block clipped this way decodes into must +/// be held, and charged, from the start. Budgets just past the largest one +/// that fits can fit again where they leave out the optional row index, so +/// each one either decodes every sample once or fails before any. +#[test] +fn streamed_rows_run_out_of_budget_before_their_first_row() { + for use_ht_block_coding in [false, true] { + let pixels = noisy_pixels(3); + let bytes = encode( + &pixels, + WIDTH, + HEIGHT, + 3, + 8, + false, + &EncodeOptions { + use_ht_block_coding, + num_decomposition_levels: 3, + code_block_height_exp: 1, + ..EncodeOptions::default() + }, + ) + .expect("encode budget fixture"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("parse fixture"); + for roi in [ + (37, 21, 130, 101), + (0, 0, WIDTH, 61), + (0, 0, WIDTH, HEIGHT - 11), + ] { + let label = format!("ht={use_ht_block_coding} {roi:?}"); + let decode = |baseline: usize| { + let mut context = DecoderContext::default(); + context.set_cpu_decode_parallelism(CpuDecodeParallelism::Serial); + let mut sink = CollectingSink::new(roi.2, roi.3); + let result = image.run_decode_rows( + &mut context, + Some(roi), + None, + true, + baseline, + Some(&mut sink), + ); + (result, sink) + }; + let (mut fits, mut fails) = (0, crate::DEFAULT_MAX_DECODE_BYTES); + while fails - fits > 1 { + let candidate = fits + (fails - fits) / 2; + match decode(candidate).0 { + Ok(_) => fits = candidate, + Err(_) => fails = candidate, + } + } + let mut failures = 0; + for over in [ + 0, + 1, + 2, + 3, + 5, + 9, + 17, + 65, + 257, + 1 << 10, + 1 << 12, + 1 << 14, + 1 << 16, + ] { + let (result, sink) = decode(fits + over); + match result { + Ok(_) => assert!( + sink.arrivals.iter().all(|&count| count == 1), + "{label} +{over}: every sample once" + ), + Err(error) => { + assert!( + crate::j2c::is_capacity_error(&error), + "{label} +{over}: {error:?}" + ); + assert!( + sink.arrivals.iter().all(|&count| count == 0), + "{label} +{over}: rows before the failure" + ); + failures += 1; + } + } + } + assert!(failures > 0, "{label}: some budget fails"); + } + } +} + /// Stitched windows of a reduced resolution, as a tiled viewer or an IIIF /// server requests them, hold few rows of code-blocks per component: fewer /// than a pool of 18 threads has workers. Such a window's streamed rows must From f5dfdee1408b4e1c1dd9f2a05cdd4d57e6d0a2aa Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 04:58:31 -0500 Subject: [PATCH 87/94] docs(changelog): note the streamed accounting fixes --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9000671b..fd2f5269 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,19 @@ Staged workspace version: `0.13.2`. status, detail and batch source index. CUDA keeps its typed source and job attribution. Unsupported statuses, raw runtime failures and combined cleanup or release failures keep their existing classes. +- Streamed decodes keep to the memory they charge. A tile whose components + reconstruct on readers of their own sized its batch of rows by the + region's width while the decode charged the tile's width, so a tile just + narrower than its region could hold twice its charge (an 8,192-wide tile + of an 8,200-wide image: 6.4 M against 3.4 M samples); its batch now keeps + to the charge, and such an image decodes in 0.98 of the time on 12 and 18 + threads, with 60.7 MiB in place of 72.6 MiB on 12. A serial HTJ2K region + decode allocated the scratch for code-blocks its region clips when it met + the first one, so a decode short of budget could fail after its first rows + reached the sink, where the retries without retained scratch or without + the packet directory no longer run; the scratch is now sized with the rest + of the Tier-1 workspace before the first row. Output of decodes that + succeed is unchanged. - Images that a batch with an explicit worker count decodes together on its stream pool, when it plans as many images at once as the pool has threads and the pool has fewer than 12, hold two bands in every ring of From f8121cc4675bb81cd745e0d7e7950b443a996f3d Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 04:07:00 -0500 Subject: [PATCH 88/94] perf(j2k-native): lift the streamed IDWT's rows with the banded schedule The row-streamed IDWT ran every vertical lifting step on one row at a time, each step its own pass over the row and its two neighbours. The banded IDWT and the planes route's fused last level run all the steps of a row's arrival in one sweep with the rows in registers (`lift_arrivals`). The streamed IDWT now lifts every level through that kernel, one arrival at a time, on the ring of rows each level already holds (a row per step and two more), so its memory is unchanged and both routes share one lifting kernel; `lift_one_row` and the per-step schedule go. A level hands a row over once its last step has run, which is when the per-step schedule finished it, so a decode reads the same band rows before each row it returns as before. Output is bit-identical. On an 18-core ARM64 machine, timers in the decode, a 5428 x 4460 RGB image at 0.5 coded bits per sample on 1 thread: last-level lifting 13.8 -> 8.9 ms, coarser levels 9.2 -> 7.9 ms; an 18000 x 12000 HTJ2K image: 151 -> 106 and 81 -> 70 ms. --- crates/j2k-native/src/j2c/idwt/banded.rs | 62 ++++- .../src/j2c/idwt/banded/last_level.rs | 4 +- crates/j2k-native/src/j2c/idwt/lines.rs | 220 ++++++++++++------ crates/j2k-native/src/j2c/idwt/tests.rs | 26 ++- crates/j2k-native/src/j2c/idwt/vertical.rs | 5 - 5 files changed, 226 insertions(+), 91 deletions(-) diff --git a/crates/j2k-native/src/j2c/idwt/banded.rs b/crates/j2k-native/src/j2c/idwt/banded.rs index 2b3fc0c3..3906251e 100644 --- a/crates/j2k-native/src/j2c/idwt/banded.rs +++ b/crates/j2k-native/src/j2c/idwt/banded.rs @@ -305,6 +305,14 @@ impl<'a> Level2d<'a> { } } + fn lifting(&self) -> Lifting { + Lifting { + transform: self.transform, + y0: self.y0, + height: self.height, + } + } + fn row_geometry(&self) -> RowGeometry { RowGeometry { width: self.width, @@ -328,6 +336,15 @@ impl<'a> Level2d<'a> { } } +/// What the vertical lifting of a level needs from it: its transform, its +/// first row, whose parity says which rows each step updates, and its height. +#[derive(Clone, Copy)] +pub(super) struct Lifting { + pub(super) transform: WaveletTransform, + pub(super) y0: usize, + pub(super) height: usize, +} + /// What the horizontal synthesis of one row needs from its level: the row's /// width, its first column, whose parity says whether it starts low-pass, the /// transform, and the 9-7I high-pass scale. @@ -667,7 +684,7 @@ fn pair_strip( worker_scratch: &mut [f32], rows_len: usize, ) { - let (_, count) = vertical_steps(outer); + let (_, count) = vertical_steps(outer.lifting()); let end = first_row + output.len() / outer.width; // The arrivals of `stream_strip` and the LL band rows they read. let lo = first_row.saturating_sub(count); @@ -700,7 +717,7 @@ enum VerticalStep { /// The vertical lifting steps of a level in order, each with the parity of /// the rows it updates, and how many there are. -fn vertical_steps(level: Level2d<'_>) -> ([(usize, VerticalStep); MAX_HALO_ROWS], usize) { +fn vertical_steps(level: Lifting) -> ([(usize, VerticalStep); MAX_HALO_ROWS], usize) { let low = level.y0 % 2; let high = 1 - low; match level.transform { @@ -861,7 +878,7 @@ fn stream_level( ) { let width = level.width; let height = level.height; - let (_, count) = vertical_steps(level); + let (_, count) = vertical_steps(level.lifting()); let (low, high) = split_horizontal_scratch(level, horizontal_scratch); let mut rows = RingRows { rows: target, @@ -874,7 +891,7 @@ fn stream_level( for arrival in arrivals.start..arrivals.end.min(height) { synthesize_level_row(simd, level, arrival, rows.row_mut(arrival), low, high); } - lift_arrivals(simd, level, arrivals, 0..height, &mut rows); + lift_arrivals(simd, level.lifting(), arrivals, 0..height, &mut rows); } } @@ -899,7 +916,7 @@ fn stream_strip( ) { let width = level.width; let height = level.height; - let (_, count) = vertical_steps(level); + let (_, count) = vertical_steps(level.lifting()); let end = first_row + output.len() / width; let lo = first_row.saturating_sub(count); let hi = (end + count).min(height); @@ -918,7 +935,7 @@ fn stream_strip( for arrival in arrivals.start..arrivals.end.min(hi) { synthesize_level_row(simd, level, arrival, rows.row_mut(arrival), low, high); } - lift_arrivals(simd, level, arrivals, lo..hi, &mut rows); + lift_arrivals(simd, level.lifting(), arrivals, lo..hi, &mut rows); } } @@ -1110,6 +1127,33 @@ trait LevelRows { fn window(&mut self, first: usize) -> Option<[&mut [f32]; ROWS]>; } +/// Run the vertical lifting steps of `arrivals` (see [`lift_arrivals`]) on +/// every row of a level `width` samples wide whose rows live in `ring`, row +/// `n` in slot `n % (ring.len() / width)`, at the CPU's SIMD level, for +/// callers outside a dispatch. The ring must hold every row the arrivals' +/// steps read (see [`lift_arrivals`]). +pub(super) fn lift_ring_arrivals( + level: Lifting, + width: usize, + ring: &mut [f32], + arrivals: Range, +) { + let slots = ring.len().checked_div(width).unwrap_or(0); + let mut rows = RingRows { + rows: ring, + width, + stride: width, + slots, + }; + dispatch!(Level::new(), simd => lift_arrivals( + simd, + level, + arrivals, + 0..level.height, + &mut rows, + )); +} + /// Arrivals [`stream_level`] and [`stream_strip`] synthesize before they run /// those arrivals' vertical steps, so one sweep of the columns runs the /// steps of several wavefronts. @@ -1136,7 +1180,7 @@ const SWEEP_WAVEFRONTS: usize = 2; #[inline(always)] fn lift_arrivals( simd: S, - level: Level2d<'_>, + level: Lifting, arrivals: Range, live: Range, rows: &mut impl LevelRows, @@ -2365,7 +2409,7 @@ mod tests { for (width, height) in [(1, 9), (7, 12), (8, 13), (9, 40), (17, 31), (33, 70)] { let rect = IntRect::from_xywh(3, y0, width, height); let level = Level2d::new(rect, [&[]; 4], transform, 1.25); - let (steps, count) = vertical_steps(level); + let (steps, count) = vertical_steps(level.lifting()); let height = height as usize; let width = width as usize; // The whole level, and strip-like windows with halos. @@ -2401,7 +2445,7 @@ mod tests { let arrivals = first..(first + batch).min(live.end + count); dispatch!(Level::new(), simd => lift_arrivals( simd, - level, + level.lifting(), arrivals.clone(), live.clone(), &mut rows, diff --git a/crates/j2k-native/src/j2c/idwt/banded/last_level.rs b/crates/j2k-native/src/j2c/idwt/banded/last_level.rs index 17566993..27fd87f9 100644 --- a/crates/j2k-native/src/j2c/idwt/banded/last_level.rs +++ b/crates/j2k-native/src/j2c/idwt/banded/last_level.rs @@ -314,7 +314,7 @@ fn stream_rows( let height = levels[0].height; let row_bytes = 3 * width; let end = first_row + output.len() / row_bytes; - let (_, count) = vertical_steps(levels[0]); + let (_, count) = vertical_steps(levels[0].lifting()); let lo = first_row.saturating_sub(count); let hi = (end + count).min(height); let ring_rows = ring_rows(batch); @@ -378,7 +378,7 @@ fn stream_rows( high, ); } - lift_arrivals(simd, *level, ahead, lo..hi, &mut rows); + lift_arrivals(simd, level.lifting(), ahead, lo..hi, &mut rows); } } emit_rows(rings, index, emitted..unread.max(emitted)); diff --git a/crates/j2k-native/src/j2c/idwt/lines.rs b/crates/j2k-native/src/j2c/idwt/lines.rs index 6026b324..c7186dff 100644 --- a/crates/j2k-native/src/j2c/idwt/lines.rs +++ b/crates/j2k-native/src/j2c/idwt/lines.rs @@ -8,12 +8,12 @@ //! window's rows top to bottom, pulling the LL rows it needs from the level //! below, and the caller receives the last level's rows one at a time. //! -//! Vertical lifting is streamed by keeping, for each step, how many rows the -//! step has finished. A step updates the rows of one parity from the rows -//! above and below them as the previous step left them. It can finish a row -//! once the previous step has finished every row it reads, and the next step -//! reaches a row only after this one has finished both of its neighbours, so -//! every read sees the value the whole-window filter would see. +//! Vertical lifting is streamed with the schedule of the banded IDWT (see +//! `banded::lift_arrivals`): when a level's row `n` arrives, step `k` +//! (1-based) runs on row `n - k` if that row has the step's parity. Each +//! step then reads its row's neighbours exactly as the previous step left +//! them, so every read sees the value the whole-window filter would see, and +//! a row is final as soon as the inputs its last step needs have arrived. use alloc::vec::Vec; @@ -21,21 +21,19 @@ use super::super::codestream::WaveletTransform; use super::super::decode::DecompositionStorage; use super::super::rect::IntRect; use super::banded::{ - horizontal_scratch_elements, synthesize_band_row, RowGeometry, MIN_SIMD_ROW_WIDTH, + horizontal_scratch_elements, lift_ring_arrivals, synthesize_band_row, Lifting, RowGeometry, + MIN_SIMD_ROW_WIDTH, }; -use super::filter_common::periodic_symmetric_extension_left; use super::horizontal::filter_row; use super::model::CoefficientSource; use super::roi::interleave_samples_roi; -use super::vertical::{ - irreversible_row_scale, lift_one_row, scale_one_row, split_rows, vertical_step_count, - vertical_steps, VerticalStep, -}; +use super::vertical::{irreversible_row_scale, scale_one_row, vertical_step_count, vertical_steps}; use crate::error::{DecodingError, ValidationError}; use crate::Result; -/// Rows each level keeps: one per lifting step, the row above the oldest -/// unfinished one, and the newest row. +/// Rows each level keeps: the rows an arrival's steps read, one per lifting +/// step above the arrival, its own row and the row above the deepest step's +/// (see `banded::lift_arrivals`). pub(crate) const fn ring_rows(transform: WaveletTransform) -> usize { vertical_step_count(transform) + 2 } @@ -118,8 +116,9 @@ struct LevelRows<'r> { ring: &'r mut [f32], /// Rows interleaved, filtered horizontally and scaled. fed: usize, - /// Rows each vertical step has finished. - finished: [usize; 4], + /// Arrivals whose vertical steps have run: one per fed row, then the + /// arrivals past the last row that finish the rows above it. + arrived: usize, } impl LevelRows<'_> { @@ -140,13 +139,47 @@ impl LevelRows<'_> { &self.ring[self.slot(row) * width..][..width] } - /// Rows that are final. + /// Rows that are final. Row `r` takes its last step at arrival `r + k`, + /// where `k` is that step: the last of the transform's steps for the rows + /// that step updates, the one before it for the others. fn done(&self, transform: WaveletTransform) -> usize { - if self.height() == 1 { - self.fed - } else { - self.finished[vertical_step_count(transform) - 1] + let height = self.height(); + if height == 1 { + return self.fed; + } + let Some(last_arrival) = self.arrived.checked_sub(1) else { + return 0; + }; + let steps = vertical_step_count(transform); + let (_, last_step_even) = vertical_steps(transform, self.window.y0)[steps - 1]; + let final_row = |row: usize| { + let last_step = if row.is_multiple_of(2) == last_step_even { + steps + } else { + steps - 1 + }; + row + last_step <= last_arrival + }; + // Every row up to `steps` above the last arrival is final, and the + // next one when its last step is the one before the transform's last. + let mut done = (last_arrival + 1).saturating_sub(steps).min(height); + while done < height && final_row(done) { + done += 1; } + done + } + + /// The vertical steps of this level's next arrival, which must not have + /// to wait for a row: every row is fed, or the arrival's row is. + fn lift_next_arrival(&mut self, transform: WaveletTransform) { + let (width, arrival) = (self.width(), self.arrived); + let level = Lifting { + transform, + y0: self.window.y0 as usize, + height: self.height(), + }; + lift_ring_arrivals(level, width, self.ring, arrival..arrival + 1); + self.arrived += 1; } } @@ -237,7 +270,7 @@ impl<'r> LineReconstruction<'r> { sub_bands: *sub_bands, ring: level_ring, fed: 0, - finished: [0; 4], + arrived: 0, }); } let widest = levels.iter().map(LevelRows::width).max().unwrap_or(0); @@ -308,18 +341,25 @@ fn finish_row( inputs: &mut Inputs<'_>, ) -> Result<()> { let ring_rows = ring_rows(inputs.transform); + let steps = vertical_step_count(inputs.transform); while levels[level].done(inputs.transform) <= row { let current = &levels[level]; - // Refilling a slot drops the row `ring_rows` above the new one, - // which no unfinished step may still read. - let oldest_read = current.done(inputs.transform).saturating_sub(1); - if current.fed >= current.height() - || (current.fed >= ring_rows && current.fed - ring_rows >= oldest_read) - { + let height = current.height(); + if current.fed < height { + // Refilling a slot drops the row `ring_rows` above the new one, + // which the previous arrival's deepest step read last: the ring + // holds a row per step and two more. + if current.arrived != current.fed { + return Err(DecodingError::CodeBlockDecodeFailure.into()); + } + feed_row(levels, level, inputs)?; + if height == 1 { + continue; + } + } else if height == 1 || current.arrived >= height + steps { return Err(DecodingError::CodeBlockDecodeFailure.into()); } - feed_row(levels, level, inputs)?; - advance_steps(&mut levels[level], inputs.transform); + levels[level].lift_next_arrival(inputs.transform); } if levels[level].fed > row + ring_rows { // The requested row has already left the ring. @@ -489,53 +529,91 @@ fn stored_row( source.coefficients.get(start..start.checked_add(len)?) } -/// Let every vertical step finish as many rows as the rows fed so far allow. -fn advance_steps(level: &mut LevelRows<'_>, transform: WaveletTransform) { - let height = level.height(); - if height == 1 { - return; - } - let steps = vertical_steps(transform, level.window.y0); - for (index, &(step, even_offsets)) in steps - .iter() - .take(vertical_step_count(transform)) - .enumerate() - { - let available = if index == 0 { - level.fed - } else { - level.finished[index - 1] - }; - while level.finished[index] < height { - let row = level.finished[index]; - if row.is_multiple_of(2) == even_offsets { - let above = periodic_symmetric_extension_left(row, 1); - let below = if row + 1 < height { - row + 1 +#[cfg(test)] +mod tests { + use super::super::filter_common::periodic_symmetric_extension_left; + use super::*; + + /// The rows final once `fed` rows of a level have arrived when every + /// step finishes, in order, each row whose inputs the previous step has + /// finished: the earliest a row can be final, which the arrival schedule + /// must match so that no row reaches the output later than that. + fn earliest_final_rows( + transform: WaveletTransform, + y0: u32, + height: usize, + fed: usize, + ) -> usize { + if height == 1 { + return fed; + } + let steps = vertical_step_count(transform); + let mut finished = [0; 4]; + for (index, &(_, even_offsets)) in vertical_steps(transform, y0)[..steps].iter().enumerate() + { + let available = if index == 0 { fed } else { finished[index - 1] }; + while finished[index] < height { + let row = finished[index]; + let ready = if row.is_multiple_of(2) == even_offsets { + let above = periodic_symmetric_extension_left(row, 1); + let below = if row + 1 < height { + row + 1 + } else { + height - 2 + }; + row.max(above).max(below) < available } else { - height - 2 + row < available }; - if row.max(above).max(below) >= available { + if !ready { break; } - lift_level_row(level, step, row, above, below); - } else if row >= available { - break; + finished[index] += 1; } - level.finished[index] += 1; } + finished[steps - 1] } -} -fn lift_level_row( - level: &mut LevelRows<'_>, - step: VerticalStep, - row: usize, - above: usize, - below: usize, -) { - let width = level.width(); - let (target, above, below) = (level.slot(row), level.slot(above), level.slot(below)); - let (row_samples, above, below) = split_rows(level.ring, width, target, above, below); - lift_one_row(step, row_samples, above, below); + /// A level hands each row over after exactly the rows the earliest + /// schedule needs: after `fed` rows, or once every row has arrived and + /// the arrivals past the last one ran, its final rows are those. + #[test] + fn rows_are_final_after_the_rows_their_steps_need() { + for transform in [ + WaveletTransform::Reversible53, + WaveletTransform::Irreversible97, + ] { + let steps = vertical_step_count(transform); + for y0 in [0, 1, 2, 7] { + for height in 1..=24 { + let mut ring = []; + let mut level = LevelRows { + window: IntRect::from_xywh( + 0, + y0, + 1, + u32::try_from(height).expect("height"), + ), + sub_bands: [0; 3], + ring: &mut ring, + fed: 0, + arrived: 0, + }; + for fed in 0..=height { + level.fed = fed; + level.arrived = if fed == height && height > 1 { + height + steps + } else { + fed + }; + assert_eq!( + level.done(transform), + earliest_final_rows(transform, y0, height, fed), + "{transform:?} y0={y0} height={height} fed={fed}" + ); + } + } + } + } + } } diff --git a/crates/j2k-native/src/j2c/idwt/tests.rs b/crates/j2k-native/src/j2c/idwt/tests.rs index 86e7d8fe..d8b30a17 100644 --- a/crates/j2k-native/src/j2c/idwt/tests.rs +++ b/crates/j2k-native/src/j2c/idwt/tests.rs @@ -686,9 +686,10 @@ fn whole_window_level( /// otherwise through the windowed interleave. Every row must match the /// whole-window region IDWT, which reads every sample through /// `CoefficientSource::get` and filters with the scalar `filter_row`, bit for -/// bit: both transforms, both column and row parities, stored rectangles -/// wider than the window's bands and clipped on either side, sub-bands with -/// nothing stored, and a second level reading the first one's rows. +/// bit: both transforms, both column and row parities, every height up to a +/// few wraps of the level's ring, stored rectangles wider than the window's +/// bands and clipped on either side, sub-bands with nothing stored, and a +/// second level reading the first one's rows. #[test] #[expect(clippy::too_many_lines, reason = "one table-driven equivalence sweep")] fn streamed_rows_match_the_whole_window_levels_bit_for_bit() { @@ -702,7 +703,24 @@ fn streamed_rows_match_the_whole_window_levels_bit_for_bit() { ] { let integral = transform == WaveletTransform::Reversible53; for (x0, y0) in [(0, 0), (1, 0), (0, 1), (1, 1), (5, 2), (8, 3)] { - for (width, height) in [(16, 9), (17, 20), (31, 2), (40, 33), (64, 12), (161, 40)] { + for (width, height) in [ + (16, 9), + (17, 20), + (31, 2), + (40, 33), + (64, 12), + (161, 40), + // Every height up to a few ring wraps, around the SIMD width. + (15, 1), + (16, 3), + (17, 4), + (24, 5), + (32, 6), + (33, 7), + (8, 8), + (47, 10), + (49, 11), + ] { for clip in [ [0, 0, 0, 0], [2, 1, 3, 2], diff --git a/crates/j2k-native/src/j2c/idwt/vertical.rs b/crates/j2k-native/src/j2c/idwt/vertical.rs index e881c176..3dd32d6a 100644 --- a/crates/j2k-native/src/j2c/idwt/vertical.rs +++ b/crates/j2k-native/src/j2c/idwt/vertical.rs @@ -72,11 +72,6 @@ pub(super) fn irreversible_row_scale(y0: u32, even_offset: bool, high_pass: f32) } } -/// [`lift_row`] at the CPU's SIMD level, for callers outside a dispatch. -pub(super) fn lift_one_row(step: VerticalStep, row: &mut [f32], above: &[f32], below: &[f32]) { - dispatch!(Level::new(), simd => lift_row(simd, step, row, above, below)); -} - /// [`scale_row`] at the CPU's SIMD level, for callers outside a dispatch. pub(super) fn scale_one_row(row: &mut [f32], factor: f32) { dispatch!(Level::new(), simd => scale_row(simd, row, factor)); From 528398660a1e175668932c905fd834bc06f9c714 Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 04:18:13 -0500 Subject: [PATCH 89/94] perf(j2k-native): finish streamed rows in place in the IDWT's ring A streamed decode on serial bands copied each finished row of every component into a scratch row, transformed the copy in place (the inverse MCT, the level shift and rounding) and packed it or handed it to the sink; only 9/7 rows with the MCT and an RGB8 destination skipped the copy through the fused ICT pack. The IDWT can't hand out its own rows for an in-place transform as soon as they are final, because the next row's last lifting step still reads them. `LineReconstruction::next_row_mut` hands a row over once no lifting step reads it any more, one arrival later than `next_row`, which the level's ring already holds, and `RowOutput::write` transforms and packs each row there: no scratch rows (their buffer and charge go) and one sweep less for every transform. The per-row RGB8 finish of three components with the MCT (`DeferredMct::pack_rgb8_row`: the fused ICT pack where it exists, else the transform and the standard pack) is now shared by the planes' fused last level, the serial rows and the pool's row batches. Output is bit-identical; rows reach the sink one arrival later than before (the last one, which nothing reconstructs after, as before), which only changes how many rows a decode that fails partway leaves written. Each row of every component is taken final (`finish_next_row`) before any takes the arrival that frees it, so the components ask for their band rows, and decode their bands, in the order they did: a tile with failing code-blocks meets the same one first and reports the same error. On a 12-core ARM64 machine, 7 interleaved rounds, against the planes route of the same build: a 4096 x 2048 lossless HTJ2K image on 1 thread 1.020 -> 0.972 of the planes' time. --- .../j2k-native/src/j2c/decode/banded_tier1.rs | 133 +++++-------- crates/j2k-native/src/j2c/decode/state.rs | 4 +- crates/j2k-native/src/j2c/idwt/lines.rs | 186 +++++++++++++++++- crates/j2k-native/src/j2c/idwt/roi.rs | 10 +- crates/j2k-native/src/j2c/idwt/tests.rs | 171 +++++++++++++++- crates/j2k-native/src/j2c/mct.rs | 35 +++- .../j2k-native/src/tests/fused_last_level.rs | 130 ++++++++++++ crates/j2k-native/src/tests/row_sink.rs | 10 +- 8 files changed, 575 insertions(+), 104 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode/banded_tier1.rs b/crates/j2k-native/src/j2c/decode/banded_tier1.rs index 67e4eb26..cdc3af31 100644 --- a/crates/j2k-native/src/j2c/decode/banded_tier1.rs +++ b/crates/j2k-native/src/j2c/decode/banded_tier1.rs @@ -160,7 +160,6 @@ fn stream_rows<'a>( profile_enabled: bool, tier1_us: &mut u128, ) -> Result<()> { - let components = tile.component_infos.len(); let mut bands = Vec::new(); let line_plans = plan_tile_lines(tile, header, tile_ctx, storage, &mut bands)?; let band_of = band_lookup(&bands)?; @@ -168,25 +167,20 @@ fn stream_rows<'a>( let (copy, region) = line_plans .first() .map_or((empty, empty), |plan| (plan.copy, plan.region)); - let scratch_len = (copy.width() as usize) - .checked_mul(components) - .ok_or(ValidationError::ImageTooLarge)?; // The index first: once the ring buffer is out of `tile_ctx`, every // error path below must put it back. let (precinct_rows, index_bytes) = index_bands(storage, tile_ctx, &bands)?; - let (mut buffer, ring_lens) = - match take_line_buffer(tile_ctx, storage, &line_plans, scratch_len) { - Ok(taken) => taken, - Err(error) => { - storage.structural_workspace_bytes -= index_bytes; - return Err(error); - } - }; + let (mut buffer, ring_lens) = match take_line_buffer(tile_ctx, storage, &line_plans, 0) { + Ok(taken) => taken, + Err(error) => { + storage.structural_workspace_bytes -= index_bytes; + return Err(error); + } + }; let mut output = RowOutput::new(tile, sink, tile_ctx, copy, region); let mut output_samples = 0usize; let result = (|| { - let rings_len = buffer.len() - scratch_len; - let (mut rings, scratch) = buffer.split_at_mut(rings_len); + let mut rings = &mut buffer[..]; let mut lines = Vec::new(); lines .try_reserve_exact(line_plans.len()) @@ -208,7 +202,7 @@ fn stream_rows<'a>( tier1_us: 0, }; let result = output - .write(&mut lines, &mut rows, scratch) + .write(&mut lines, &mut rows) .and_then(|()| rows.decode_remaining_bands()); *tier1_us = rows.tier1_us; output_samples = lines.iter().map(LineReconstruction::output_samples).sum(); @@ -315,8 +309,8 @@ pub(super) struct RowOutput<'o, 't> { /// The rows, in the coordinates of `region`, that the tile reconstructs. copy: IntRect, region: IntRect, - /// The inverse MCT of three 8-bit components, which the RGB8 pack can - /// take over (see [`Self::fused_rgb8_pack`]). + /// The inverse MCT of three 8-bit components, which finishes their rows + /// for an RGB8 destination (see [`DeferredMct::pack_rgb8_row`]). mct: Option, /// Samples handed to the sink. pub(super) streamed: usize, @@ -356,13 +350,6 @@ impl<'o, 't> RowOutput<'o, 't> { } } - /// The inverse MCT when the RGB8 pack fuses it (see - /// [`DeferredMct::fuses_rgb8_pack`]): rows then go to the pack as the - /// IDWT left them, never transformed in place. - fn fused_rgb8_pack(&self) -> Option { - self.mct.filter(|mct| mct.fuses_rgb8_pack()) - } - /// The sink's RGB8 rows for `rows` rows of the tile from row `y`, when /// it has a destination for three 8-bit components. fn rgb8_rows(&mut self, y: u32, rows: usize) -> Result>> { @@ -388,77 +375,62 @@ impl<'o, 't> RowOutput<'o, 't> { })) } - /// Advance every component's reconstruction a row at a time, copying - /// each row into `scratch`, one row per component, and hand them over: - /// packed into the sink's RGB8 rows when it has them, through - /// [`DeferredMct::pack_ict_rgb8_rows`] straight from the IDWT's rows - /// where that pack transforms them, else through `write_rows`. + /// Advance every component's reconstruction a row at a time and finish + /// each row in the IDWT's ring once no lifting step reads it: transformed + /// in place and packed into the sink's RGB8 rows when it has them, the + /// inverse MCT fused into the pack where [`DeferredMct::pack_rgb8_row`] + /// fuses it, else handed to `write_rows`. pub(super) fn write( &mut self, lines: &mut [LineReconstruction<'_>], rows: &mut dyn SubBandRows, - scratch: &mut [f32], ) -> Result<()> { let (copy, components) = (self.copy, lines.len()); let width = copy.width() as usize; if lines.is_empty() || width == 0 { return Ok(()); } - // With an RGB8 destination, rows go from the IDWT's rings straight - // to the pack, which transforms them on the way. - let fused = self - .fused_rgb8_pack() - .filter(|_| matches!(self.rgb8_rows(copy.y0, 0), Ok(Some(_)))); + let (tile, round_irreversible_output, mct) = + (self.tile, self.round_irreversible_output, self.mct); for y in copy.y0..copy.y1 { - if let Some(mct) = fused { - let mut samples: [&[f32]; 3] = [&[]; 3]; - for (line, row) in lines.iter_mut().zip(samples.iter_mut()) { - let (row_y, line_row) = line - .next_row(rows)? - .ok_or(DecodingError::CodeBlockDecodeFailure)?; - if row_y != y || line_row.len() != width { - return Err(DecodingError::CodeBlockDecodeFailure.into()); - } - *row = line_row; - } - let rgb8 = self - .rgb8_rows(y, 1)? - .ok_or(DecodingError::CodeBlockDecodeFailure)?; - let pixels = rgb8 - .out - .get_mut(rgb8.x..rgb8.x + width * 3) - .ok_or(DecodingError::CodeBlockDecodeFailure)?; - mct.pack_ict_rgb8_rows(samples, pixels); - self.streamed = self.streamed.saturating_add(width * components); - continue; + // Every component's row final before any takes the arrival that + // frees it, so the bands decode in the order rows handed over + // final read them, and a failing tile meets the same band first. + for line in lines.iter_mut() { + line.finish_next_row(rows)?; } let mut outputs: [&mut [f32]; 4] = Default::default(); - for ((line, output), chunk) in lines - .iter_mut() - .zip(outputs.iter_mut()) - .zip(scratch.chunks_exact_mut(width)) - { + for (line, output) in lines.iter_mut().zip(outputs.iter_mut()) { let (row_y, row) = line - .next_row(rows)? + .next_row_mut(rows)? .ok_or(DecodingError::CodeBlockDecodeFailure)?; if row_y != y || row.len() != width { return Err(DecodingError::CodeBlockDecodeFailure.into()); } - chunk.copy_from_slice(row); - *output = chunk; + *output = row; } let outputs = &mut outputs[..components]; - store::transform_streamed_rows(self.tile, self.round_irreversible_output, outputs)?; if let Some(rgb8) = self.rgb8_rows(y, 1)? { let pixels = rgb8 .out .get_mut(rgb8.x..rgb8.x + width * 3) .ok_or(DecodingError::CodeBlockDecodeFailure)?; - let [red, green, blue] = [&*outputs[0], &*outputs[1], &*outputs[2]]; - crate::color::pack_rgb8([red, green, blue], pixels); + match (mct, outputs) { + (Some(mct), [red, green, blue]) => { + mct.pack_rgb8_row([red, green, blue], pixels, crate::color::pack_rgb8); + } + (_, outputs) => { + store::transform_streamed_rows(tile, round_irreversible_output, outputs)?; + let [red, green, blue, ..] = outputs else { + return Err(DecodingError::CodeBlockDecodeFailure.into()); + }; + crate::color::pack_rgb8([red, green, blue], pixels); + } + } self.streamed = self.streamed.saturating_add(width * components); continue; } + store::transform_streamed_rows(tile, round_irreversible_output, outputs)?; let component_rows: [ComponentPlane<'_>; 4] = core::array::from_fn(|index| ComponentPlane { samples: outputs.get(index).map_or(&[][..], |row| &**row), @@ -586,10 +558,10 @@ impl RowOutput<'_, '_> { ) -> Result<()> { let width = self.copy.width() as usize; let mut parts = batch_parts(buffer, batch_len, rows * width, components)?; - let fused = self.fused_rgb8_pack(); + let mct = self.mct; let destination = self.rgb8_rows(y, rows)?; let packed = destination.is_some(); - let fused = fused.filter(|_| packed); + let mct = mct.filter(|_| packed); match alongside { Some(alongside) => { let mut finished = Ok(()); @@ -597,7 +569,7 @@ impl RowOutput<'_, '_> { overlap( || { finished = finish_rows( - (transform, fused), + (transform, mct), &mut parts[..components], width, destination, @@ -610,7 +582,7 @@ impl RowOutput<'_, '_> { finished?; } None => finish_rows( - (transform, fused), + (transform, mct), &mut parts[..components], width, destination, @@ -817,27 +789,28 @@ struct BatchRows<'p, 'o> { )] impl BatchRows<'_, '_> { /// Transform the rows one at a time, as [`RowOutput::write`] does, then - /// pack them into the RGB8 rows if there are any; with a `fused` MCT, - /// the pack transforms each row as it reads it instead. + /// pack them into the RGB8 rows if there are any; with the `mct` of + /// three components and RGB8 rows, each row finishes through + /// [`DeferredMct::pack_rgb8_row`] instead. fn finish( mut self, - ((tile, round_irreversible_output), fused): ((&Tile<'_>, bool), Option), + ((tile, round_irreversible_output), mct): ((&Tile<'_>, bool), Option), components: usize, width: usize, ) -> Result<()> { let rows = self.parts[0].len().checked_div(width).unwrap_or(0); - if let (Some(mct), Some(rgb8)) = (fused, self.rgb8.as_mut()) { - let [red, green, blue, _] = &self.parts; + if let (Some(mct), Some(rgb8)) = (mct, self.rgb8.as_mut()) { + let [red, green, blue, _] = &mut self.parts; let samples = red - .chunks_exact(width) - .zip(green.chunks_exact(width)) - .zip(blue.chunks_exact(width)); + .chunks_exact_mut(width) + .zip(green.chunks_exact_mut(width)) + .zip(blue.chunks_exact_mut(width)); for (line, ((red, green), blue)) in rgb8.out.chunks_exact_mut(rgb8.stride).zip(samples) { let pixels = line .get_mut(rgb8.x..rgb8.x + width * 3) .ok_or(DecodingError::CodeBlockDecodeFailure)?; - mct.pack_ict_rgb8_rows([red, green, blue], pixels); + mct.pack_rgb8_row([red, green, blue], pixels, crate::color::pack_rgb8); } return Ok(()); } diff --git a/crates/j2k-native/src/j2c/decode/state.rs b/crates/j2k-native/src/j2c/decode/state.rs index 6e2e333b..9025a4d2 100644 --- a/crates/j2k-native/src/j2c/decode/state.rs +++ b/crates/j2k-native/src/j2c/decode/state.rs @@ -140,8 +140,8 @@ pub(crate) struct DecompositionStorage<'a> { /// them through one reader whatever its pool's workers (see /// [`parallel_bands::one_reader`]), which holds one short batch of rows. pub(crate) one_synthesis_reader: bool, - /// Whether every component's IDWT rings, and a scratch row each, are - /// live at once, because the tile streams its rows to a sink. + /// Whether every component's IDWT rings are live at once, because the + /// tile streams its rows to a sink, finishing them in the rings. pub(crate) lockstep_lines: bool, /// The f32 samples of IDWT workspace the ROI plan charged for this tile. pub(crate) roi_idwt_charge_elements: usize, diff --git a/crates/j2k-native/src/j2c/idwt/lines.rs b/crates/j2k-native/src/j2c/idwt/lines.rs index c7186dff..fe12c13c 100644 --- a/crates/j2k-native/src/j2c/idwt/lines.rs +++ b/crates/j2k-native/src/j2c/idwt/lines.rs @@ -139,6 +139,12 @@ impl LevelRows<'_> { &self.ring[self.slot(row) * width..][..width] } + fn row_mut(&mut self, row: usize) -> &mut [f32] { + let width = self.width(); + let slot = self.slot(row); + &mut self.ring[slot * width..][..width] + } + /// Rows that are final. Row `r` takes its last step at arrival `r + k`, /// where `k` is that step: the last of the transform's steps for the rows /// that step updates, the one before it for the others. @@ -169,6 +175,19 @@ impl LevelRows<'_> { done } + /// Rows that no vertical step reads any more: row `r` is last read by + /// the last step of row `r + 1` (row `r - 1`'s comes earlier), so it is + /// unread once the row below it is final, and the last row once every + /// row is. + fn unread(&self, transform: WaveletTransform) -> usize { + let done = self.done(transform); + if done == self.height() { + done + } else { + done.saturating_sub(1) + } + } + /// The vertical steps of this level's next arrival, which must not have /// to wait for a row: every row is fed, or the arrival's row is. fn lift_next_arrival(&mut self, transform: WaveletTransform) { @@ -312,6 +331,49 @@ impl<'r> LineReconstruction<'r> { let Some(y) = self.rows.next() else { return Ok(None); }; + let row = self.advance(rows, y, Ready::Final)?; + let last = self.levels.len() - 1; + Ok(Some((y, &self.levels[last].row(row)[self.columns.clone()]))) + } + + /// [`Self::next_row`], handed over once no vertical step reads the row + /// any more, so that the caller may transform it in place. That takes + /// the arrival of one more row than [`Self::next_row`] waits for, which + /// the ring holds; the last row needs none, since nothing reconstructs + /// after it, so the bands a decode reads stay those final rows read. + pub(crate) fn next_row_mut( + &mut self, + rows: &mut dyn SubBandRows, + ) -> Result> { + let Some(y) = self.rows.next() else { + return Ok(None); + }; + let ready = if self.rows.is_empty() { + Ready::Final + } else { + Ready::Unread + }; + let row = self.advance(rows, y, ready)?; + let last = self.levels.len() - 1; + let columns = self.columns.clone(); + Ok(Some((y, &mut self.levels[last].row_mut(row)[columns]))) + } + + /// Advance the last level until its next row is final, as + /// [`Self::next_row`] does, without handing the row over. A caller that + /// takes a row of several reconstructions with [`Self::next_row_mut`] + /// takes them all here first: their bands then decode in the order that + /// handing the rows over final reads them. + pub(crate) fn finish_next_row(&mut self, rows: &mut dyn SubBandRows) -> Result<()> { + self.rows + .clone() + .next() + .map_or(Ok(()), |y| self.advance(rows, y, Ready::Final).map(drop)) + } + + /// Advance the last level until its absolute row `y` is `ready`, + /// returning its row in the level's window. + fn advance(&mut self, rows: &mut dyn SubBandRows, y: u32, ready: Ready) -> Result { let last = self.levels.len() - 1; let row = (y - self.levels[last].window.y0) as usize; let mut inputs = Inputs { @@ -320,8 +382,8 @@ impl<'r> LineReconstruction<'r> { transform: self.transform, scratch: &mut *self.scratch, }; - finish_row(&mut self.levels, last, row, &mut inputs)?; - Ok(Some((y, &self.levels[last].row(row)[self.columns.clone()]))) + advance_until(&mut self.levels, last, row, &mut inputs, ready)?; + Ok(row) } /// The samples reconstructed so far, over every level. @@ -332,6 +394,15 @@ impl<'r> LineReconstruction<'r> { } } +/// How far a level's row must be along before it is handed over. +#[derive(Clone, Copy)] +enum Ready { + /// Its last vertical step has run (see [`LevelRows::done`]). + Final, + /// No vertical step reads it any more (see [`LevelRows::unread`]). + Unread, +} + /// Advance `levels[level]` until `row` of its window is final. Rows must be /// requested in increasing order. fn finish_row( @@ -339,10 +410,27 @@ fn finish_row( level: usize, row: usize, inputs: &mut Inputs<'_>, +) -> Result<()> { + advance_until(levels, level, row, inputs, Ready::Final) +} + +/// Advance `levels[level]` until `row` of its window is `ready`. Rows must +/// be requested in increasing order. +fn advance_until( + levels: &mut [LevelRows<'_>], + level: usize, + row: usize, + inputs: &mut Inputs<'_>, + ready: Ready, ) -> Result<()> { let ring_rows = ring_rows(inputs.transform); let steps = vertical_step_count(inputs.transform); - while levels[level].done(inputs.transform) <= row { + let transform = inputs.transform; + let ready_rows = |level: &LevelRows<'_>| match ready { + Ready::Final => level.done(transform), + Ready::Unread => level.unread(transform), + }; + while ready_rows(&levels[level]) <= row { let current = &levels[level]; let height = current.height(); if current.fed < height { @@ -574,6 +662,98 @@ mod tests { finished[steps - 1] } + /// The arrivals after which row `row` of a level is final, and after + /// which no vertical step reads it: the arrival of the last step that + /// updates it, and of the last step that updates a row whose mirrored + /// neighbours include it. + fn last_update_and_read( + transform: WaveletTransform, + y0: u32, + height: usize, + row: usize, + ) -> (usize, usize) { + let steps = vertical_step_count(transform); + let (mut update, mut read) = (0, 0); + for (index, &(_, even_offsets)) in vertical_steps(transform, y0)[..steps].iter().enumerate() + { + let arrival = |target: usize| target + index + 1; + for target in 0..height { + if target.is_multiple_of(2) != even_offsets { + continue; + } + let above = periodic_symmetric_extension_left(target, 1); + let below = if target + 1 < height { + target + 1 + } else { + height - 2 + }; + if target == row { + update = update.max(arrival(target)); + } + if above == row || below == row { + read = read.max(arrival(target)); + } + } + } + (update, read.max(update)) + } + + /// After every arrival, a level's final rows are the rows whose last + /// update has run, and its unread rows those no step will read again, + /// each counted from the first row while every row so far qualifies. + #[test] + fn final_and_unread_rows_follow_the_arrivals() { + for transform in [ + WaveletTransform::Reversible53, + WaveletTransform::Irreversible97, + ] { + let steps = vertical_step_count(transform); + for y0 in [0, 1, 4, 9] { + for height in 2..=20 { + let rows: Vec<(usize, usize)> = (0..height) + .map(|row| last_update_and_read(transform, y0, height, row)) + .collect(); + let mut ring = []; + let mut level = LevelRows { + window: IntRect::from_xywh( + 0, + y0, + 1, + u32::try_from(height).expect("height"), + ), + sub_bands: [0; 3], + ring: &mut ring, + fed: 0, + arrived: 0, + }; + let mut handed_over = 0; + for arrived in 0..=height + steps { + level.fed = arrived.min(height); + level.arrived = arrived; + let prefix = |past: fn(&(usize, usize)) -> usize| { + rows.iter().take_while(|row| past(row) < arrived).count() + }; + let label = + format!("{transform:?} y0={y0} height={height} arrived={arrived}"); + assert_eq!(level.done(transform), prefix(|row| row.0), "final {label}"); + assert_eq!( + level.unread(transform), + prefix(|row| row.1), + "unread {label}" + ); + // Every row this arrival makes unread is still in + // the ring, which holds the last `ring_rows` fed. + assert!( + handed_over + ring_rows(transform) >= level.fed, + "ring {label}" + ); + handed_over = level.unread(transform); + } + } + } + } + } + /// A level hands each row over after exactly the rows the earliest /// schedule needs: after `fed` rows, or once every row has arrived and /// the arrivals past the last one ran, its final rows are those. diff --git a/crates/j2k-native/src/j2c/idwt/roi.rs b/crates/j2k-native/src/j2c/idwt/roi.rs index 2b2ff075..cb382239 100644 --- a/crates/j2k-native/src/j2c/idwt/roi.rs +++ b/crates/j2k-native/src/j2c/idwt/roi.rs @@ -74,8 +74,8 @@ pub(crate) fn streamed_batch_rows( /// at the 9/7 ring depth whatever the component's transform. With no active /// level it holds nothing, since it copies the LL window straight into the /// plane. When the tile streams its rows to a sink, every component's rings -/// are live at once, with a scratch row each as wide as its final window, or -/// one or two batches of such rows from pool bands (see +/// are live at once, the rows finishing in place in them, with one or two +/// batches of rows as wide as each final window from pool bands (see /// [`streamed_batch_rows`]), so the peak is their sum. pub(crate) fn roi_idwt_peak_elements( storage: &DecompositionStorage<'_>, @@ -115,7 +115,7 @@ pub(crate) fn roi_idwt_peak_elements( .map_while(|decomposition_idx| roi_plan.idwt_window(decomposition_idx)); let rings = ring_samples(windows, WaveletTransform::Irreversible97)?; if storage.lockstep_lines { - let scratch_row = roi_plan.final_window(component_idx).map_or(0, |window| { + let batch_rows = roi_plan.final_window(component_idx).map_or(0, |window| { let (width, height) = (window.width() as usize, window.height() as usize); if storage.parallel_bands { // Two batches, when each component reconstructs @@ -137,12 +137,12 @@ pub(crate) fn roi_idwt_peak_elements( )) .saturating_mul(batches) } else { - width + 0 } }); lockstep = lockstep .checked_add(rings) - .and_then(|total| total.checked_add(scratch_row)) + .and_then(|total| total.checked_add(batch_rows)) .ok_or(ValidationError::ImageTooLarge)?; } else { peak = peak.max(rings); diff --git a/crates/j2k-native/src/j2c/idwt/tests.rs b/crates/j2k-native/src/j2c/idwt/tests.rs index d8b30a17..18269f5b 100644 --- a/crates/j2k-native/src/j2c/idwt/tests.rs +++ b/crates/j2k-native/src/j2c/idwt/tests.rs @@ -661,6 +661,139 @@ fn adjusted(rect: IntRect, [left, top, right, bottom]: [i64; 4]) -> IntRect { ) } +/// The rows `copy` of the streamed IDWT of `windows`, each taken through +/// `LineReconstruction::next_row_mut` and overwritten with NaN as soon as it +/// is read, so a row a later step still read would poison the rows after it. +fn streamed_rows_overwritten( + rows: &mut StoredBands, + sub_bands: &[[usize; 3]], + windows: &[IntRect], + transform: WaveletTransform, + copy: IntRect, +) -> Vec<(u32, Vec)> { + let mut ring = vec![ + f32::NAN; + super::lines::ring_samples(windows.iter().copied(), transform) + .expect("ring") + ]; + let mut lines = + super::lines::LineReconstruction::new(sub_bands, windows, 0, transform, &mut ring, copy) + .expect("plan the streamed rows"); + let mut out = Vec::new(); + while let Some((y, row)) = lines.next_row_mut(rows).expect("streamed row") { + out.push((y, bits(row))); + row.fill(f32::NAN); + } + out +} + +/// Band rows a reconstruction of one component asks for, logged in the +/// order every component asks. +struct LoggedBands<'a> { + bands: &'a StoredBands, + component: usize, + log: &'a core::cell::RefCell>, +} + +impl super::lines::SubBandRows for LoggedBands<'_> { + fn prepare_row(&mut self, sub_band: usize, y: u32) -> Result<()> { + self.log.borrow_mut().push((self.component, sub_band, y)); + Ok(()) + } + + fn samples(&self, sub_band: usize) -> (&[f32], IntRect, u32) { + let (samples, rect) = &self.bands.bands[sub_band]; + (samples, *rect, rect.width()) + } +} + +/// A streamed tile finishes each row of its components in place, which +/// needs one arrival more than a final row; taking every component's row +/// final first must keep the band rows they ask for, and so the bands they +/// decode, in the order that handing the rows over final asks for them: +/// over three components, both transforms, two levels, up to four skipped +/// first rows, and the window's bottom edge or rows short of it. +#[test] +fn rows_finished_in_place_ask_for_bands_as_final_rows_do() { + use super::banded::packed_band_rects; + use super::lines::{ring_samples, LineReconstruction}; + + for transform in [ + WaveletTransform::Reversible53, + WaveletTransform::Irreversible97, + ] { + for (x0, y0, width, height) in [(0, 0, 9, 7), (1, 1, 20, 13), (3, 2, 33, 21)] { + let first = IntRect::from_xywh(x0, y0, width, height); + let next = IntRect::from_xywh(2 * x0 + 1, 2 * y0, 2 * width - 1, 2 * height); + let mut seed = 11_u64; + let bands = StoredBands { + bands: packed_band_rects(first) + .into_iter() + .chain(packed_band_rects(next).into_iter().skip(1)) + .map(|rect| { + seed += 1; + let len = (rect.width() * rect.height()) as usize; + (pseudo_random_coefficients(len, seed, false), rect) + }) + .collect(), + }; + let windows = [first, next]; + for (skipped, cut) in [(0, 0), (1, 0), (2, 1), (3, 2), (4, 3), (0, 5), (1, 6)] { + let copy = IntRect::from_ltrb(next.x0, next.y0 + skipped, next.x1, next.y1 - cut); + let order = |in_place: bool| { + let log = core::cell::RefCell::new(Vec::new()); + let ring = ring_samples(windows, transform).expect("ring"); + let mut rings = vec![vec![f32::NAN; ring]; 3]; + let mut lines: Vec<_> = rings + .iter_mut() + .map(|ring| { + LineReconstruction::new( + &[[1, 2, 3], [4, 5, 6]], + &windows, + 0, + transform, + ring, + copy, + ) + .expect("plan the streamed rows") + }) + .collect(); + let mut sources: Vec<_> = (0..3) + .map(|component| LoggedBands { + bands: &bands, + component, + log: &log, + }) + .collect(); + for y in copy.y0..copy.y1 { + if in_place { + for (line, source) in lines.iter_mut().zip(sources.iter_mut()) { + line.finish_next_row(source).expect("final row"); + } + for (line, source) in lines.iter_mut().zip(sources.iter_mut()) { + let (row_y, _) = + line.next_row_mut(source).expect("row").expect("a row"); + assert_eq!(row_y, y); + } + } else { + for (line, source) in lines.iter_mut().zip(sources.iter_mut()) { + let (row_y, _) = + line.next_row(source).expect("row").expect("a row"); + assert_eq!(row_y, y); + } + } + } + log.into_inner() + }; + let label = format!("{transform:?} {first:?} skipped {skipped} cut {cut}"); + let final_order = order(false); + assert!(!final_order.is_empty(), "{label}"); + assert_eq!(order(true), final_order, "{label}"); + } + } + } +} + /// The whole-window region IDWT of one level, as `roi::fill_level_roi` runs it. fn whole_window_level( sources: [CoefficientSource<'_>; 4], @@ -785,6 +918,24 @@ fn streamed_rows_match_the_whole_window_levels_bit_for_bit() { ) .expect("streamed level"); assert_eq!(bits(&streamed), bits(&expected), "one level {label}"); + let overwritten = streamed_rows_overwritten( + &mut StoredBands { + bands: bands.clone(), + }, + &[[1, 2, 3]], + &[window], + transform, + window, + ); + assert_eq!(overwritten.len(), height as usize, "rows {label}"); + for (y, row) in overwritten { + let start = ((y - window.y0) * width) as usize; + assert_eq!( + row, + bits(&expected[start..][..width as usize]), + "one level, rows overwritten as read, {label} row {y}" + ); + } // A second level whose LL is the first level's window, // clipped or grown the same way. @@ -823,7 +974,9 @@ fn streamed_rows_match_the_whole_window_levels_bit_for_bit() { next, transform, ); - let mut rows = StoredBands { bands: levels }; + let mut rows = StoredBands { + bands: levels.clone(), + }; let windows = [first, next]; let mut ring = vec![f32::NAN; ring_samples(windows, transform).expect("two-level ring")]; @@ -854,6 +1007,22 @@ fn streamed_rows_match_the_whole_window_levels_bit_for_bit() { "two levels {label} row {y}" ); } + let overwritten = streamed_rows_overwritten( + &mut StoredBands { bands: levels }, + &[[1, 2, 3], [4, 5, 6]], + &windows, + transform, + copy, + ); + assert_eq!(overwritten.len(), copy.height() as usize, "rows {label}"); + for (y, row) in overwritten { + let start = ((y - next.y0) * next.width() + copy.x0 - next.x0) as usize; + assert_eq!( + row, + bits(&expected[start..][..copy.width() as usize]), + "two levels, rows overwritten as read, {label} row {y}" + ); + } } } } diff --git a/crates/j2k-native/src/j2c/mct.rs b/crates/j2k-native/src/j2c/mct.rs index 6c3f1d37..9adcac07 100644 --- a/crates/j2k-native/src/j2c/mct.rs +++ b/crates/j2k-native/src/j2c/mct.rs @@ -214,6 +214,32 @@ impl DeferredMct { } } + /// Transform one row of each of the three components and pack it as + /// RGB8 into `output`: through the fused ICT pack where this build has + /// one (see [`Self::fuses_rgb8_pack`]), which only reads the rows, + /// otherwise transformed in place and packed by `pack`, which must be + /// the standard RGB8 pack where the fused one exists. Both routes finish + /// their rows here: the planes' last level as it reconstructs them (see + /// [`Self::pack_rgb8`]) and the streamed IDWT's rows. + #[expect( + clippy::inline_always, + reason = "the row finish inlines into the planes' dispatched row streamer" + )] + #[inline(always)] + pub(crate) fn pack_rgb8_row( + self, + [r, g, b]: [&mut [f32]; 3], + output: &mut [u8], + pack: impl Fn([&[f32]; 3], &mut [u8]), + ) { + if self.fuses_rgb8_pack() { + self.pack_ict_rgb8_rows([r, g, b], output); + } else { + self.transform_chunk((&mut *r, &mut *g, &mut *b)); + pack([r, g, b], output); + } + } + /// Transform the planes in place, as [`apply_inverse`] would have. Planes /// that hold a last level's sub-bands are left to [`Self::pack_rgb8`]. pub(crate) fn apply( @@ -262,14 +288,7 @@ impl DeferredMct { scratch, output, parallel, - |[r, g, b], output| { - if self.fuses_rgb8_pack() { - self.pack_ict_rgb8_rows([r, g, b], output); - } else { - self.transform_chunk((&mut *r, &mut *g, &mut *b)); - pack([r, g, b], output); - } - }, + |rows, output| self.pack_rgb8_row(rows, output, &pack), ); } #[cfg(feature = "parallel")] diff --git a/crates/j2k-native/src/tests/fused_last_level.rs b/crates/j2k-native/src/tests/fused_last_level.rs index d5e608a6..a1f3fbd7 100644 --- a/crates/j2k-native/src/tests/fused_last_level.rs +++ b/crates/j2k-native/src/tests/fused_last_level.rs @@ -220,3 +220,133 @@ fn fused_region_pack_matches_staged_samples() { } } } + +/// The rows `roi` of an RGB8 image of `width` samples per row. +fn crop(rgb: &[u8], width: u32, (x, y, w, h): (u32, u32, u32, u32)) -> Vec { + (y..y + h) + .flat_map(|row| { + let start = ((row * width + x) * 3) as usize; + rgb[start..start + (w * 3) as usize].iter().copied() + }) + .collect() +} + +/// The streamed rows of `bytes`, packed into an RGB8 destination or handed +/// over as rows, must be the staged decode's bytes, on one thread and on +/// pools whose components reconstruct through one reader or a reader each, +/// for the whole image, a reduced one, and regions. +fn assert_streamed_rows_match_staged(bytes: &[u8], label: &str) { + #[cfg(feature = "parallel")] + let pools: Vec<(usize, rayon::ThreadPool)> = [1, 2, 4, 12] + .into_iter() + .map(|threads| { + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("build pool"); + (threads, pool) + }) + .collect(); + for reduction in [0, 1] { + let image = Image::new_with_reduction(bytes, &DecodeSettings::default(), reduction) + .expect("parse streamed fixture"); + let (w, h) = (image.width(), image.height()); + let mut staged = DecoderContext::default(); + staged.set_cpu_decode_parallelism(CpuDecodeParallelism::Serial); + staged.tile_decode_context.staged_rgb8_pack = true; + let expected = image + .decode_with_context(&mut staged) + .expect("staged decode") + .data; + for roi in [ + (0, 0, w, h), + (1, 1, w - 1, h - 1), + (w / 3, h / 3, w / 3, h / 4), + ] { + let expected = crop(&expected, w, roi); + let rows = |parallelism, offers_destination| { + let mut context = DecoderContext::default(); + context.set_cpu_decode_parallelism(parallelism); + let mut sink = super::row_sink::Rgb8Sink { + width: roi.2 as usize, + rgb: vec![0; (roi.2 * roi.3 * 3) as usize], + offers_destination, + rectangles: 0, + }; + image + .decode_region_rows_for_integer_output_with_context( + roi, + &mut context, + &mut sink, + ) + .expect("decode streamed rows"); + sink.rgb + }; + let label = format!("{label} reduction={reduction} roi={roi:?}"); + for offers_destination in [true, false] { + assert_eq!( + rows(CpuDecodeParallelism::Serial, offers_destination), + expected, + "{label} serial destination={offers_destination}" + ); + #[cfg(feature = "parallel")] + for (threads, pool) in &pools { + assert_eq!( + pool.install(|| rows(CpuDecodeParallelism::Auto, offers_destination)), + expected, + "{label} threads={threads} destination={offers_destination}" + ); + } + } + } + } +} + +/// The streamed route finishes each row of the last level in the IDWT's +/// ring, as the planes' fused last level finishes its rows in its own: at +/// the fused level's sizes and origins, and at the staged ones. +#[test] +fn streamed_rows_match_the_fused_last_level() { + for case in CASES + .iter() + .filter(|case| case.components == 3 && case.bit_depth == 8) + { + let (width, height) = case.size; + for reversible in [true, false] { + let samples = noisy_samples(width * height * 3); + let bytes = encode( + &samples, + width, + height, + 3, + 8, + false, + &EncodeOptions { + reversible, + num_decomposition_levels: case.levels, + tile_size: case.tile, + ..EncodeOptions::default() + }, + ) + .expect("encode streamed fixture"); + let label = format!( + "{width}x{height} levels={} tile={:?} reversible={reversible}", + case.levels, case.tile + ); + assert_streamed_rows_match_staged(&bytes, &label); + } + } + let fixtures: [(&str, &[u8]); 2] = [ + ( + "5/3 75x56 at (5, 3)", + include_bytes!("../../fixtures/image_origin/openjpeg_rgb_75x56_origin5x3_53.j2k"), + ), + ( + "9/7 72x59 at (7, 1)", + include_bytes!("../../fixtures/image_origin/openjpeg_rgb_72x59_origin7x1_97.j2k"), + ), + ]; + for (name, bytes) in fixtures { + assert_streamed_rows_match_staged(bytes, name); + } +} diff --git a/crates/j2k-native/src/tests/row_sink.rs b/crates/j2k-native/src/tests/row_sink.rs index 0a07ff72..a4f057fb 100644 --- a/crates/j2k-native/src/tests/row_sink.rs +++ b/crates/j2k-native/src/tests/row_sink.rs @@ -496,11 +496,11 @@ fn pool_band_rows_fit_every_components_clipped_blocks() { /// A sink with an RGB8 destination, which [`ComponentRowSink::write_rows`] /// fills the way the destination's contract says: each 8-bit sample rounded /// and saturated. -struct Rgb8Sink { - width: usize, - rgb: Vec, - offers_destination: bool, - rectangles: usize, +pub(super) struct Rgb8Sink { + pub(super) width: usize, + pub(super) rgb: Vec, + pub(super) offers_destination: bool, + pub(super) rectangles: usize, } impl ComponentRowSink for Rgb8Sink { From aa4c1ce5dd6cd765ee73142c1227bc381d19f2bb Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 04:32:08 -0500 Subject: [PATCH 90/94] perf(j2k-native): finish a single pool reader's rows as serial bands do A tile streaming from pool bands through one reader (classic tiles below the reader-per-component rule, and one-component tiles) reconstructed 32 rows of every component into a row batch, copying each finished row out of the IDWT's ring, then transformed and packed the batch on the same thread: a copy and a buffer that overlap nothing. That reader now goes through `RowOutput::write`, finishing each row in place in the ring as serial bands do; it still decodes bands itself while it waits. The batch, its charge and the batch path's one-reader branch go: only a reader per component, whose rows meet on the pool, holds row batches. Rows reach the sink one at a time rather than 32 at a time, which only changes how many rows a decode that fails partway leaves written. Tests inject a failing band at many points on serial bands, one reader and a reader per component, into an RGB8 destination and as rows: every row handed over is the clean decode's, once, from the top. A band stranded after rows were handed over ends the decode with an error, before them the tile still decodes on serial bands; and a decode over its budget, of the whole image or of regions clipped at the bottom, fails before its first row, so the retries without retained scratch or without the packet directory still run and succeed. Output is bit-identical. On a 12-core ARM64 machine, 7 interleaved rounds, against the planes route of the same build, 6 threads: a 5428 x 4460 RGB image at 0.5 coded bits per sample 1.036 -> 1.018 of the planes' time, a 6061 x 4252 one 0.965 -> 0.929; peak heap 36.7 -> 34.7 and 41.7 -> 39.4 MiB. --- .../j2k-native/src/j2c/decode/banded_tier1.rs | 60 +- .../src/j2c/decode/parallel_bands.rs | 107 ++-- crates/j2k-native/src/j2c/decode/state.rs | 11 +- crates/j2k-native/src/j2c/idwt/roi.rs | 43 +- crates/j2k-native/src/j2c/idwt/tests.rs | 50 +- crates/j2k-native/src/tests.rs | 1 + crates/j2k-native/src/tests/row_sink.rs | 13 +- .../src/tests/streamed_publication.rs | 529 ++++++++++++++++++ 8 files changed, 667 insertions(+), 147 deletions(-) create mode 100644 crates/j2k-native/src/tests/streamed_publication.rs diff --git a/crates/j2k-native/src/j2c/decode/banded_tier1.rs b/crates/j2k-native/src/j2c/decode/banded_tier1.rs index cdc3af31..6f817939 100644 --- a/crates/j2k-native/src/j2c/decode/banded_tier1.rs +++ b/crates/j2k-native/src/j2c/decode/banded_tier1.rs @@ -455,14 +455,13 @@ impl<'o, 't> RowOutput<'o, 't> { expect(dead_code, reason = "only the parallel band reader writes batches") )] impl RowOutput<'_, '_> { - /// [`Self::write`] through one reader per component, which may run on - /// different threads, `batch` rows at a time. Each component's - /// reconstruction advances `batch` rows into its own rows of `scratch`, - /// on the pool when there is a reader per component, then the rows are - /// transformed and handed over, packed straight into the sink's RGB8 - /// destination when it has one for three 8-bit components. With a - /// reader per component, a batch is transformed and packed on the pool - /// while the next one reconstructs, so `scratch` holds two batches. + /// [`Self::write`] through one reader per component, each on its own + /// pool task, `batch` rows at a time. Each component's reconstruction + /// advances `batch` rows into its own rows of `scratch`, then the rows + /// are transformed and handed over, packed straight into the sink's RGB8 + /// destination when it has one for three 8-bit components. With room for + /// two batches in `scratch`, a batch is transformed and packed on the + /// pool while the next one reconstructs. pub(super) fn write_batches( &mut self, lines: &mut [LineReconstruction<'_>], @@ -482,12 +481,12 @@ impl RowOutput<'_, '_> { .checked_mul(components) .ok_or(ValidationError::ImageTooLarge)?; let transform = (self.tile, self.round_irreversible_output); - let tasks = if readers.len() > 1 { readers.len() } else { 1 }; + let tasks = readers.len(); let (now, rest) = scratch.split_at_mut(half.min(scratch.len())); - // With a reader per component and room for two batches, a batch is - // transformed and handed over on the pool while the next one - // reconstructs in the other half of `scratch`. - let pipelined = readers.len() > 1 && rest.len() >= half; + // With room for two batches, a batch is transformed and handed over + // on the pool while the next one reconstructs in the other half of + // `scratch`. + let pipelined = rest.len() >= half; let mut halves = [now, rest]; // The rows, from a row, of the batch in `halves[1]` left to finish. let mut pending = None; @@ -670,9 +669,8 @@ fn overlap(spawned: impl FnOnce() + Send, inline: impl FnOnce()) { } /// Reconstruct the next `parts[c].len() / width` rows of each component `c`, -/// from row `first_y`, into `parts[c]`: on the pool, one component per task, -/// when there is a reader per component, otherwise all through the first, -/// a row of each component at a time. +/// from row `first_y`, into `parts[c]`, through its own reader: on the pool, +/// one component per task. #[cfg_attr( not(feature = "parallel"), expect(dead_code, reason = "only the parallel band reader writes batches") @@ -684,27 +682,8 @@ fn reconstruct_batch( first_y: u32, width: usize, ) -> Result<()> { - if readers.len() < lines.len() || lines.len() < 2 { - let reader = readers - .first_mut() - .ok_or(DecodingError::CodeBlockDecodeFailure)?; - // A row of each component at a time, as `RowOutput::write` reads - // them, so the bands of every component are taken in turn and the - // pool keeps decoding all of them. - let rows = parts.first().map_or(0, |part| part.len() / width); - for row in 0..rows { - let y = u32::try_from(row) - .ok() - .and_then(|row| first_y.checked_add(row)) - .ok_or(DecodingError::CodeBlockDecodeFailure)?; - for (line, part) in lines.iter_mut().zip(parts.iter_mut()) { - let out = part - .get_mut(row * width..(row + 1) * width) - .ok_or(DecodingError::CodeBlockDecodeFailure)?; - reconstruct_rows_into(line, &mut **reader, out, y, width)?; - } - } - return Ok(()); + if readers.len() < lines.len() { + return Err(DecodingError::CodeBlockDecodeFailure.into()); } let mut results: [Result<()>; 4] = core::array::from_fn(|_| Ok(())); #[cfg(feature = "parallel")] @@ -1292,6 +1271,13 @@ impl BandedRows<'_, '_> { /// Decode the code-blocks that start at `rows.start` into the /// sub-band's coefficients, which then hold the band's stored rows. fn decode_band(&mut self, index: usize, rows: Range) -> Result<()> { + #[cfg(test)] + if let Some(left) = self.tile_ctx.fail_band_after { + self.tile_ctx.fail_band_after = left.checked_sub(1); + if left == 0 { + return Err(DecodingError::CodeBlockDecodeFailure.into()); + } + } let start = profile::profile_now(self.profile_enabled); let bands = &self.bands[index]; let (sub_band, resolution, stored) = (bands.sub_band, bands.resolution, bands.stored); diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index 3dd9e260..002bf936 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -280,10 +280,7 @@ fn decode_route( header, transform, )?; - #[cfg(test)] - let strand_once = tile_ctx.strand_pool_band; - #[cfg(not(test))] - let strand_once = false; + let strand = test_strand(tile_ctx); let mut coefficients = core::mem::take(&mut storage.coefficients); let slots = split_slots(&rings, &bands, storage, &mut coefficients); let source = PayloadSource::new(storage); @@ -307,7 +304,7 @@ fn decode_route( run, profile_enabled, tier1_us, - strand_once, + strand, ), CodeBlockCoding::HighThroughput => run_bands( (route.workers(), 1, false), @@ -321,7 +318,7 @@ fn decode_route( run, profile_enabled, tier1_us, - strand_once, + strand, ), } }); @@ -495,23 +492,21 @@ pub(super) fn decode_tile_rows<'a>( { tile_ctx.debug_counters.thread_scratch_charged += live.live_bytes() - unscratched; } - // Rows reconstruct in batches as many as fit rows of the region's width, - // but never more than the tile's width is charged for (see - // `roi_idwt_peak_elements`): two of them with a reader per component of - // an HTJ2K tile, so one finishes while the next reconstructs; a classic + // A reader per component reconstructs its rows in batches, as many as + // fit rows of the region's width but never more than the tile's width is + // charged for (see `roi_idwt_peak_elements`): two of them for an HTJ2K + // tile, so one finishes while the next reconstructs; a classic // tile's readers wait on Tier-1 instead, and hold one batch of half the - // rows, with deep band rings when they read whole rows. One reader holds - // one short batch, which it finishes before it reconstructs the next. + // rows, with deep band rings when they read whole rows. One reader + // finishes each row in the IDWT's ring, as serial bands do. let pipelined = readers > 1 && coding == CodeBlockCoding::HighThroughput; - let batch_rows = |width: u32| { - streamed_batch_rows( - width as usize, - copy.height() as usize, - readers > 1 && !pipelined, - readers == 1, - ) + let batch_rows = + |width: u32| streamed_batch_rows(width as usize, copy.height() as usize, !pipelined); + let batch = if readers > 1 { + batch_rows(region.width()).min(batch_rows(copy.width())) + } else { + 0 }; - let batch = batch_rows(region.width()).min(batch_rows(copy.width())); let scratch_len = (copy.width() as usize) .checked_mul(components) .and_then(|samples| samples.checked_mul(batch)) @@ -535,10 +530,13 @@ pub(super) fn decode_tile_rows<'a>( // One reader for every component of a classic tile leaves the pool's // threads to Tier-1, and decodes bands itself while it waits. let helps = readers == 1 && coding == CodeBlockCoding::Classic; + let strand = test_strand(tile_ctx); #[cfg(test)] - let strand_once = tile_ctx.strand_pool_band; + let fail_after = test_countdown(tile_ctx.fail_band_after); + #[cfg(test)] + let failed_band = || test_countdown_ends(&fail_after); #[cfg(not(test))] - let strand_once = false; + let failed_band = || false; let mut tier1_us = 0; let mut output_samples = 0usize; let mut coefficients = core::mem::take(&mut storage.coefficients); @@ -557,7 +555,10 @@ pub(super) fn decode_tile_rows<'a>( lines.push(plan.reconstruction(ring)?); } let run = |readers: &mut [&mut (dyn SubBandRows + Send)]| -> Result<()> { - let written = output.write_batches(&mut lines, readers, scratch, batch); + let written = match readers { + [reader] => output.write(&mut lines, &mut **reader), + readers => output.write_batches(&mut lines, readers, scratch, batch), + }; output_samples = lines.iter().map(LineReconstruction::output_samples).sum(); written }; @@ -569,13 +570,16 @@ pub(super) fn decode_tile_rows<'a>( slots, (&mut tile_ctx.classic_task_workspaces, &mut scratches), |strip, output, workspace, scratch| { + if failed_band() { + return Err(DecodingError::CodeBlockDecodeFailure.into()); + } let (route, strip) = route_strip(&routes, &strip_starts, strip)?; route.decode_classic(strip, output, workspace, (scratch, source)) }, run, profile_enabled, &mut tier1_us, - strand_once, + strand, ), CodeBlockCoding::HighThroughput => run_bands( (workers, readers, helps), @@ -584,13 +588,16 @@ pub(super) fn decode_tile_rows<'a>( slots, (&mut tile_ctx.ht_task_workspaces, &mut scratches), |strip, output, workspace, scratch| { + if failed_band() { + return Err(DecodingError::CodeBlockDecodeFailure.into()); + } let (route, strip) = route_strip(&routes, &strip_starts, strip)?; route.decode_ht(strip, output, workspace, (scratch, source)) }, run, profile_enabled, &mut tier1_us, - strand_once, + strand, ), } }); @@ -601,6 +608,7 @@ pub(super) fn decode_tile_rows<'a>( tile_ctx.idwt_line_rings = buffer; #[cfg(test)] { + tile_ctx.fail_band_after = usize::try_from(fail_after.into_inner()).ok(); let counters = &mut tile_ctx.debug_counters; counters.thread_scratch_held += BlockScratch::held_bytes(&scratches); counters.thread_block_samples = counters @@ -1479,9 +1487,10 @@ struct Shared<'p, 'c, 'w, W> { /// The slot of each band decoding in parts, which its parts share. outputs: Vec>, max_workers: usize, - /// Test-only: the first worker to start a band abandons it. + /// Test-only: the worker that starts the band task after this many + /// abandons it (a negative count: none does). #[cfg(test)] - strand_once: core::sync::atomic::AtomicBool, + strand_after: core::sync::atomic::AtomicIsize, state: Mutex>, /// Signalled when a band is decoded or a decode fails. decoded: Condvar, @@ -1664,10 +1673,10 @@ fn run_bands( run: impl FnOnce(&mut [&mut (dyn SubBandRows + Send)]) -> Result<()>, profile_enabled: bool, tier1_us: &mut u128, - strand_once: bool, + strand: Option, ) -> Result { #[cfg(not(test))] - let _ = strand_once; + let _ = strand; let workspaces_needed = workers + readers.max(1); let mut workspaces: Vec<(&mut W, &mut BlockScratch)> = Vec::new(); workspaces @@ -1715,7 +1724,7 @@ fn run_bands( outputs: try_filled_with(bands.len(), || Mutex::new(<&mut [f32]>::default()))?, max_workers: workers, #[cfg(test)] - strand_once: core::sync::atomic::AtomicBool::new(strand_once), + strand_after: test_countdown(strand), state: Mutex::new(state), decoded: Condvar::new(), }; @@ -1789,6 +1798,35 @@ fn run_bands( error.map_or(Ok(true), Err) } +/// The band task a test strands (see `TileDecodeContext::strand_pool_band`). +#[cfg(test)] +fn test_strand(tile_ctx: &TileDecodeContext) -> Option { + tile_ctx + .strand_pool_band + .then_some(tile_ctx.strand_pool_band_after) +} + +/// No band task is stranded outside tests. +#[cfg(not(test))] +const fn test_strand(_: &TileDecodeContext) -> Option { + None +} + +/// Test-only: count one more event down on `left`, and whether it was the +/// one the count waited for (a negative count waits for none). +#[cfg(test)] +fn test_countdown_ends(left: &core::sync::atomic::AtomicIsize) -> bool { + left.fetch_sub(1, core::sync::atomic::Ordering::Relaxed) == 0 +} + +/// Test-only: a countdown of `events` for [`test_countdown_ends`]. +#[cfg(test)] +fn test_countdown(events: Option) -> core::sync::atomic::AtomicIsize { + core::sync::atomic::AtomicIsize::new( + events.map_or(-1, |events| isize::try_from(events).unwrap_or(isize::MAX)), + ) +} + fn try_filled(len: usize, value: T) -> Result> { let mut values = Vec::new(); values @@ -1872,10 +1910,7 @@ fn work( }; drop(state); #[cfg(test)] - if shared - .strand_once - .swap(false, core::sync::atomic::Ordering::Relaxed) - { + if test_countdown_ends(&shared.strand_after) { // Leave the band running, as the stranding bug would. if let Ok(mut state) = shared.lock() { state.active -= 1; @@ -3030,7 +3065,7 @@ mod tests { read, false, &mut 0, - false, + None, ), CodeBlockCoding::HighThroughput => run_bands( (workers, readers, helps), @@ -3045,7 +3080,7 @@ mod tests { read, false, &mut 0, - false, + None, ), }; let running = j2k_alloc_probe::peak_bytes() - base; diff --git a/crates/j2k-native/src/j2c/decode/state.rs b/crates/j2k-native/src/j2c/decode/state.rs index 9025a4d2..c05947f3 100644 --- a/crates/j2k-native/src/j2c/decode/state.rs +++ b/crates/j2k-native/src/j2c/decode/state.rs @@ -138,7 +138,8 @@ pub(crate) struct DecompositionStorage<'a> { pub(crate) shared_pool_band_rings: bool, /// Whether a tile streaming its rows from `parallel_bands` reconstructs /// them through one reader whatever its pool's workers (see - /// [`parallel_bands::one_reader`]), which holds one short batch of rows. + /// [`parallel_bands::one_reader`]), which finishes each row in the IDWT's + /// rings and holds no batch of rows. pub(crate) one_synthesis_reader: bool, /// Whether every component's IDWT rings are live at once, because the /// tile streams its rows to a sink, finishing them in the rings. @@ -285,6 +286,14 @@ pub(crate) struct TileDecodeContext { /// band is left running with no worker to finish it. #[cfg(all(test, feature = "parallel"))] pub(crate) strand_pool_band: bool, + /// Test-only: with `strand_pool_band`, the band task started after this + /// many others is the one abandoned. + #[cfg(all(test, feature = "parallel"))] + pub(crate) strand_pool_band_after: usize, + /// Test-only: a streamed tile's band decode after this many fails as a + /// corrupt code-block's would, on serial and pool bands. + #[cfg(test)] + pub(crate) fail_band_after: Option, /// Test-only: each pool band splits into this many parts, as far as its /// blocks allow, whatever the pool's size. #[cfg(all(test, feature = "parallel"))] diff --git a/crates/j2k-native/src/j2c/idwt/roi.rs b/crates/j2k-native/src/j2c/idwt/roi.rs index cb382239..c68d33ae 100644 --- a/crates/j2k-native/src/j2c/idwt/roi.rs +++ b/crates/j2k-native/src/j2c/idwt/roi.rs @@ -33,28 +33,16 @@ const STREAMED_MIN_BATCHES: usize = 4; /// halved: halving the 116-row batches of an 18000-wide tile cost 2 to 5%. const STREAMED_DEEP_MIN_ROWS: usize = 128; -/// Rows of each component a tile reconstructing through one reader holds -/// at most in its batch. The reader transforms and hands over each batch -/// before it reconstructs the next, so longer batches overlap nothing. -const STREAMED_ONE_READER_ROWS: usize = 32; - -/// Rows of each component a tile streaming from pool bands reconstructs at -/// once, for `height` rows of `width` samples. A tile with a reader per -/// component that holds `one_batch` (a classic tile; HTJ2K tiles pipeline -/// two) transforms it before the next one reconstructs, so its batches take -/// half the rows while that leaves at least `STREAMED_DEEP_MIN_ROWS`: as -/// fast, with half the rows held. A tile with `one_reader` holds one batch -/// of at most `STREAMED_ONE_READER_ROWS`. -pub(crate) fn streamed_batch_rows( - width: usize, - height: usize, - one_batch: bool, - one_reader: bool, -) -> usize { +/// Rows of each component a tile streaming from pool bands through a reader +/// per component reconstructs at once, for `height` rows of `width` samples. +/// A tile that holds `one_batch` (a classic tile; HTJ2K tiles pipeline two) +/// transforms it before the next one reconstructs, so its batches take half +/// the rows while that leaves at least `STREAMED_DEEP_MIN_ROWS`: as fast, +/// with half the rows held. A tile with one reader holds no batch: it +/// finishes each row in the IDWT's ring. +pub(crate) fn streamed_batch_rows(width: usize, height: usize, one_batch: bool) -> usize { let rows = STREAMED_BATCH_SAMPLES / width.max(1); - let rows = if one_reader { - rows.min(STREAMED_ONE_READER_ROWS) - } else if one_batch && rows / 2 >= STREAMED_DEEP_MIN_ROWS { + let rows = if one_batch && rows / 2 >= STREAMED_DEEP_MIN_ROWS { rows / 2 } else { rows @@ -117,23 +105,16 @@ pub(crate) fn roi_idwt_peak_elements( if storage.lockstep_lines { let batch_rows = roi_plan.final_window(component_idx).map_or(0, |window| { let (width, height) = (window.width() as usize, window.height() as usize); - if storage.parallel_bands { + if storage.parallel_bands && !storage.one_synthesis_reader { // Two batches, when each component reconstructs // on its own reader, unless the band rings hold - // more bands instead; one when the tile has one - // reader. - let one_reader = storage.one_synthesis_reader; - let batches = if storage.deep_band_rings || one_reader { - 1 - } else { - 2 - }; + // more bands instead; one reader holds none. + let batches = if storage.deep_band_rings { 1 } else { 2 }; width .saturating_mul(streamed_batch_rows( width, height, storage.deep_band_rings, - one_reader, )) .saturating_mul(batches) } else { diff --git a/crates/j2k-native/src/j2c/idwt/tests.rs b/crates/j2k-native/src/j2c/idwt/tests.rs index 18269f5b..d776b261 100644 --- a/crates/j2k-native/src/j2c/idwt/tests.rs +++ b/crates/j2k-native/src/j2c/idwt/tests.rs @@ -1104,49 +1104,19 @@ fn deep_band_rings_halve_the_streamed_row_batch() { // 5661 x 8287 (one tile): 370 rows of each component in each of two // pipelined batches, 185 rows in the one batch of deep rings. - assert_eq!(streamed_batch_rows(5661, 8287, false, false), 370); - assert_eq!(streamed_batch_rows(5661, 8287, true, false), 185); + assert_eq!(streamed_batch_rows(5661, 8287, false), 370); + assert_eq!(streamed_batch_rows(5661, 8287, true), 185); // 4096 wide: 512 rows, or 256 with deep rings. - assert_eq!(streamed_batch_rows(4096, 4096, false, false), 512); - assert_eq!(streamed_batch_rows(4096, 4096, true, false), 256); + assert_eq!(streamed_batch_rows(4096, 4096, false), 512); + assert_eq!(streamed_batch_rows(4096, 4096, true), 256); for deep_band_rings in [false, true] { // 18000 wide: 116 rows, too few to halve. - assert_eq!( - streamed_batch_rows(18000, 12000, deep_band_rings, false), - 116 - ); + assert_eq!(streamed_batch_rows(18000, 12000, deep_band_rings), 116); // 8200 wide: 255 rows, too few to halve. - assert_eq!( - streamed_batch_rows(8200, 12000, deep_band_rings, false), - 255 - ); - assert_eq!(streamed_batch_rows(256, 200, deep_band_rings, false), 50); - assert_eq!(streamed_batch_rows(256, 3, deep_band_rings, false), 1); - assert_eq!( - streamed_batch_rows(3 << 20, 4096, deep_band_rings, false), - 1 - ); - assert_eq!( - streamed_batch_rows(1024, 1 << 16, deep_band_rings, false), - 512 - ); + assert_eq!(streamed_batch_rows(8200, 12000, deep_band_rings), 255); + assert_eq!(streamed_batch_rows(256, 200, deep_band_rings), 50); + assert_eq!(streamed_batch_rows(256, 3, deep_band_rings), 1); + assert_eq!(streamed_batch_rows(3 << 20, 4096, deep_band_rings), 1); + assert_eq!(streamed_batch_rows(1024, 1 << 16, deep_band_rings), 512); } } - -/// A tile reconstructing through one reader finishes each batch before the -/// next, so its batch holds at most 32 rows of each component, within the -/// limits every streamed batch keeps. -#[test] -fn one_reader_holds_a_short_streamed_row_batch() { - use super::roi::streamed_batch_rows; - - // 5661 x 8287 at six threads: 32 rows instead of 370. - assert_eq!(streamed_batch_rows(5661, 8287, false, true), 32); - assert_eq!(streamed_batch_rows(18000, 12000, false, true), 32); - // 100000 wide: 20 rows fit the batch's samples. - assert_eq!(streamed_batch_rows(100_000, 12000, false, true), 20); - // 100 rows: at least four batches. - assert_eq!(streamed_batch_rows(256, 100, false, true), 25); - assert_eq!(streamed_batch_rows(256, 3, false, true), 1); - assert_eq!(streamed_batch_rows(3 << 20, 4096, false, true), 1); -} diff --git a/crates/j2k-native/src/tests.rs b/crates/j2k-native/src/tests.rs index 31b79b64..8544577e 100644 --- a/crates/j2k-native/src/tests.rs +++ b/crates/j2k-native/src/tests.rs @@ -19,6 +19,7 @@ mod packet_directory; mod parallel_bands; mod progression_matrix; mod row_sink; +mod streamed_publication; mod workspace_peak; mod workspace_reuse; diff --git a/crates/j2k-native/src/tests/row_sink.rs b/crates/j2k-native/src/tests/row_sink.rs index a4f057fb..6ac13c0c 100644 --- a/crates/j2k-native/src/tests/row_sink.rs +++ b/crates/j2k-native/src/tests/row_sink.rs @@ -915,8 +915,9 @@ fn streamed_rows_charge_every_components_rings() { /// A classic tile of several components reconstructs each on a reader of /// its own from 12 pool threads, and on 10 and 11 only from a size; the -/// readers then hold a batch of rows each where one reader would hold a -/// short one. The ROI plan must charge that batch on every such pool: the +/// readers then hold a batch of rows each, where one reader finishes its +/// rows in the IDWT's rings. The ROI plan must charge that batch on every +/// such pool: the /// line buffer each streamed tile takes stays within the IDWT charge, and /// the rows match the serial planes. The unit tests' size gate lies between /// this full image and its half, so both sides of it run on 10 and 11 @@ -979,6 +980,14 @@ fn classic_streamed_rows_charge_the_batches_of_their_readers() { counters.uncharged_line_samples, 0, "{label}: line buffer beyond the charge" ); + // One reader finishes its rows in the IDWT's rings; a reader per + // component holds a batch of them. + assert_eq!( + counters.line_batch_samples == 0, + readers == 1, + "{label}: row batch {}", + counters.line_batch_samples + ); } } } diff --git a/crates/j2k-native/src/tests/streamed_publication.rs b/crates/j2k-native/src/tests/streamed_publication.rs new file mode 100644 index 00000000..7f56cef5 --- /dev/null +++ b/crates/j2k-native/src/tests/streamed_publication.rs @@ -0,0 +1,529 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! A streamed decode hands its rows to the sink as it finishes them, so a +//! decode that fails partway leaves some rows written. Which rows that is +//! depends on the route: rows one at a time on serial bands and with one +//! pool reader, whole batches with a reader per component. Whatever the +//! route, every row the sink receives must be the row a clean decode +//! produces, received once and in order from the top, with nothing after a +//! row that is missing; the error must be the one the failure raises; and a +//! decode that runs out of budget must fail before its first row, so that +//! the retries that rerun it (without the retained scratch, or without the +//! image's packet directory) still can. + +use alloc::vec; +use alloc::vec::Vec; + +use super::banded_tier1::{noisy_pixels, HEIGHT, WIDTH}; +use crate::error::{DecodeError, DecodingError}; +use crate::{ + encode, ComponentPlane, ComponentRowSink, CpuDecodeParallelism, DecodeSettings, DecoderContext, + EncodeOptions, Image, +}; + +/// What an RGB8 destination holds where nothing was written. +const UNWRITTEN: u8 = 0xa5; + +/// The rows a sink received, as sample bits per component, with the number +/// of times each pixel arrived; or, with a destination, the RGB8 bytes the +/// decoder packed into it. +struct Published { + width: usize, + height: usize, + planes: Vec>, + arrivals: Vec, + rgb: Option>, +} + +impl Published { + fn new((width, height): (u32, u32), destination: bool) -> Self { + let samples = (width * height) as usize; + Self { + width: width as usize, + height: height as usize, + planes: vec![vec![0; samples]; 3], + arrivals: vec![0; samples], + rgb: destination.then(|| vec![UNWRITTEN; samples * 3]), + } + } + + /// The rows from the top that hold the clean decode's samples, checking + /// that every other row holds nothing and that no pixel arrived twice. + fn rows(&self, clean: &Self, label: &str) -> usize { + let mut published = 0; + for y in 0..self.height { + let label = format!("{label}: row {y}"); + let written = self.rgb.as_ref().map_or_else( + || self.sink_row_written(clean, y, &label), + |rgb| self.rgb_row_written(rgb, clean, y, &label), + ); + if written { + assert_eq!(published, y, "{label} after a missing row"); + published += 1; + } + } + published + } + + /// Whether row `y` of the destination holds the clean row; any other + /// content than the clean row must be bytes nothing wrote. + fn rgb_row_written(&self, rgb: &[u8], clean: &Self, y: usize, label: &str) -> bool { + let bytes = y * self.width * 3..(y + 1) * self.width * 3; + let clean = &clean.rgb.as_ref().expect("clean destination")[bytes.clone()]; + let bytes = &rgb[bytes]; + let written = bytes == clean; + assert!( + written || bytes.iter().all(|&byte| byte == UNWRITTEN), + "{label} written but not the clean row" + ); + written + } + + /// Whether row `y` reached the sink, checking that all of it arrived at + /// most once and that, if it arrived, it holds the clean row's samples. + fn sink_row_written(&self, clean: &Self, y: usize, label: &str) -> bool { + let row = y * self.width..(y + 1) * self.width; + let arrivals = &self.arrivals[row.clone()]; + assert!( + arrivals.iter().all(|&count| count == arrivals[0]) && arrivals[0] <= 1, + "{label} arrived {arrivals:?}" + ); + let written = arrivals[0] == 1; + if written { + for (plane, clean) in self.planes.iter().zip(&clean.planes) { + assert_eq!(plane[row.clone()], clean[row.clone()], "{label}"); + } + } + written + } + + fn untouched(&self) -> bool { + self.arrivals.iter().all(|&count| count == 0) + && self + .rgb + .as_ref() + .is_none_or(|rgb| rgb.iter().all(|&byte| byte == UNWRITTEN)) + } +} + +impl ComponentRowSink for Published { + fn write_rows(&mut self, x: u32, y: u32, planes: &[ComponentPlane<'_>]) { + let (width, height) = planes[0].dimensions(); + for row in 0..height as usize { + for column in 0..width as usize { + let at = (y as usize + row) * self.width + x as usize + column; + for (component, plane) in planes.iter().enumerate() { + self.planes[component][at] = + plane.samples()[row * width as usize + column].to_bits(); + } + self.arrivals[at] += 1; + } + } + } + + fn rgb8_region(&mut self, width: u32, height: u32) -> Option<&mut [u8]> { + let len = (width as usize * 3).checked_mul(height as usize)?; + self.rgb.as_mut().map(|rgb| &mut rgb[..len]) + } +} + +/// A fixture with many bands per component: 8-row code-blocks over three +/// levels of three noisy components. +fn fixture(reversible: bool, use_ht_block_coding: bool, rpcl: bool) -> Vec { + encode( + &noisy_pixels(3), + WIDTH, + HEIGHT, + 3, + 8, + false, + &EncodeOptions { + reversible, + use_ht_block_coding, + num_decomposition_levels: 3, + code_block_height_exp: 1, + progression_order: if rpcl { + crate::EncodeProgressionOrder::Rpcl + } else { + EncodeOptions::default().progression_order + }, + ..EncodeOptions::default() + }, + ) + .expect("encode publication fixture") +} + +/// How a streamed decode runs: on serial bands, or on a pool of this many +/// threads (one reader below 12 for classic tiles, a reader per component +/// from 12 or for HTJ2K ones). +#[derive(Clone, Copy, Debug)] +enum Route { + Serial, + #[cfg(feature = "parallel")] + Pool(usize), +} + +fn routes() -> Vec { + #[cfg(feature = "parallel")] + return vec![Route::Serial, Route::Pool(4), Route::Pool(12)]; + #[cfg(not(feature = "parallel"))] + return vec![Route::Serial]; +} + +struct Outcome { + result: crate::Result<()>, + sink: Published, + streamed_samples: usize, + #[cfg_attr(not(feature = "parallel"), expect(dead_code, reason = "pool counters"))] + readers: usize, + #[cfg_attr(not(feature = "parallel"), expect(dead_code, reason = "pool counters"))] + fallbacks: usize, +} + +/// Stream the whole image along `route`, into a destination or as rows, on +/// a context `configure` sets up, from `baseline` bytes. +fn stream<'a>( + image: &Image<'a>, + route: Route, + destination: bool, + baseline: usize, + configure: impl FnOnce(&mut DecoderContext<'a>) + Send, +) -> Outcome { + stream_region(image, route, FULL, destination, baseline, configure) +} + +/// The whole image, as a region. +const FULL: (u32, u32, u32, u32) = (0, 0, WIDTH, HEIGHT); + +/// Stream `region` of the image along `route`, as [`stream`] does. +fn stream_region<'a>( + image: &Image<'a>, + route: Route, + region: (u32, u32, u32, u32), + destination: bool, + baseline: usize, + configure: impl FnOnce(&mut DecoderContext<'a>) + Send, +) -> Outcome { + let run = || { + let mut context = DecoderContext::default(); + context.set_cpu_decode_parallelism(match route { + Route::Serial => CpuDecodeParallelism::Serial, + #[cfg(feature = "parallel")] + Route::Pool(_) => CpuDecodeParallelism::Auto, + }); + configure(&mut context); + let mut sink = Published::new((region.2, region.3), destination); + let result = image + .run_decode_rows( + &mut context, + Some(region), + None, + true, + baseline, + Some(&mut sink), + ) + .map(drop); + let tile_ctx = &context.tile_decode_context; + // A decode past its budget may fail before it plans its rows. + assert!( + tile_ctx.stream_rows || result.is_err(), + "{route:?}: the decode streams" + ); + #[cfg(feature = "parallel")] + let (readers, fallbacks) = ( + tile_ctx.debug_counters.pool_band_readers, + tile_ctx.debug_counters.pool_band_fallbacks, + ); + #[cfg(not(feature = "parallel"))] + let (readers, fallbacks) = (0, 0); + Outcome { + result, + sink, + streamed_samples: tile_ctx.streamed_samples, + readers, + fallbacks, + } + }; + match route { + Route::Serial => run(), + #[cfg(feature = "parallel")] + Route::Pool(threads) => rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("build pool") + .install(run), + } +} + +fn is_band_failure(result: &crate::Result<()>) -> bool { + matches!( + result, + Err(DecodeError::Decoding(DecodingError::CodeBlockDecodeFailure)) + ) +} + +/// A band decode that fails partway through a streamed decode leaves only +/// clean rows, once and from the top, and the failure's error: none before +/// the first band, rows one at a time on serial bands and with one pool +/// reader, whole batches with a reader per component. On serial bands a +/// later failure leaves no fewer rows; on the pool, which band fails is the +/// one a worker happens to take at that count. +#[test] +fn a_failing_band_leaves_only_clean_rows_from_the_top() { + for (reversible, ht) in [(false, false), (true, false), (false, true)] { + let bytes = fixture(reversible, ht, false); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("parse fixture"); + for route in routes() { + for destination in [false, true] { + let clean = stream(&image, route, destination, 0, |_| {}); + clean.result.expect("clean decode"); + assert_eq!(clean.sink.rows(&clean.sink, "clean"), HEIGHT as usize); + #[cfg(feature = "parallel")] + if let Route::Pool(threads) = route { + let readers = if ht || threads >= 12 { 3 } else { 1 }; + assert_eq!(clean.readers, readers, "{route:?} ht={ht}: readers"); + } + let mut published = Vec::new(); + for after in [0, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, 10_000] { + let label = format!( + "reversible={reversible} ht={ht} {route:?} destination={destination} \ + failing band {after}" + ); + let outcome = stream(&image, route, destination, 0, |context| { + context.tile_decode_context.fail_band_after = Some(after); + }); + let rows = outcome.sink.rows(&clean.sink, &label); + if outcome.result.is_ok() { + // Past the tile's last band nothing fails. + assert_eq!(rows, HEIGHT as usize, "{label}"); + continue; + } + assert!( + is_band_failure(&outcome.result), + "{label}: {:?}", + outcome.result + ); + published.push(rows); + } + let label = format!("reversible={reversible} ht={ht} {route:?} {destination}"); + assert_eq!( + published.first(), + Some(&0), + "{label}: the first band fails first" + ); + if matches!(route, Route::Serial) { + assert!( + published.windows(2).all(|pair| pair[0] <= pair[1]), + "{label}: later failures leave no fewer rows: {published:?}" + ); + assert!( + published + .iter() + .any(|&rows| rows > 0 && rows < HEIGHT as usize), + "{label}: some failure comes partway: {published:?}" + ); + } + } + } + } +} + +/// A band the pool leaves running with no worker to finish it is a +/// scheduler fault. Before any row reaches the sink, the tile decodes again +/// on serial bands, as fast decodes do; after, the rows can't be taken +/// back, so the decode fails, having handed over only clean rows, once. +/// Which rows precede a strand depends on how far the workers ran ahead, so +/// each kind of tile must show both outcomes over its pools and outputs. +#[cfg(feature = "parallel")] +#[test] +fn a_band_stranded_after_published_rows_ends_the_decode() { + for ht in [false, true] { + let bytes = fixture(false, ht, false); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("parse fixture"); + let (mut fell_back, mut failed_after_rows) = (0, 0); + for threads in [4, 12] { + let route = Route::Pool(threads); + for destination in [false, true] { + let clean = stream(&image, route, destination, 0, |_| {}); + clean.result.expect("clean decode"); + for after in [ + 0, 1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 192, 256, + ] { + let label = format!( + "ht={ht} threads={threads} destination={destination} stranded task {after}" + ); + let outcome = stream(&image, route, destination, 0, |context| { + context.tile_decode_context.strand_pool_band = true; + context.tile_decode_context.strand_pool_band_after = after; + }); + let rows = outcome.sink.rows(&clean.sink, &label); + if outcome.result.is_ok() { + assert_eq!(rows, HEIGHT as usize, "{label}"); + fell_back += usize::from(outcome.fallbacks == 1); + } else { + assert!(is_band_failure(&outcome.result), "{label}"); + assert_eq!(outcome.fallbacks, 1, "{label}"); + assert!(rows > 0, "{label}: stranded before any row, yet failed"); + failed_after_rows += 1; + } + } + } + } + assert!( + fell_back > 0, + "ht={ht}: a strand before the first row falls back" + ); + assert!( + failed_after_rows > 0, + "ht={ht}: a strand after published rows fails" + ); + } +} + +/// The largest baseline at which a fresh decode of `region` along `route` +/// fits its budget. +fn fitting_baseline( + image: &Image<'_>, + route: Route, + region: (u32, u32, u32, u32), + destination: bool, +) -> usize { + let (mut fits, mut fails) = (0, crate::DEFAULT_MAX_DECODE_BYTES); + while fails - fits > 1 { + let candidate = fits + (fails - fits) / 2; + let outcome = stream_region(image, route, region, destination, candidate, |_| {}); + match outcome.result { + Ok(()) => fits = candidate, + Err(_) => fails = candidate, + } + } + fits +} + +/// A streamed decode charges everything it holds before its first row, so +/// whatever budget it starts from, a capacity failure comes before any row +/// reaches the sink: the retained-scratch and packet-directory retries, +/// which rerun only a decode that has handed nothing over, stay open on +/// every route. A decode that succeeds through either retry still does, +/// with every row once. Besides the whole image, a region that clips the +/// code-blocks of its last rows, which decode only after its first rows +/// went out. +#[test] +fn capacity_failures_come_before_the_first_row() { + for ht in [false, true] { + let bytes = fixture(false, ht, true); + for region in [FULL, (0, 0, WIDTH, HEIGHT - 11), (0, 0, WIDTH, 61)] { + for route in routes() { + for destination in [false, true] { + let label = format!("ht={ht} {region:?} {route:?} destination={destination}"); + capacity_failures_leave_the_sink_untouched( + &bytes, + route, + region, + destination, + &label, + ); + } + } + } + } +} + +fn capacity_failures_leave_the_sink_untouched( + bytes: &[u8], + route: Route, + region: (u32, u32, u32, u32), + destination: bool, + label: &str, +) { + let rows = region.3 as usize; + let fresh = Image::new(bytes, &DecodeSettings::default()).expect("parse fixture"); + let clean = stream_region(&fresh, route, region, destination, 0, |_| {}); + clean.result.expect("clean decode"); + let fits = fitting_baseline(&fresh, route, region, destination); + // Past the largest budget that fits, one that leaves out the optional + // row index can fit again: each decodes every row once or fails first. + let mut failures = 0; + for over in [ + 1, + 2, + 3, + 5, + 9, + 17, + 65, + 257, + 1 << 10, + 1 << 12, + 1 << 14, + 1 << 16, + 1 << 18, + ] { + let outcome = stream_region(&fresh, route, region, destination, fits + over, |_| {}); + let label = format!("{label} +{over}"); + match outcome.result { + Ok(()) => assert_eq!(outcome.sink.rows(&clean.sink, &label), rows), + Err(error) => { + assert!(crate::j2c::is_capacity_error(&error), "{label}: {error:?}"); + assert_eq!(outcome.streamed_samples, 0, "{label}"); + assert!(outcome.sink.untouched(), "{label}: rows before the failure"); + failures += 1; + } + } + } + assert!(failures > 0, "{label}: some budget fails"); + for under in [0, 1, 1 << 10, 1 << 16] { + let outcome = stream_region(&fresh, route, region, destination, fits - under, |_| {}); + outcome.result.expect("within the budget"); + assert_eq!(outcome.sink.rows(&clean.sink, label), rows); + } + + // A context holding the scratch of a planes decode starts over the + // budget; the retry without it fits. + let mut reused = DecoderContext::default(); + drop( + fresh + .decode_region_components_for_integer_output_with_context(region, &mut reused) + .expect("planes decode"), + ); + assert!(reused.tile_decode_context.retained_scratch_bytes().unwrap() > 0); + let outcome = stream_region(&fresh, route, region, destination, fits, |context| { + core::mem::swap(context, &mut reused); + context.set_cpu_decode_parallelism(match route { + Route::Serial => CpuDecodeParallelism::Serial, + #[cfg(feature = "parallel")] + Route::Pool(_) => CpuDecodeParallelism::Auto, + }); + }); + outcome + .result + .expect("a reused context fits as a fresh one"); + assert_eq!(outcome.sink.rows(&clean.sink, label), rows); + + // An image whose repeated regions built a packet directory gives it up + // for a request that fits without it. + #[cfg(feature = "std")] + { + let image = Image::new(bytes, &DecodeSettings::default()).expect("parse fixture"); + let metadata = image.retained_allocation_bytes().expect("metadata bytes"); + for _ in 0..2 { + stream_region(&image, route, region, destination, metadata, |_| {}) + .result + .expect("decode with the directory"); + } + assert!( + image.retained_allocation_bytes().expect("retained bytes") > metadata, + "{label}: repeated decodes build a directory" + ); + let outcome = stream_region(&image, route, region, destination, fits, |_| {}); + outcome + .result + .expect("evicting the directory fits the request"); + assert_eq!(outcome.sink.rows(&clean.sink, label), rows); + assert_eq!( + image.retained_allocation_bytes().expect("retained bytes"), + metadata, + "{label}: the directory is evicted" + ); + } +} From 87d3dc4f75ac7500992fc7732bb75a456957acb3 Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 05:34:45 -0500 Subject: [PATCH 91/94] fix(j2k-native): count packed rows as handed over when the next batch fails A tile whose components reconstruct on readers of their own, two batches of rows at a time, packs one batch into the sink's RGB8 destination while the next reconstructs. When that reconstruction failed, the error returned before the packed batch was counted as streamed, though its rows were already in the destination. A band stranded there (a scheduler fault) then found nothing handed over and decoded the tile again on serial bands, writing those rows a second time; any other error left the count short. Packed rows now count as soon as they are packed, whatever failed alongside, so a strand after them ends the decode with an error, as for rows handed to `write_rows`. Errors keep their precedence and output is unchanged. The publication tests' destination now counts every row written into it, blanking what it has seen each time the decoder asks for it, so a row written twice shows even with the same bytes; the stranded-band test fails without this change. --- .../j2k-native/src/j2c/decode/banded_tier1.rs | 22 +++--- .../src/tests/streamed_publication.rs | 72 +++++++++++-------- 2 files changed, 58 insertions(+), 36 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode/banded_tier1.rs b/crates/j2k-native/src/j2c/decode/banded_tier1.rs index 6f817939..08463f0e 100644 --- a/crates/j2k-native/src/j2c/decode/banded_tier1.rs +++ b/crates/j2k-native/src/j2c/decode/banded_tier1.rs @@ -561,7 +561,7 @@ impl RowOutput<'_, '_> { let destination = self.rgb8_rows(y, rows)?; let packed = destination.is_some(); let mct = mct.filter(|_| packed); - match alongside { + let finished = match alongside { Some(alongside) => { let mut finished = Ok(()); let mut alongside_result = Ok(()); @@ -577,8 +577,7 @@ impl RowOutput<'_, '_> { }, || alongside_result = alongside(), ); - alongside_result?; - finished?; + alongside_result.and(finished) } None => finish_rows( (transform, mct), @@ -586,12 +585,19 @@ impl RowOutput<'_, '_> { width, destination, tasks, - )?, - } - if !packed { - self.write_component_rows(&parts[..components], y, rows); + ), + }; + let samples = rows * width * components; + if packed { + // The rows are in the destination once they are packed, whatever + // failed alongside, so they count as handed over: a tile that + // then fails must not decode them again. + self.streamed = self.streamed.saturating_add(samples); + return finished; } - self.streamed = self.streamed.saturating_add(rows * width * components); + finished?; + self.write_component_rows(&parts[..components], y, rows); + self.streamed = self.streamed.saturating_add(samples); Ok(()) } diff --git a/crates/j2k-native/src/tests/streamed_publication.rs b/crates/j2k-native/src/tests/streamed_publication.rs index 7f56cef5..92d23d30 100644 --- a/crates/j2k-native/src/tests/streamed_publication.rs +++ b/crates/j2k-native/src/tests/streamed_publication.rs @@ -26,13 +26,17 @@ const UNWRITTEN: u8 = 0xa5; /// The rows a sink received, as sample bits per component, with the number /// of times each pixel arrived; or, with a destination, the RGB8 bytes the -/// decoder packed into it. +/// decoder packed into it. Each time the decoder asks for the destination, +/// the rows written since the last time are counted, moved out and the +/// destination blanked again, so a row written twice counts twice even with +/// the same bytes. struct Published { width: usize, height: usize, planes: Vec>, arrivals: Vec, rgb: Option>, + received: Vec, } impl Published { @@ -44,6 +48,31 @@ impl Published { planes: vec![vec![0; samples]; 3], arrivals: vec![0; samples], rgb: destination.then(|| vec![UNWRITTEN; samples * 3]), + received: vec![UNWRITTEN; if destination { samples * 3 } else { 0 }], + } + } + + /// Count and move out the destination's pixels written since the last + /// call: of every row whose first or last pixel was written (the + /// decoder writes whole rows of a tile this wide), or with `all`, of + /// every row. + fn collect(&mut self, all: bool) { + let Some(rgb) = self.rgb.as_mut() else { + return; + }; + let blank = |pixel: &[u8]| pixel.iter().all(|&byte| byte == UNWRITTEN); + for (y, row) in rgb.chunks_exact_mut(self.width * 3).enumerate() { + if !all && blank(&row[..3]) && blank(&row[row.len() - 3..]) { + continue; + } + for (x, pixel) in row.as_chunks_mut::<3>().0.iter_mut().enumerate() { + if !blank(pixel) { + let at = y * self.width + x; + self.arrivals[at] += 1; + self.received[at * 3..][..3].copy_from_slice(pixel); + pixel.fill(UNWRITTEN); + } + } } } @@ -53,11 +82,7 @@ impl Published { let mut published = 0; for y in 0..self.height { let label = format!("{label}: row {y}"); - let written = self.rgb.as_ref().map_or_else( - || self.sink_row_written(clean, y, &label), - |rgb| self.rgb_row_written(rgb, clean, y, &label), - ); - if written { + if self.row_written(clean, y, &label) { assert_eq!(published, y, "{label} after a missing row"); published += 1; } @@ -65,23 +90,9 @@ impl Published { published } - /// Whether row `y` of the destination holds the clean row; any other - /// content than the clean row must be bytes nothing wrote. - fn rgb_row_written(&self, rgb: &[u8], clean: &Self, y: usize, label: &str) -> bool { - let bytes = y * self.width * 3..(y + 1) * self.width * 3; - let clean = &clean.rgb.as_ref().expect("clean destination")[bytes.clone()]; - let bytes = &rgb[bytes]; - let written = bytes == clean; - assert!( - written || bytes.iter().all(|&byte| byte == UNWRITTEN), - "{label} written but not the clean row" - ); - written - } - - /// Whether row `y` reached the sink, checking that all of it arrived at - /// most once and that, if it arrived, it holds the clean row's samples. - fn sink_row_written(&self, clean: &Self, y: usize, label: &str) -> bool { + /// Whether row `y` arrived, checking that all of it arrived at most once + /// and that, if it arrived, it holds the clean row's samples or bytes. + fn row_written(&self, clean: &Self, y: usize, label: &str) -> bool { let row = y * self.width..(y + 1) * self.width; let arrivals = &self.arrivals[row.clone()]; assert!( @@ -89,7 +100,14 @@ impl Published { "{label} arrived {arrivals:?}" ); let written = arrivals[0] == 1; - if written { + if written && self.rgb.is_some() { + let bytes = row.start * 3..row.end * 3; + assert_eq!( + self.received[bytes.clone()], + clean.received[bytes], + "{label}" + ); + } else if written { for (plane, clean) in self.planes.iter().zip(&clean.planes) { assert_eq!(plane[row.clone()], clean[row.clone()], "{label}"); } @@ -99,10 +117,6 @@ impl Published { fn untouched(&self) -> bool { self.arrivals.iter().all(|&count| count == 0) - && self - .rgb - .as_ref() - .is_none_or(|rgb| rgb.iter().all(|&byte| byte == UNWRITTEN)) } } @@ -122,6 +136,7 @@ impl ComponentRowSink for Published { } fn rgb8_region(&mut self, width: u32, height: u32) -> Option<&mut [u8]> { + self.collect(false); let len = (width as usize * 3).checked_mul(height as usize)?; self.rgb.as_mut().map(|rgb| &mut rgb[..len]) } @@ -223,6 +238,7 @@ fn stream_region<'a>( Some(&mut sink), ) .map(drop); + sink.collect(true); let tile_ctx = &context.tile_decode_context; // A decode past its budget may fail before it plans its rows. assert!( From e1d401f8882c65eee7068310d64c8f4f5416344f Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 05:54:40 -0500 Subject: [PATCH 92/94] perf(j2k-native): keep pipelined row batches to 256 rows An HTJ2K tile streaming from pool bands through a reader per component holds two batches of every component's rows, each as many rows as 2 Mi samples fit, up to 512: on a 4096-wide RGB tile, 2 x 512 rows of 3 components, 48 of its 72 MiB peak. Two batches now keep to 256 rows each (`STREAMED_PIPELINED_MAX_ROWS`), the smallest size measured that keeps the batches' pipeline at speed. A tile holding one batch (classic) and tiles whose batches are shorter already, such as 18000-wide ones (116 rows) or 1024 x 1024 regions (256), are unchanged. Output is bit-identical. A 4096 x 2048 HTJ2K image at 2 bits per pixel, 7 interleaved rounds, against 512-row batches of the same build (a second copy of that build in brackets): on an 18-core ARM64 machine 1.005 [0.997] of the time on 12 threads, 0.986 [1.005] on 18, 0.983 [0.985] on all cores (11 rounds; a lossless image of that size 1.012 [0.998]); on a 10-core one 0.994 [0.993] on 10 threads. Peak heap 71.8 -> 47.8 MiB; the first decode takes 0.87-0.91 of the time. 128-row batches would hold 36 MiB but take 3-10% longer, and 6-15% on 1024 x 1024 regions: each batch ends at a barrier where the readers wait for the slowest component, and smaller batches meet it more often. --- crates/j2k-native/src/j2c/idwt/roi.rs | 16 +++++++++++++--- crates/j2k-native/src/j2c/idwt/tests.rs | 21 ++++++++++++--------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/crates/j2k-native/src/j2c/idwt/roi.rs b/crates/j2k-native/src/j2c/idwt/roi.rs index c68d33ae..24b62d50 100644 --- a/crates/j2k-native/src/j2c/idwt/roi.rs +++ b/crates/j2k-native/src/j2c/idwt/roi.rs @@ -33,16 +33,26 @@ const STREAMED_MIN_BATCHES: usize = 4; /// halved: halving the 116-row batches of an 18000-wide tile cost 2 to 5%. const STREAMED_DEEP_MIN_ROWS: usize = 128; +/// Rows a batch of a tile holding two batches keeps at most. HTJ2K tiles +/// pipeline two: on 10 to 18 threads, 256-row batches of a 4096-wide tile +/// decode as fast as 512-row ones (0.99-1.01 of their time) and hold two +/// thirds of the memory (48 vs 72 MiB); 128-row ones hold half (36 MiB) but +/// take 3-10% longer, and 1024 x 1024 regions 6-15% longer. +const STREAMED_PIPELINED_MAX_ROWS: usize = 256; + /// Rows of each component a tile streaming from pool bands through a reader /// per component reconstructs at once, for `height` rows of `width` samples. /// A tile that holds `one_batch` (a classic tile; HTJ2K tiles pipeline two) /// transforms it before the next one reconstructs, so its batches take half /// the rows while that leaves at least `STREAMED_DEEP_MIN_ROWS`: as fast, -/// with half the rows held. A tile with one reader holds no batch: it -/// finishes each row in the IDWT's ring. +/// with half the rows held. Two batches keep to +/// `STREAMED_PIPELINED_MAX_ROWS` each. A tile with one reader holds no +/// batch: it finishes each row in the IDWT's ring. pub(crate) fn streamed_batch_rows(width: usize, height: usize, one_batch: bool) -> usize { let rows = STREAMED_BATCH_SAMPLES / width.max(1); - let rows = if one_batch && rows / 2 >= STREAMED_DEEP_MIN_ROWS { + let rows = if !one_batch { + rows.min(STREAMED_PIPELINED_MAX_ROWS) + } else if rows / 2 >= STREAMED_DEEP_MIN_ROWS { rows / 2 } else { rows diff --git a/crates/j2k-native/src/j2c/idwt/tests.rs b/crates/j2k-native/src/j2c/idwt/tests.rs index d776b261..8ca683e0 100644 --- a/crates/j2k-native/src/j2c/idwt/tests.rs +++ b/crates/j2k-native/src/j2c/idwt/tests.rs @@ -1095,20 +1095,24 @@ fn wide_empty_band_rows_match_the_whole_window_level_bit_for_bit() { /// A tile streaming from pool bands with deep band rings holds one batch of /// rows instead of two, so its batch takes half the rows unless that leaves -/// fewer than 128; tiles too short for full batches still split their rows -/// into at least four batches, and a row wider than a batch is a batch of -/// its own. +/// fewer than 128; two pipelined batches keep to 256 rows each; tiles too +/// short for full batches still split their rows into at least four +/// batches, and a row wider than a batch is a batch of its own. #[test] fn deep_band_rings_halve_the_streamed_row_batch() { use super::roi::streamed_batch_rows; - // 5661 x 8287 (one tile): 370 rows of each component in each of two - // pipelined batches, 185 rows in the one batch of deep rings. - assert_eq!(streamed_batch_rows(5661, 8287, false), 370); + // 5661 x 8287 (one tile): 256 rows of each component in each of two + // pipelined batches (370 fit their samples), 185 rows in the one batch + // of deep rings. + assert_eq!(streamed_batch_rows(5661, 8287, false), 256); assert_eq!(streamed_batch_rows(5661, 8287, true), 185); - // 4096 wide: 512 rows, or 256 with deep rings. - assert_eq!(streamed_batch_rows(4096, 4096, false), 512); + // 4096 wide: 256 rows either way (512 fit, or half of them). + assert_eq!(streamed_batch_rows(4096, 4096, false), 256); assert_eq!(streamed_batch_rows(4096, 4096, true), 256); + // 1024 wide: 256 rows in each of two batches, 512 in one. + assert_eq!(streamed_batch_rows(1024, 1 << 16, false), 256); + assert_eq!(streamed_batch_rows(1024, 1 << 16, true), 512); for deep_band_rings in [false, true] { // 18000 wide: 116 rows, too few to halve. assert_eq!(streamed_batch_rows(18000, 12000, deep_band_rings), 116); @@ -1117,6 +1121,5 @@ fn deep_band_rings_halve_the_streamed_row_batch() { assert_eq!(streamed_batch_rows(256, 200, deep_band_rings), 50); assert_eq!(streamed_batch_rows(256, 3, deep_band_rings), 1); assert_eq!(streamed_batch_rows(3 << 20, 4096, deep_band_rings), 1); - assert_eq!(streamed_batch_rows(1024, 1 << 16, deep_band_rings), 512); } } From 7b4528dd12eb4f4bebfc267f1e25704d789691ea Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 05:54:40 -0500 Subject: [PATCH 93/94] docs(api): record streamed decoding behavior and 0.13.2 review --- CHANGELOG.md | 54 +++++++++++++------ crates/j2k/src/owned_batch/cpu.rs | 4 +- .../public-api/public-api-review-0.13.2.yml | 14 ++--- .../reviewed-public-api-diff-0.13.2.md | 8 +-- 4 files changed, 51 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd2f5269..1df411a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,19 +15,41 @@ Staged workspace version: `0.13.2`. status, detail and batch source index. CUDA keeps its typed source and job attribution. Unsupported statuses, raw runtime failures and combined cleanup or release failures keep their existing classes. -- Streamed decodes keep to the memory they charge. A tile whose components - reconstruct on readers of their own sized its batch of rows by the - region's width while the decode charged the tile's width, so a tile just - narrower than its region could hold twice its charge (an 8,192-wide tile - of an 8,200-wide image: 6.4 M against 3.4 M samples); its batch now keeps - to the charge, and such an image decodes in 0.98 of the time on 12 and 18 - threads, with 60.7 MiB in place of 72.6 MiB on 12. A serial HTJ2K region - decode allocated the scratch for code-blocks its region clips when it met - the first one, so a decode short of budget could fail after its first rows - reached the sink, where the retries without retained scratch or without - the packet directory no longer run; the scratch is now sized with the rest - of the Tier-1 workspace before the first row. Output of decodes that - succeed is unchanged. +- Row-streamed decodes on serial bands finish directly from the inverse + transform's own rows. The streamed inverse transform lifts every level + with the banded transform's kernels, one arrival at a time on each level's + ring of rows; on serial bands, a row is finished directly from the ring + (inverse colour transform, level shift, rounding and packing) once no + lifting step reads it, in place of a copy into scratch rows, and the 8-bit + RGB finish is shared with the component planes' fused last level. A tile + on the worker pool with one reader finishes its rows the same way instead + of copying them into batches of up to 32 rows, and an HTJ2K tile whose + components reconstruct on readers of their own keeps each of its two + pipelined row batches to at most 256 rows. On an 18-core ARM64 machine a + 5428 x 4460 RGB image at 1.5 bits per pixel streams in 0.92 of its + previous time on 6 threads, a 4096 x 2048 HTJ2K image in 0.90 to 0.95 on 1 + to 18 threads and with 48 MiB in place of 72 MiB on a pool, an 18000 x + 12000 HTJ2K image in 0.88 on 18 threads and 1024 x 1024 HTJ2K regions in + 0.90 to 0.94. Output is unchanged; a decode that fails partway can leave a + different number of rows written, each of them a row the successful decode + writes. +- Streamed decodes keep to the memory they charge and hand each row over + once. A tile whose components reconstruct on readers of their own sized + its batch of rows by the region's width while the decode charged the + tile's width, so a tile just narrower than its region could hold twice its + charge (an 8,192-wide tile of an 8,200-wide image: 6.4 M against 3.4 M + samples); its batch now keeps to the charge, and such an image decodes in + 0.98 of the time on 12 and 18 threads, with 60.7 MiB in place of 72.6 MiB + on 12. A serial HTJ2K region decode allocated the scratch for code-blocks + its region clips when it met the first one, so a decode short of budget + could fail after its first rows reached the sink, where the retries + without retained scratch or without the packet directory no longer run; + the scratch is now sized with the rest of the Tier-1 workspace before the + first row. A tile packing two batches of rows into an RGB8 destination + counted a packed batch only once the next batch had reconstructed, so a + scheduler fault there could decode the tile again and write those rows + twice; the packed rows now count at once. Output of decodes that succeed + is unchanged. - Images that a batch with an explicit worker count decodes together on its stream pool, when it plans as many images at once as the pool has threads and the pool has fewer than 12, hold two bands in every ring of @@ -170,9 +192,9 @@ Staged workspace version: `0.13.2`. and exclude those items from the prepared groups. Backend-neutral `prepare_batch` and `prepare_batch_from_images` keep their existing behavior: decoding them on the CPU can still fail the whole call when one - image's output exceeds the allocation cap. On either preparation path, a - group whose combined output exceeds that cap still fails the whole decode - call. + image's output exceeds the allocation cap. On either preparation path, + a group whose combined output exceeds that cap is decoded as several + result groups, each with an output buffer within the cap. Staged windows are admitted by planned execution memory, and an admission claim that cannot fit beside the shared plan fails instead of waiting indefinitely. Successful output is unchanged. diff --git a/crates/j2k/src/owned_batch/cpu.rs b/crates/j2k/src/owned_batch/cpu.rs index 94ff7ea7..bc36a6d6 100644 --- a/crates/j2k/src/owned_batch/cpu.rs +++ b/crates/j2k/src/owned_batch/cpu.rs @@ -347,8 +347,8 @@ impl CpuBatchDecoder { /// /// Inputs whose output allocation or planned execution exceeds a memory /// limit are recorded as indexed preparation errors in the returned batch. - /// Preparation checks individual images; a group's combined output - /// allocation is checked during decoding. + /// Preparation checks individual images. During decoding, a group too + /// large for one output allocation is split into smaller result groups. pub fn prepare( &self, inputs: Vec, diff --git a/xtask/release-evidence/public-api/public-api-review-0.13.2.yml b/xtask/release-evidence/public-api/public-api-review-0.13.2.yml index a5536d87..f256ed65 100644 --- a/xtask/release-evidence/public-api/public-api-review-0.13.2.yml +++ b/xtask/release-evidence/public-api/public-api-review-0.13.2.yml @@ -84,10 +84,10 @@ reviews: j2k-native: removed_fingerprint: "none" added_fingerprint: "fnv1a64:22b943651a07318e" - hidden_count: 586 - hidden_fingerprint: "fnv1a64:733f3aa607999371" + hidden_count: 595 + hidden_fingerprint: "fnv1a64:0d8774f02739bc1e" rationale: "Compared the complete ordinary API inventory against v0.13.1. Adds DecodeErrorClass::ResourceLimit { requested, cap } to an already non-exhaustive enum; no existing item changes and existing variant discriminants are preserved. AllocationTooLarge and ImageTooLarge now classify as ResourceLimit instead of the generic Backend class. This is a compatible refinement of the extensible error taxonomy: invalid-input and allocator-failure classes stay unchanged. Callers that want to handle memory limits separately can match the new variant; existing wildcard matches remain valid." - hidden_rationale: "Adds referenced_classic_plan_execution_bytes and referenced_htj2k_plan_execution_bytes for sibling CPU batch adapters to check planned live execution memory before allocation. Both helpers return Result and do not alter existing signatures. Every other hidden item is unchanged from v0.13.1." + hidden_rationale: "Adds referenced_classic_plan_execution_bytes and referenced_htj2k_plan_execution_bytes for sibling CPU batch adapters to check planned live execution memory before allocation. For the facade's streamed routes, j2k-native also adds decode_native_region_components_rows_with_context, full_decode_planes_floor, output_tile_rects, region_decode_streams_rows and streamed_rows_native_precision; cpu_decode_pool_threads reports the pool thread count for facade routing, DecoderContext::set_shared_pool marks CpuBatchDecoder's shared-pool images, and pack_rgb8 shares the existing pixel conversion. ComponentRowSink::rgb8_region is an optional default method, so existing sink implementations remain valid. These are unsupported implementation helpers; existing signatures are unchanged." j2k-jpeg: removed_fingerprint: "none" added_fingerprint: "none" @@ -105,10 +105,10 @@ reviews: j2k: removed_fingerprint: "none" added_fingerprint: "fnv1a64:78b9622111db3437" - hidden_count: 108 - hidden_fingerprint: "fnv1a64:01e74ad6c3ff416e" - rationale: "Compared the complete ordinary API inventory against v0.13.1. Adds J2kError::ResourceLimit { requested, cap, source } to an already non-exhaustive enum; no existing item changes and existing variant discriminants are preserved. Budget and image-size failures now use this variant instead of NativeDecode, retaining the structured native cause. Callers specifically handling budget failures through NativeDecode should also handle ResourceLimit. Batches prepared by CpuBatchDecoder report items whose output allocation or planned execution cannot fit as indexed preparation errors and exclude those items from the prepared groups. Backend-neutral prepare_batch and prepare_batch_from_images retain whole-call output-cap failures for a single oversized image when subsequently decoded on the CPU. On either preparation path, a group whose combined output exceeds the allocation cap still fails the whole decode call. These correct admission timing within the CPU-owned preparation paths and preserve the existing per-item failure model; successful decode output and signatures are unchanged. Adds BackendErrorKind::Decode at the end of its existing non-exhaustive enum and BackendError::decode for failures during backend decode execution. Existing discriminants and constructors are preserved. The public error documentation and changelog describe the refinements." - hidden_rationale: "Compared the complete hidden API inventory against v0.13.1; it is unchanged." + hidden_count: 109 + hidden_fingerprint: "fnv1a64:0427dd567531c8dd" + rationale: "Compared the complete ordinary API inventory against v0.13.1. Adds J2kError::ResourceLimit { requested, cap, source } to an already non-exhaustive enum; no existing item changes and existing variant discriminants are preserved. Budget and image-size failures now use this variant instead of NativeDecode, retaining the structured native cause. Callers specifically handling budget failures through NativeDecode should also handle ResourceLimit. Batches prepared by CpuBatchDecoder report items whose output allocation or planned execution cannot fit as indexed preparation errors and exclude those items from the prepared groups. Backend-neutral prepare_batch and prepare_batch_from_images retain whole-call output-cap failures for a single oversized image when subsequently decoded on the CPU. On either preparation path, groups whose combined output exceeds the allocation cap are decoded into several result groups, each with a buffer within the cap. Automatic streamed routes also allow requests that previously exceeded working-memory limits to succeed. The documented deep-sample conversion fixes make 16-bit outputs of images deeper than 16 bits hold the native sample value clamped to the 16-bit range (two's complement when signed), as scaled decodes already did, and quantize 8-bit outputs over the full component precision, correcting the prior wider-sample byte interpretation and 16-bit scaling ceiling. These preserve the output contracts and ordinary decode signatures. Adds BackendErrorKind::Decode at the end of its existing non-exhaustive enum and BackendError::decode for failures during backend decode execution. Existing discriminants and constructors are preserved. The public error documentation and changelog describe the refinements." + hidden_rationale: "Adds set_full_decode_planes_budget as an unsupported test hook for isolated integration tests of automatic planes/streamed routing. Existing public decode methods continue choosing their routes automatically; no supported user configuration is added. All other hidden facade items are unchanged." j2k-transcode: removed_fingerprint: "none" added_fingerprint: "none" diff --git a/xtask/release-evidence/public-api/reviewed-public-api-diff-0.13.2.md b/xtask/release-evidence/public-api/reviewed-public-api-diff-0.13.2.md index 2fcd0488..4815642d 100644 --- a/xtask/release-evidence/public-api/reviewed-public-api-diff-0.13.2.md +++ b/xtask/release-evidence/public-api/reviewed-public-api-diff-0.13.2.md @@ -22,10 +22,10 @@ This report is generated by `cargo xtask semver --write-report`. Normal `cargo x | `j2k-cuda-transcode-engine` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 84 | `fnv1a64:b2fcaa9b0171aba7` | | `j2k-metal-support` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 11 | `fnv1a64:02139606f1202938` | | `j2k-mpsgraph-support` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 0 | `none` | -| `j2k-native` | `0.13.1` | `0.13.2` | `minor` | 3 | 0 | `none` | `fnv1a64:22b943651a07318e` | 586 | `fnv1a64:733f3aa607999371` | +| `j2k-native` | `0.13.1` | `0.13.2` | `minor` | 3 | 0 | `none` | `fnv1a64:22b943651a07318e` | 595 | `fnv1a64:0d8774f02739bc1e` | | `j2k-jpeg` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 701 | `fnv1a64:bd0cf738a0f96592` | | `j2k-tilecodec` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 32 | `fnv1a64:eb677842559d9b0e` | -| `j2k` | `0.13.1` | `0.13.2` | `minor` | 6 | 0 | `none` | `fnv1a64:78b9622111db3437` | 108 | `fnv1a64:01e74ad6c3ff416e` | +| `j2k` | `0.13.1` | `0.13.2` | `minor` | 6 | 0 | `none` | `fnv1a64:78b9622111db3437` | 109 | `fnv1a64:0427dd567531c8dd` | | `j2k-transcode` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 458 | `fnv1a64:449a76476549c1e2` | | `j2k-transcode-cuda` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 20 | `fnv1a64:be22ca1d5e12b043` | | `j2k-jpeg-metal` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 83 | `fnv1a64:79397ffc221464c3` | @@ -174,7 +174,7 @@ None. ### `j2k-native` -Baseline items: 422. Candidate items: 425. Computed release type: `minor`. Rustdoc-hidden candidate items: 586. Full hidden-inventory fingerprint: `fnv1a64:733f3aa607999371`. +Baseline items: 422. Candidate items: 425. Computed release type: `minor`. Rustdoc-hidden candidate items: 595. Full hidden-inventory fingerprint: `fnv1a64:0d8774f02739bc1e`. #### Removed or changed baseline API items @@ -214,7 +214,7 @@ None. ### `j2k` -Baseline items: 804. Candidate items: 810. Computed release type: `minor`. Rustdoc-hidden candidate items: 108. Full hidden-inventory fingerprint: `fnv1a64:01e74ad6c3ff416e`. +Baseline items: 804. Candidate items: 810. Computed release type: `minor`. Rustdoc-hidden candidate items: 109. Full hidden-inventory fingerprint: `fnv1a64:0427dd567531c8dd`. #### Removed or changed baseline API items From 01333628095e98560ee88cabdc7d72aa7d08825c Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 18:45:24 -0500 Subject: [PATCH 94/94] refactor(j2k): consolidate decode planning and output helpers Share record sizing, tile bounds, streamed line planning, classic row reconstruction, HT scratch preparation, strip execution and typed row-sink plumbing. Avoid flat VLC reservation when SIMD is disabled while preserving the specialized kernels and public APIs. --- CHANGELOG.md | 2 +- .../src/direct_cpu/allocation/referenced.rs | 216 ++++++++---------- crates/j2k-native/src/image/output_api.rs | 46 +--- crates/j2k-native/src/j2c/bitplane.rs | 1 - .../src/j2c/bitplane/reconstruction.rs | 28 +++ crates/j2k-native/src/j2c/build.rs | 1 + crates/j2k-native/src/j2c/build/allocation.rs | 55 ++++- .../src/j2c/build/allocation/plan.rs | 10 +- .../src/j2c/build/allocation/tests.rs | 15 +- .../j2k-native/src/j2c/decode/banded_tier1.rs | 130 ++++------- .../src/j2c/decode/parallel_bands.rs | 72 ++---- .../j2k-native/src/j2c/decode/planes_floor.rs | 173 +++++--------- .../src/j2c/decode/subband/classic.rs | 30 +-- .../src/j2c/ht_block_decode/magnitude.rs | 52 ++--- .../j2c/ht_block_decode/pipeline/scratch.rs | 10 +- .../src/j2c/ht_block_decode/readers.rs | 12 + .../src/j2c/ht_block_decode/tests.rs | 4 + crates/j2k-native/src/j2c/idwt/banded.rs | 158 +++++-------- crates/j2k-native/src/j2c/mod.rs | 3 +- crates/j2k-native/src/j2c/tile.rs | 87 +++---- crates/j2k-native/src/scalar.rs | 13 +- .../j2k-native/src/scalar/classic_decode.rs | 36 +-- crates/j2k/src/decode/output.rs | 109 ++++++++- crates/j2k/src/decode/output/u16.rs | 72 +----- crates/j2k/src/decode/output/u8.rs | 83 +------ 25 files changed, 615 insertions(+), 803 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1df411a3..3d429fd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -290,7 +290,7 @@ Staged workspace version: `0.13.2`. (a 4096 x 2048 RGB image at 2 bpp 1% faster on 6 threads and 4-5% on 18, in about half the memory); and full-resolution images with a tile for every thread. Those now decode their tiles at once on the calling thread - and the pool's threads, each tile on one thread, instead of one tile after + and scoped worker threads, each tile on one thread, instead of one tile after another: a 4096 x 8192 RGB image of 1024 x 1024 tiles decodes 8% faster than on the planes with 6 threads and 23% faster with 18, in less than half the memory. Tiled classic images with fewer tiles stream them one after another diff --git a/crates/j2k-native/src/direct_cpu/allocation/referenced.rs b/crates/j2k-native/src/direct_cpu/allocation/referenced.rs index e40c2f44..a7603445 100644 --- a/crates/j2k-native/src/direct_cpu/allocation/referenced.rs +++ b/crates/j2k-native/src/direct_cpu/allocation/referenced.rs @@ -20,20 +20,89 @@ use view::{ ReferencedPlanView, }; +/// Accounting inputs shared by an estimator and the matching reservation. +#[derive(Clone, Copy)] +struct ReferencedRequirements<'a> { + plan: ReferencedPlanView<'a>, + parallel: bool, + retained_plan_bytes: usize, + compressed_payload_bytes: usize, + retained_classic_workspace_dimensions: Option<(u32, u32)>, + retained_ht_workspace_dimensions: Option<(u32, u32)>, + classic_tasks: Option, +} + +impl<'a> ReferencedRequirements<'a> { + fn htj2k(plan: &'a J2kReferencedHtj2kPlan, parallel: bool) -> Result { + let view = ReferencedPlanView::Htj2k(plan); + Ok(Self { + plan: view, + parallel, + retained_plan_bytes: plan.retained_allocation_bytes()?, + compressed_payload_bytes: max_referenced_payload_bytes(plan)?, + retained_classic_workspace_dimensions: max_referenced_dimensions(view, false), + retained_ht_workspace_dimensions: max_referenced_dimensions(view, true), + classic_tasks: None, + }) + } + + fn classic(plan: &'a J2kReferencedClassicPlan, parallel: bool) -> Result { + let dimensions = max_referenced_classic_dimensions(plan); + Ok(Self { + plan: ReferencedPlanView::Classic(plan), + parallel, + retained_plan_bytes: plan.retained_allocation_bytes()?, + compressed_payload_bytes: max_referenced_classic_payload_bytes(plan), + retained_classic_workspace_dimensions: dimensions, + retained_ht_workspace_dimensions: None, + classic_tasks: classic_task_shape(plan), + }) + } + + fn staged_htj2k(plan: &'a J2kReferencedHtj2kPlan) -> Result { + Ok(Self { + plan: ReferencedPlanView::Htj2k(plan), + parallel: false, + retained_plan_bytes: plan.retained_allocation_bytes()?, + compressed_payload_bytes: 0, + retained_classic_workspace_dimensions: None, + retained_ht_workspace_dimensions: None, + classic_tasks: None, + }) + } + + fn staged_classic(plan: &'a J2kReferencedClassicPlan) -> Result { + Ok(Self { + plan: ReferencedPlanView::Classic(plan), + parallel: false, + retained_plan_bytes: plan.retained_allocation_bytes()?, + compressed_payload_bytes: 0, + retained_classic_workspace_dimensions: None, + retained_ht_workspace_dimensions: None, + classic_tasks: None, + }) + } + + fn validate(self, scratch: &J2kDirectCpuScratch) -> Result { + validate_referenced_aggregate_plan( + self.plan, + self.parallel, + self.retained_plan_bytes, + self.compressed_payload_bytes, + self.retained_classic_workspace_dimensions, + self.retained_ht_workspace_dimensions, + self.classic_tasks, + scratch, + ) + } +} + pub(in super::super) fn prepare_referenced_direct_scratch( plan: &J2kReferencedHtj2kPlan, scratch: &mut J2kDirectCpuScratch, ) -> Result { - let view = ReferencedPlanView::Htj2k(plan); - let parallel = scratch.parallel; prepare_referenced_component_scratch( - view, - parallel, - plan.retained_allocation_bytes()?, - max_referenced_payload_bytes(plan)?, - max_referenced_dimensions(view, false), - max_referenced_dimensions(view, true), - None, + ReferencedRequirements::htj2k(plan, scratch.parallel)?, scratch, ) } @@ -44,20 +113,11 @@ pub(in super::super) fn planned_referenced_htj2k_execution_bytes( plan: &J2kReferencedHtj2kPlan, parallel: bool, ) -> Result { - let view = ReferencedPlanView::Htj2k(plan); let mut scratch = J2kDirectCpuScratch::new(); scratch.parallel = parallel; - validate_referenced_aggregate_plan( - view, - parallel, - plan.retained_allocation_bytes()?, - max_referenced_payload_bytes(plan)?, - max_referenced_dimensions(view, false), - max_referenced_dimensions(view, true), - None, - &scratch, - ) - .map(DirectWorkspaceBudget::peak_bytes) + ReferencedRequirements::htj2k(plan, parallel)? + .validate(&scratch) + .map(DirectWorkspaceBudget::peak_bytes) } /// Planned peak bytes of one referenced classic execution into empty @@ -68,18 +128,9 @@ pub(in super::super) fn planned_referenced_classic_execution_bytes( ) -> Result { let mut scratch = J2kDirectCpuScratch::new(); scratch.parallel = parallel; - let classic_dimensions = max_referenced_classic_dimensions(plan); - validate_referenced_aggregate_plan( - ReferencedPlanView::Classic(plan), - parallel, - plan.retained_allocation_bytes()?, - max_referenced_classic_payload_bytes(plan), - classic_dimensions, - None, - classic_task_shape(plan), - &scratch, - ) - .map(DirectWorkspaceBudget::peak_bytes) + ReferencedRequirements::classic(plan, parallel)? + .validate(&scratch) + .map(DirectWorkspaceBudget::peak_bytes) } /// Shape of the extra classic workspaces parallel execution of `plan` uses. @@ -95,18 +146,8 @@ pub(in super::super) fn prepare_referenced_classic_scratch( plan: &J2kReferencedClassicPlan, scratch: &mut J2kDirectCpuScratch, ) -> Result { - let view = ReferencedPlanView::Classic(plan); - let parallel = scratch.parallel; - let classic_dimensions = max_referenced_classic_dimensions(plan); - let classic_tasks = classic_task_shape(plan); prepare_referenced_component_scratch( - view, - parallel, - plan.retained_allocation_bytes()?, - max_referenced_classic_payload_bytes(plan), - classic_dimensions, - None, - classic_tasks, + ReferencedRequirements::classic(plan, scratch.parallel)?, scratch, ) } @@ -115,66 +156,29 @@ pub(in super::super) fn prepare_referenced_htj2k_staged_scratch( plan: &J2kReferencedHtj2kPlan, scratch: &mut J2kDirectCpuScratch, ) -> Result { - prepare_referenced_component_scratch( - ReferencedPlanView::Htj2k(plan), - false, - plan.retained_allocation_bytes()?, - 0, - None, - None, - None, - scratch, - ) + prepare_referenced_component_scratch(ReferencedRequirements::staged_htj2k(plan)?, scratch) } pub(in super::super) fn prepare_referenced_classic_staged_scratch( plan: &J2kReferencedClassicPlan, scratch: &mut J2kDirectCpuScratch, ) -> Result { - prepare_referenced_component_scratch( - ReferencedPlanView::Classic(plan), - false, - plan.retained_allocation_bytes()?, - 0, - None, - None, - None, - scratch, - ) + prepare_referenced_component_scratch(ReferencedRequirements::staged_classic(plan)?, scratch) } -#[expect( - clippy::too_many_arguments, - reason = "normalization, accounting, and reservation share every retained owner shape explicitly" -)] fn prepare_referenced_component_scratch( - plan: ReferencedPlanView<'_>, - parallel: bool, - retained_plan_bytes: usize, - compressed_payload_bytes: usize, - retained_classic_workspace_dimensions: Option<(u32, u32)>, - retained_ht_workspace_dimensions: Option<(u32, u32)>, - classic_tasks: Option, + requirements: ReferencedRequirements<'_>, scratch: &mut J2kDirectCpuScratch, ) -> Result { normalize_referenced_scratch( - plan, - compressed_payload_bytes, - retained_classic_workspace_dimensions.is_some(), - retained_ht_workspace_dimensions.is_some(), - classic_tasks.is_some(), + requirements.plan, + requirements.compressed_payload_bytes, + requirements.retained_classic_workspace_dimensions.is_some(), + requirements.retained_ht_workspace_dimensions.is_some(), + requirements.classic_tasks.is_some(), scratch, )?; - if let Err(error) = validate_referenced_aggregate_plan( - plan, - parallel, - retained_plan_bytes, - compressed_payload_bytes, - retained_classic_workspace_dimensions, - retained_ht_workspace_dimensions, - classic_tasks, - scratch, - ) { + if let Err(error) = requirements.validate(scratch) { if !matches!( error, DecodeError::Validation(ValidationError::ImageTooLarge) @@ -182,39 +186,21 @@ fn prepare_referenced_component_scratch( return Err(error); } scratch.clear(); - validate_referenced_aggregate_plan( - plan, - parallel, - retained_plan_bytes, - compressed_payload_bytes, - retained_classic_workspace_dimensions, - retained_ht_workspace_dimensions, - classic_tasks, - scratch, - )?; + requirements.validate(scratch)?; } if let Err(error) = reserve_referenced_scratch( - plan, - compressed_payload_bytes, - retained_classic_workspace_dimensions, - retained_ht_workspace_dimensions, - classic_tasks, + requirements.plan, + requirements.compressed_payload_bytes, + requirements.retained_classic_workspace_dimensions, + requirements.retained_ht_workspace_dimensions, + requirements.classic_tasks, scratch, ) { scratch.clear(); return Err(error); } - match validate_referenced_aggregate_plan( - plan, - parallel, - retained_plan_bytes, - compressed_payload_bytes, - retained_classic_workspace_dimensions, - retained_ht_workspace_dimensions, - classic_tasks, - scratch, - ) { + match requirements.validate(scratch) { Ok(workspace_budget) => Ok(workspace_budget), Err(error) => { scratch.clear(); diff --git a/crates/j2k-native/src/image/output_api.rs b/crates/j2k-native/src/image/output_api.rs index 478d9fdc..22ad2aa0 100644 --- a/crates/j2k-native/src/image/output_api.rs +++ b/crates/j2k-native/src/image/output_api.rs @@ -593,47 +593,17 @@ impl<'a> Image<'a> { size.checked_x_shrink_factor()?, size.checked_y_shrink_factor()?, ); - let (x_origin, y_origin) = ( - size.image_area_x_offset.div_ceil(x_shrink), - size.image_area_y_offset.div_ceil(y_shrink), - ); - // One axis of tile `index` on the shrunk grid, relative to the image. - let span = |index: u32, - offset: u32, - tile: u32, - area: u32, - extent: u32, - shrink: u32, - origin: u32| { - let start = offset.saturating_add(index.saturating_mul(tile)).max(area); - let end = offset - .saturating_add(index.saturating_add(1).saturating_mul(tile)) - .min(extent); - let start = start.div_ceil(shrink).saturating_sub(origin); - let end = end.div_ceil(shrink).saturating_sub(origin); - (start, end.saturating_sub(start)) - }; + let (x_origin, y_origin) = j2c::output_grid_offset(size); let mut rects = Vec::new(); for ty in 0..size.num_y_tiles() { - let (y, height) = span( - ty, - size.tile_y_offset, - size.tile_height, - size.image_area_y_offset, - size.reference_grid_height, - y_shrink, - y_origin, - ); for tx in 0..size.num_x_tiles() { - let (x, width) = span( - tx, - size.tile_x_offset, - size.tile_width, - size.image_area_x_offset, - size.reference_grid_width, - x_shrink, - x_origin, - ); + let rect = j2c::tile_rect(size, tx, ty); + let x = rect.x0.div_ceil(x_shrink).saturating_sub(x_origin); + let y = rect.y0.div_ceil(y_shrink).saturating_sub(y_origin); + let x1 = rect.x1.div_ceil(x_shrink).saturating_sub(x_origin); + let y1 = rect.y1.div_ceil(y_shrink).saturating_sub(y_origin); + let width = x1.saturating_sub(x); + let height = y1.saturating_sub(y); if width > 0 && height > 0 { rects.push((x, y, width, height)); } diff --git a/crates/j2k-native/src/j2c/bitplane.rs b/crates/j2k-native/src/j2c/bitplane.rs index 6bdd2d7c..d46a85e4 100644 --- a/crates/j2k-native/src/j2c/bitplane.rs +++ b/crates/j2k-native/src/j2c/bitplane.rs @@ -16,7 +16,6 @@ pub(crate) use facade::{ decode, decode_code_block_segments_validated, decode_code_block_segments_validated_profiled, }; pub(crate) use observer::J2kBlockDecodeStats; -pub(crate) use reconstruction::MidpointReconstructor; #[cfg(test)] pub(crate) use state::FORCE_PER_COEFFICIENT; pub(crate) use state::{ diff --git a/crates/j2k-native/src/j2c/bitplane/reconstruction.rs b/crates/j2k-native/src/j2c/bitplane/reconstruction.rs index fd2898e1..813f72f9 100644 --- a/crates/j2k-native/src/j2c/bitplane/reconstruction.rs +++ b/crates/j2k-native/src/j2c/bitplane/reconstruction.rs @@ -84,6 +84,34 @@ impl MidpointReconstructor { } } + /// Convert a decoded row with the block's midpoint, ROI and scaling policy. + #[inline] + #[expect( + clippy::cast_precision_loss, + reason = "reconstructed integer coefficients enter the codec f32 domain" + )] + pub(crate) fn reconstruct_row( + self, + coefficients: &[Coefficient], + output: &mut [f32], + step: f32, + irreversible_midpoint: bool, + ) { + if irreversible_midpoint { + self.dequantize_row(coefficients, output, step); + } else if self.roi_shift == 0 && self.reversible_is_exact() { + Self::scale_exact_reversible_row(coefficients, output, step); + } else { + for (sample, &coefficient) in output.iter_mut().zip(coefficients) { + *sample = crate::apply_roi_maxshift_inverse_i64( + self.reconstruct_reversible(coefficient), + self.roi_shift, + ) as f32 + * step; + } + } + } + /// Reconstruct and dequantize a row: `output[i] = reconstruct(c[i]) * step` /// for as many elements as both slices hold. /// diff --git a/crates/j2k-native/src/j2c/build.rs b/crates/j2k-native/src/j2c/build.rs index f07a88fe..8a2e010d 100644 --- a/crates/j2k-native/src/j2c/build.rs +++ b/crates/j2k-native/src/j2c/build.rs @@ -19,6 +19,7 @@ use allocation::{ code_block_grid, prepare_decomposition_storage, push_preallocated, tag_tree_node_count, DecompositionAllocationPlan, }; +pub(in crate::j2c) use allocation::{global_code_block_count, IdwtWorkspaceTracker, RecordCounts}; pub(crate) use selection::{select_region_precincts, PrecinctSelection, PrecinctUse}; /// Build and allocate all necessary structures to process the code-blocks diff --git a/crates/j2k-native/src/j2c/build/allocation.rs b/crates/j2k-native/src/j2c/build/allocation.rs index 378dedf9..e78ad269 100644 --- a/crates/j2k-native/src/j2c/build/allocation.rs +++ b/crates/j2k-native/src/j2c/build/allocation.rs @@ -15,6 +15,7 @@ mod plan; mod reuse; pub(super) use plan::{code_block_grid, tag_tree_node_count}; +pub(in crate::j2c) use plan::{global_code_block_count, IdwtWorkspaceTracker}; pub(super) use reuse::push_preallocated; /// How sub-band coefficient ranges share [`DecompositionStorage::coefficients`]. @@ -62,6 +63,40 @@ impl BuildWorkspace { } } +/// Logical counts of the records every built tile owns. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(in crate::j2c) struct RecordCounts { + pub(in crate::j2c) tile_decompositions: usize, + pub(in crate::j2c) decompositions: usize, + pub(in crate::j2c) sub_bands: usize, + pub(in crate::j2c) precincts: usize, + pub(in crate::j2c) code_blocks: usize, + pub(in crate::j2c) layers: usize, + pub(in crate::j2c) tag_tree_nodes: usize, +} + +impl RecordCounts { + /// These records' exact bytes, independent of the decode allocation cap. + pub(in crate::j2c) fn checked_bytes(self) -> Result { + let mut total = 0usize; + let mut include = |size: usize, count: usize| -> Result<()> { + total = size + .checked_mul(count) + .and_then(|bytes| total.checked_add(bytes)) + .ok_or(ValidationError::ImageTooLarge)?; + Ok(()) + }; + include(size_of::(), self.tile_decompositions)?; + include(size_of::(), self.decompositions)?; + include(size_of::(), self.sub_bands)?; + include(size_of::(), self.precincts)?; + include(size_of::(), self.code_blocks)?; + include(size_of::(), self.layers)?; + include(size_of::(), self.tag_tree_nodes)?; + Ok(total) + } +} + pub(super) struct DecompositionAllocationPlan { retained_baseline_bytes: usize, pub(super) total_bytes: usize, @@ -166,6 +201,18 @@ impl DecompositionAllocationPlan { Ok(()) } + fn record_counts(&self) -> RecordCounts { + RecordCounts { + tile_decompositions: self.tile_decompositions, + decompositions: self.decompositions, + sub_bands: self.sub_bands, + precincts: self.precincts, + code_blocks: self.code_blocks, + layers: self.layers, + tag_tree_nodes: self.tag_tree_nodes, + } + } + /// Charge the unbuilt precincts' records and tag-tree roots. fn include_unbuilt_precincts(&self, total_bytes: &mut usize) -> Result<()> { checked_include_elements::(total_bytes, self.unbuilt_precincts)?; @@ -179,14 +226,8 @@ impl DecompositionAllocationPlan { /// Reject a logically impossible request before walking every precinct. fn validate_minimum_live_workspace(&self, idwt_workspace_elements: usize) -> Result<()> { let mut total_bytes = self.retained_baseline_bytes; - checked_include_elements::(&mut total_bytes, self.tile_decompositions)?; - checked_include_elements::(&mut total_bytes, self.decompositions)?; - checked_include_elements::(&mut total_bytes, self.sub_bands)?; - checked_include_elements::(&mut total_bytes, self.precincts)?; - checked_include_elements::(&mut total_bytes, self.code_blocks)?; - checked_include_elements::(&mut total_bytes, self.layers)?; + checked_include_elements::(&mut total_bytes, self.record_counts().checked_bytes()?)?; checked_include_elements::(&mut total_bytes, self.parse_only_layers)?; - checked_include_elements::(&mut total_bytes, self.tag_tree_nodes)?; checked_include_elements::(&mut total_bytes, self.parse_only_code_blocks)?; checked_include_elements::(&mut total_bytes, self.parse_only_tag_tree_nodes)?; self.include_unbuilt_precincts(&mut total_bytes)?; diff --git a/crates/j2k-native/src/j2c/build/allocation/plan.rs b/crates/j2k-native/src/j2c/build/allocation/plan.rs index 663394d9..f1a0a4e4 100644 --- a/crates/j2k-native/src/j2c/build/allocation/plan.rs +++ b/crates/j2k-native/src/j2c/build/allocation/plan.rs @@ -318,7 +318,7 @@ fn include_code_blocks( DecompositionAllocationPlan::checked_add_count(&mut plan.layers, layer_count) } -fn global_code_block_count( +pub(in crate::j2c) fn global_code_block_count( resolution_tile: &ResolutionTile<'_>, sub_band_rect: IntRect, ) -> Result { @@ -453,7 +453,7 @@ fn checked_rect_elements(rect: IntRect) -> Result { } #[derive(Clone, Copy)] -pub(super) struct IdwtWorkspaceTracker { +pub(in crate::j2c) struct IdwtWorkspaceTracker { active_decomposition_count: Option, transform: WaveletTransform, observed_count: usize, @@ -463,7 +463,7 @@ pub(super) struct IdwtWorkspaceTracker { } impl IdwtWorkspaceTracker { - pub(super) fn new( + pub(in crate::j2c) fn new( active_decomposition_count: Option, transform: WaveletTransform, ) -> Self { @@ -477,7 +477,7 @@ impl IdwtWorkspaceTracker { } } - pub(super) fn observe(&mut self, rect: IntRect) { + pub(in crate::j2c) fn observe(&mut self, rect: IntRect) { if self .active_decomposition_count .is_some_and(|count| self.observed_count < count) @@ -489,7 +489,7 @@ impl IdwtWorkspaceTracker { self.observed_count = self.observed_count.saturating_add(1); } - pub(super) fn finish(self, ll_rect: IntRect) -> Result> { + pub(in crate::j2c) fn finish(self, ll_rect: IntRect) -> Result> { if self.active_decomposition_count.is_none() { return Ok(None); } diff --git a/crates/j2k-native/src/j2c/build/allocation/tests.rs b/crates/j2k-native/src/j2c/build/allocation/tests.rs index d2b20a7c..2d0005ca 100644 --- a/crates/j2k-native/src/j2c/build/allocation/tests.rs +++ b/crates/j2k-native/src/j2c/build/allocation/tests.rs @@ -4,7 +4,7 @@ use super::plan::{tag_tree_node_count, IdwtWorkspaceTracker}; use super::reuse::{discard_stale_capacity, ReallocationBudget}; use super::{ release_unused_roi_workspace, roi_workspace_bytes, BuildWorkspace, DecompositionAllocationPlan, - Segment, DEFAULT_MAX_DECODE_BYTES, + RecordCounts, Segment, DEFAULT_MAX_DECODE_BYTES, }; use crate::error::{DecodeError, ValidationError}; use crate::j2c::build::{Decomposition, SubBand, SubBandType}; @@ -24,6 +24,19 @@ fn coefficient_plan(baseline: usize, coefficients: usize) -> DecompositionAlloca plan } +#[test] +fn logical_record_bytes_are_independent_of_the_decode_cap() { + let count = DEFAULT_MAX_DECODE_BYTES / size_of::() + 1; + let bytes = RecordCounts { + sub_bands: count, + ..RecordCounts::default() + } + .checked_bytes() + .expect("record byte count fits usize"); + assert_eq!(bytes, count * size_of::()); + assert!(bytes > DEFAULT_MAX_DECODE_BYTES); +} + #[test] fn metadata_plan_retains_logical_samples_without_allocating_pixel_storage() { let mut storage = DecompositionStorage { diff --git a/crates/j2k-native/src/j2c/decode/banded_tier1.rs b/crates/j2k-native/src/j2c/decode/banded_tier1.rs index 08463f0e..320e3467 100644 --- a/crates/j2k-native/src/j2c/decode/banded_tier1.rs +++ b/crates/j2k-native/src/j2c/decode/banded_tier1.rs @@ -161,7 +161,9 @@ fn stream_rows<'a>( tier1_us: &mut u128, ) -> Result<()> { let mut bands = Vec::new(); - let line_plans = plan_tile_lines(tile, header, tile_ctx, storage, &mut bands)?; + let active_levels = (0..tile.component_infos.len()) + .map(|component| push_component_bands(storage, header, component, &mut bands)); + let line_plans = plan_tile_lines(tile, header, tile_ctx, storage, active_levels)?; let band_of = band_lookup(&bands)?; let empty = IntRect::from_xywh(0, 0, 0, 0); let (copy, region) = line_plans @@ -180,16 +182,7 @@ fn stream_rows<'a>( let mut output = RowOutput::new(tile, sink, tile_ctx, copy, region); let mut output_samples = 0usize; let result = (|| { - let mut rings = &mut buffer[..]; - let mut lines = Vec::new(); - lines - .try_reserve_exact(line_plans.len()) - .map_err(|_| DecodingError::HostAllocationFailed)?; - for (plan, len) in line_plans.iter().zip(&ring_lens) { - let (ring, tail) = core::mem::take(&mut rings).split_at_mut(*len); - rings = tail; - lines.push(plan.reconstruction(ring)?); - } + let mut lines = line_reconstructions(&line_plans, &ring_lens, &mut buffer)?; let mut rows = BandedRows { header, component_infos: &tile.component_infos, @@ -218,40 +211,40 @@ fn stream_rows<'a>( result } -/// Add every component's bands to `bands` and plan its reconstruction. -/// Returns no plans when no component has rows to reconstruct. -fn plan_tile_lines( +/// Plan every component's streamed reconstruction of its active levels. +/// No plans when no component has rows to reconstruct; every component of +/// a 1x1-sampled tile reconstructs the same rows. +/// Pull each component's active levels before constructing its line plan, +/// preserving the serial caller's band-push and error order. +pub(super) fn plan_tile_lines( tile: &Tile<'_>, header: &Header<'_>, tile_ctx: &TileDecodeContext, storage: &DecompositionStorage<'_>, - bands: &mut Vec, + active_levels: impl ExactSizeIterator>, ) -> Result> { let mut line_plans = Vec::new(); line_plans - .try_reserve_exact(tile.component_infos.len()) + .try_reserve_exact(active_levels.len()) .map_err(|_| DecodingError::HostAllocationFailed)?; - let mut missing = 0; - for (component_idx, component_info) in tile.component_infos.iter().enumerate() { - let active_levels = push_component_bands(storage, header, component_idx, bands)?; - match LinePlan::new( + for (component_idx, (component_info, levels)) in + tile.component_infos.iter().zip(active_levels).enumerate() + { + if let Some(plan) = LinePlan::new( storage, tile_ctx, component_idx, - active_levels, + levels?, header, component_info.wavelet_transform(), )? { - Some(plan) => line_plans.push(plan), - None => missing += 1, + line_plans.push(plan); } } - if missing == tile.component_infos.len() { - line_plans.clear(); + if line_plans.is_empty() { return Ok(line_plans); } - // Every component of a 1x1-sampled tile reconstructs the same rows. - if missing > 0 + if line_plans.len() != tile.component_infos.len() || line_plans .iter() .any(|plan| plan.copy != line_plans[0].copy || plan.region != line_plans[0].region) @@ -261,6 +254,24 @@ fn plan_tile_lines( Ok(line_plans) } +/// Build every component's reconstruction in its own slice of the ring buffer. +pub(super) fn line_reconstructions<'a>( + plans: &[LinePlan], + ring_lens: &[usize], + mut rings: &'a mut [f32], +) -> Result>> { + let mut lines = Vec::new(); + lines + .try_reserve_exact(plans.len()) + .map_err(|_| DecodingError::HostAllocationFailed)?; + for (plan, len) in plans.iter().zip(ring_lens) { + let (ring, tail) = core::mem::take(&mut rings).split_at_mut(*len); + rings = tail; + lines.push(plan.reconstruction(ring)?); + } + Ok(lines) +} + /// Take the context's ring buffer, grown to hold every plan's rings and /// `scratch_len` more samples, and return it with each plan's ring length. pub(super) fn take_line_buffer( @@ -370,6 +381,7 @@ impl<'o, 't> RowOutput<'o, 't> { .ok_or(DecodingError::CodeBlockDecodeFailure)?; Ok(Some(Rgb8Rows { out, + #[cfg(feature = "parallel")] stride, x: (copy.x0 - region.x0) as usize * 3, })) @@ -450,10 +462,7 @@ impl<'o, 't> RowOutput<'o, 't> { } } -#[cfg_attr( - not(feature = "parallel"), - expect(dead_code, reason = "only the parallel band reader writes batches") -)] +#[cfg(feature = "parallel")] impl RowOutput<'_, '_> { /// [`Self::write`] through one reader per component, each on its own /// pool task, `batch` rows at a time. Each component's reconstruction @@ -626,10 +635,7 @@ impl RowOutput<'_, '_> { } /// Each component's first `len` samples of its `batch_len` in `buffer`. -#[cfg_attr( - not(feature = "parallel"), - expect(dead_code, reason = "only the parallel band reader writes batches") -)] +#[cfg(feature = "parallel")] fn batch_parts( buffer: &mut [f32], batch_len: usize, @@ -657,30 +663,18 @@ fn batch_parts( /// Run `spawned` on the pool while `inline` runs here, and return once both /// have. -#[cfg_attr( - not(feature = "parallel"), - expect(dead_code, reason = "only the parallel band reader writes batches") -)] +#[cfg(feature = "parallel")] fn overlap(spawned: impl FnOnce() + Send, inline: impl FnOnce()) { - #[cfg(feature = "parallel")] rayon::in_place_scope(|scope| { scope.spawn(move |_| spawned()); inline(); }); - #[cfg(not(feature = "parallel"))] - { - spawned(); - inline(); - } } /// Reconstruct the next `parts[c].len() / width` rows of each component `c`, /// from row `first_y`, into `parts[c]`, through its own reader: on the pool, /// one component per task. -#[cfg_attr( - not(feature = "parallel"), - expect(dead_code, reason = "only the parallel band reader writes batches") -)] +#[cfg(feature = "parallel")] fn reconstruct_batch( lines: &mut [LineReconstruction<'_>], readers: &mut [&mut (dyn SubBandRows + Send)], @@ -692,7 +686,6 @@ fn reconstruct_batch( return Err(DecodingError::CodeBlockDecodeFailure.into()); } let mut results: [Result<()>; 4] = core::array::from_fn(|_| Ok(())); - #[cfg(feature = "parallel")] rayon::in_place_scope(|scope| { let mut tasks = lines .iter_mut() @@ -709,24 +702,12 @@ fn reconstruct_batch( *result = reconstruct_rows_into(line, &mut **reader, part, first_y, width); } }); - #[cfg(not(feature = "parallel"))] - for (((line, reader), part), result) in lines - .iter_mut() - .zip(readers.iter_mut()) - .zip(parts.iter_mut()) - .zip(results.iter_mut()) - { - *result = reconstruct_rows_into(line, &mut **reader, part, first_y, width); - } results.into_iter().collect() } /// The next rows of `line`, from row `first_y`, through `rows`, into `out`, /// `width` samples each. -#[cfg_attr( - not(feature = "parallel"), - expect(dead_code, reason = "only the parallel band reader writes batches") -)] +#[cfg(feature = "parallel")] fn reconstruct_rows_into( line: &mut LineReconstruction<'_>, rows: &mut dyn SubBandRows, @@ -753,25 +734,20 @@ fn reconstruct_rows_into( struct Rgb8Rows<'o> { /// Whole output rows, `stride` bytes each, one per reconstructed row. out: &'o mut [u8], + #[cfg(feature = "parallel")] stride: usize, /// Where the reconstructed samples start in each output row, in bytes. x: usize, } /// One range of a batch's rows: each component's rows and their output. -#[cfg_attr( - not(feature = "parallel"), - expect(dead_code, reason = "only the parallel band reader writes batches") -)] +#[cfg(feature = "parallel")] struct BatchRows<'p, 'o> { parts: [&'p mut [f32]; 4], rgb8: Option>, } -#[cfg_attr( - not(feature = "parallel"), - expect(dead_code, reason = "only the parallel band reader writes batches") -)] +#[cfg(feature = "parallel")] impl BatchRows<'_, '_> { /// Transform the rows one at a time, as [`RowOutput::write`] does, then /// pack them into the RGB8 rows if there are any; with the `mct` of @@ -833,10 +809,7 @@ impl BatchRows<'_, '_> { /// Transform each component's rows in `parts`, `width` samples each, and /// pack them into `rgb8` when given, in up to `tasks` ranges of rows, all but /// the first spawned on the pool. -#[cfg_attr( - not(feature = "parallel"), - expect(dead_code, reason = "only the parallel band reader writes batches") -)] +#[cfg(feature = "parallel")] fn finish_rows( transform: ((&Tile<'_>, bool), Option), parts: &mut [&mut [f32]], @@ -891,7 +864,6 @@ fn finish_rows( }); } let mut results: [Result<()>; 4] = core::array::from_fn(|_| Ok(())); - #[cfg(feature = "parallel")] rayon::in_place_scope(|scope| { let mut jobs = ranges.into_iter().zip(results.iter_mut()); let first = jobs.next(); @@ -904,12 +876,6 @@ fn finish_rows( *result = range.finish(transform, components, width); } }); - #[cfg(not(feature = "parallel"))] - for (range, result) in ranges.into_iter().zip(results.iter_mut()) { - if let Some(range) = range { - *result = range.finish(transform, components, width); - } - } results.into_iter().collect() } diff --git a/crates/j2k-native/src/j2c/decode/parallel_bands.rs b/crates/j2k-native/src/j2c/decode/parallel_bands.rs index 002bf936..d300c8d7 100644 --- a/crates/j2k-native/src/j2c/decode/parallel_bands.rs +++ b/crates/j2k-native/src/j2c/decode/parallel_bands.rs @@ -426,7 +426,13 @@ pub(super) fn decode_tile_rows<'a>( if declines(live.include_bytes(line_lists))? { return Ok(false); } - let line_plans = plan_tile_lines(tile, header, tile_ctx, storage, &active_levels)?; + let line_plans = banded_tier1::plan_tile_lines( + tile, + header, + tile_ctx, + storage, + active_levels.iter().copied().map(Ok), + )?; let empty = IntRect::from_xywh(0, 0, 0, 0); let (copy, region) = line_plans .first() @@ -544,16 +550,8 @@ pub(super) fn decode_tile_rows<'a>( let source = PayloadSource::new(storage); let result = slots.and_then(|slots| { let rings_len = buffer.len() - scratch_len; - let (mut line_rings, scratch) = buffer.split_at_mut(rings_len); - let mut lines = Vec::new(); - lines - .try_reserve_exact(line_plans.len()) - .map_err(|_| DecodingError::HostAllocationFailed)?; - for (plan, len) in line_plans.iter().zip(&ring_lens) { - let (ring, tail) = core::mem::take(&mut line_rings).split_at_mut(*len); - line_rings = tail; - lines.push(plan.reconstruction(ring)?); - } + let (line_rings, scratch) = buffer.split_at_mut(rings_len); + let mut lines = banded_tier1::line_reconstructions(&line_plans, &ring_lens, line_rings)?; let run = |readers: &mut [&mut (dyn SubBandRows + Send)]| -> Result<()> { let written = match readers { [reader] => output.write(&mut lines, &mut **reader), @@ -928,47 +926,6 @@ fn plan_tile_rings( Ok((rings, bands, strip_starts, active_levels)) } -/// Plan every component's streamed reconstruction of its `active_levels`. -/// No plans when no component has rows to reconstruct; every component of -/// a 1x1-sampled tile reconstructs the same rows. -fn plan_tile_lines( - tile: &Tile<'_>, - header: &Header<'_>, - tile_ctx: &TileDecodeContext, - storage: &DecompositionStorage<'_>, - active_levels: &[usize], -) -> Result> { - let mut line_plans = Vec::new(); - line_plans - .try_reserve_exact(active_levels.len()) - .map_err(|_| DecodingError::HostAllocationFailed)?; - for (component_idx, (component_info, levels)) in - tile.component_infos.iter().zip(active_levels).enumerate() - { - if let Some(plan) = LinePlan::new( - storage, - tile_ctx, - component_idx, - *levels, - header, - component_info.wavelet_transform(), - )? { - line_plans.push(plan); - } - } - if line_plans.is_empty() { - return Ok(line_plans); - } - if line_plans.len() != tile.component_infos.len() - || line_plans - .iter() - .any(|plan| plan.copy != line_plans[0].copy || plan.region != line_plans[0].region) - { - return Err(DecodingError::CodeBlockDecodeFailure.into()); - } - Ok(line_plans) -} - /// The bytes the executor holds for `rings` and `bands` while `workers` /// pool threads and `readers` readers decode them, besides the rings and /// bands themselves (see [`ring_plan_bytes`]) and the slots' samples: each @@ -2984,9 +2941,14 @@ mod tests { j2k_alloc_probe::peak_bytes() - base, j2k_alloc_probe::live_bytes() - base, ); - let line_plans = - plan_tile_lines(&tiles[0], &image.header, tile_ctx, storage, &active_levels) - .expect("plan lines"); + let line_plans = banded_tier1::plan_tile_lines( + &tiles[0], + &image.header, + tile_ctx, + storage, + active_levels.iter().copied().map(Ok), + ) + .expect("plan lines"); let copy = line_plans .first() .map_or(IntRect::from_xywh(0, 0, 0, 0), |plan| plan.copy); diff --git a/crates/j2k-native/src/j2c/decode/planes_floor.rs b/crates/j2k-native/src/j2c/decode/planes_floor.rs index 7fde899a..27b2b5a9 100644 --- a/crates/j2k-native/src/j2c/decode/planes_floor.rs +++ b/crates/j2k-native/src/j2c/decode/planes_floor.rs @@ -15,15 +15,11 @@ use core::mem::size_of; -use super::TileDecompositions; use crate::checked_decode_sample_count; use crate::error::{Result, ValidationError}; -use crate::j2c::build::{CodeBlock, Decomposition, Layer, Precinct, SubBand, SubBandType}; +use crate::j2c::build::{global_code_block_count, IdwtWorkspaceTracker, RecordCounts, SubBandType}; use crate::j2c::codestream::Header; -use crate::j2c::codestream::WaveletTransform; -use crate::j2c::idwt::planned_workspace_elements; use crate::j2c::rect::IntRect; -use crate::j2c::tag_tree::TagNode; use crate::j2c::tile::{self, ResolutionTile, Tile}; use crate::j2c::ComponentData; use crate::math::{SimdBuffer, SIMD_WIDTH}; @@ -147,144 +143,91 @@ fn tile_streams_rows(tile: &Tile<'_>, components: usize) -> bool { /// its largest component's `f32` coefficients at full resolution, and the /// largest component IDWT workspace at the output resolution. fn tile_workspace_floor(tile: &Tile<'_>, skipped: u8) -> usize { - let mut records = Records::default(); + let mut records = RecordCounts::default(); let mut coefficients = 0_usize; let mut idwt = 0_usize; for component_tile in tile.component_tiles() { coefficients = coefficients.max(rect_samples(component_tile.rect)); records.tile_decompositions = records.tile_decompositions.saturating_add(1); - let active = component_tile - .component_info - .num_decomposition_levels() - .saturating_sub(skipped); - let mut highest = None; - let mut second_highest = None; - let mut third_highest = None; + let active = usize::from( + component_tile + .component_info + .num_decomposition_levels() + .saturating_sub(skipped), + ); + let mut idwt_tracker = IdwtWorkspaceTracker::new( + Some(active), + component_tile.component_info.wavelet_transform(), + ); let mut ll_rect = IntRect::from_xywh(0, 0, 0, 0); for resolution_tile in component_tile.resolution_tiles() { if resolution_tile.resolution == 0 { ll_rect = resolution_tile.sub_band_rect(SubBandType::LowLow); - records.add_sub_band(&resolution_tile, SubBandType::LowLow, tile.num_layers); + add_sub_band_records( + &mut records, + &resolution_tile, + SubBandType::LowLow, + tile.num_layers, + ); continue; } records.decompositions = records.decompositions.saturating_add(1); - if resolution_tile.resolution <= active { - third_highest = second_highest; - second_highest = highest; - highest = Some(resolution_tile.rect); - } + idwt_tracker.observe(resolution_tile.rect); for sub_band_type in [ SubBandType::HighLow, SubBandType::LowHigh, SubBandType::HighHigh, ] { - records.add_sub_band(&resolution_tile, sub_band_type, tile.num_layers); + add_sub_band_records( + &mut records, + &resolution_tile, + sub_band_type, + tile.num_layers, + ); } } - let transform = component_tile.component_info.wavelet_transform(); - idwt = idwt.max(idwt_workspace( - highest, - [second_highest, third_highest], - ll_rect, - transform, - )); + idwt = idwt.max( + idwt_tracker + .finish(ll_rect) + .ok() + .flatten() + .unwrap_or(usize::MAX), + ); } - records.bytes().saturating_add( + let records_bytes = records.checked_bytes().unwrap_or(usize::MAX); + records_bytes.saturating_add( coefficients .saturating_add(idwt) .saturating_mul(size_of::()), ) } -/// `IdwtWorkspaceTracker::finish`: the IDWT workspace of the highest active -/// level and the two below it (see `idwt::planned_workspace_elements`), or -/// the LL band when no level is active. A size `build` cannot represent is -/// refused there, so it counts as more than any budget. -fn idwt_workspace( - highest: Option, - below: [Option; 2], - ll_rect: IntRect, - transform: WaveletTransform, -) -> usize { - let Some(highest) = highest else { - return rect_samples(ll_rect); - }; - planned_workspace_elements(highest, below, transform).unwrap_or(usize::MAX) -} - fn rect_samples(rect: IntRect) -> usize { (rect.width() as usize).saturating_mul(rect.height() as usize) } -/// Record counts `build` plans for a tile, tag-tree nodes only as the two -/// trees of each precinct need at least: a root, and a leaf per code-block. -#[derive(Default)] -struct Records { - tile_decompositions: usize, - decompositions: usize, - sub_bands: usize, - precincts: usize, - code_blocks: usize, - layers: usize, - tag_tree_nodes: usize, -} - -impl Records { - fn add_sub_band( - &mut self, - resolution_tile: &ResolutionTile<'_>, - sub_band_type: SubBandType, - layers: u8, - ) { - let precincts = usize::try_from(resolution_tile.num_precincts()).unwrap_or(usize::MAX); - let code_blocks = code_block_count( - resolution_tile, - resolution_tile.sub_band_rect(sub_band_type), - ); - self.sub_bands = self.sub_bands.saturating_add(1); - self.precincts = self.precincts.saturating_add(precincts); - self.code_blocks = self.code_blocks.saturating_add(code_blocks); - self.layers = self - .layers - .saturating_add(code_blocks.saturating_mul(usize::from(layers))); - self.tag_tree_nodes = self - .tag_tree_nodes - .saturating_add(precincts.max(code_blocks).saturating_mul(2)); - } - - fn bytes(&self) -> usize { - [ - self.tile_decompositions - .saturating_mul(size_of::()), - self.decompositions - .saturating_mul(size_of::()), - self.sub_bands.saturating_mul(size_of::()), - self.precincts.saturating_mul(size_of::()), - self.code_blocks.saturating_mul(size_of::()), - self.layers.saturating_mul(size_of::()), - self.tag_tree_nodes.saturating_mul(size_of::()), - ] - .into_iter() - .fold(0, usize::saturating_add) - } -} - -/// The code-blocks of a sub-band, as `build` counts them before walking -/// precincts. -fn code_block_count(resolution_tile: &ResolutionTile<'_>, sub_band_rect: IntRect) -> usize { - if sub_band_rect.is_empty() { - return 0; - } - let cells = |start: u32, end: u32, size: u32| end.div_ceil(size).saturating_sub(start / size); - let columns = cells( - sub_band_rect.x0, - sub_band_rect.x1, - resolution_tile.code_block_width(), - ); - let rows = cells( - sub_band_rect.y0, - sub_band_rect.y1, - resolution_tile.code_block_height(), - ); - (columns as usize).saturating_mul(rows as usize) +/// Add one sub-band's records to the floor, counting tag-tree nodes only as +/// the two trees of each precinct need at least: a root and one leaf per +/// code-block. +fn add_sub_band_records( + records: &mut RecordCounts, + resolution_tile: &ResolutionTile<'_>, + sub_band_type: SubBandType, + layers: u8, +) { + let precincts = usize::try_from(resolution_tile.num_precincts()).unwrap_or(usize::MAX); + let code_blocks = global_code_block_count( + resolution_tile, + resolution_tile.sub_band_rect(sub_band_type), + ) + .unwrap_or(usize::MAX); + records.sub_bands = records.sub_bands.saturating_add(1); + records.precincts = records.precincts.saturating_add(precincts); + records.code_blocks = records.code_blocks.saturating_add(code_blocks); + records.layers = records + .layers + .saturating_add(code_blocks.saturating_mul(usize::from(layers))); + records.tag_tree_nodes = records + .tag_tree_nodes + .saturating_add(precincts.max(code_blocks).saturating_mul(2)); } diff --git a/crates/j2k-native/src/j2c/decode/subband/classic.rs b/crates/j2k-native/src/j2c/decode/subband/classic.rs index 0cdf6737..0ea09f25 100644 --- a/crates/j2k-native/src/j2c/decode/subband/classic.rs +++ b/crates/j2k-native/src/j2c/decode/subband/classic.rs @@ -85,10 +85,6 @@ impl ClassicJobParameters { } } -#[expect( - clippy::cast_precision_loss, - reason = "the codec float domain intentionally receives bounded integer samples at this rounding boundary" -)] #[expect( clippy::too_many_arguments, reason = "the Tier-1 boundary keeps geometry, state buffers, and validated options explicit" @@ -332,31 +328,17 @@ pub(super) fn decode_sub_band_classic_blocks( code_block.number_of_coding_passes, component_info.roi_shift, ); - let exact = !irreversible_midpoint - && component_info.roi_shift == 0 - && midpoint.reversible_is_exact(); placement.store_rows( tile_ctx.bit_plane_decode_context.coefficient_rows(), base_store, output_stride, |coefficients, out_row| { - if irreversible_midpoint { - midpoint.dequantize_row(coefficients, out_row, dequantization_step); - } else if exact { - bitplane::MidpointReconstructor::scale_exact_reversible_row( - coefficients, - out_row, - dequantization_step, - ); - } else { - for (output, coefficient) in out_row.iter_mut().zip(coefficients) { - *output = apply_roi_maxshift_inverse_i64( - midpoint.reconstruct_reversible(*coefficient), - component_info.roi_shift, - ) as f32 - * dequantization_step; - } - } + midpoint.reconstruct_row( + coefficients, + out_row, + dequantization_step, + irreversible_midpoint, + ); }, )?; } diff --git a/crates/j2k-native/src/j2c/ht_block_decode/magnitude.rs b/crates/j2k-native/src/j2c/ht_block_decode/magnitude.rs index ac89adb3..c6158e72 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/magnitude.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/magnitude.rs @@ -24,6 +24,26 @@ pub(super) fn supports_cleanup_to_f32(missing_msbs: u32, width: u32) -> bool { width != 0 && width.is_multiple_of(4) && uses_simd_rows(missing_msbs) } +/// Validate cleanup scratch and clear the preceding quad row's magnitudes. +fn prepare_cleanup_scratch( + width: u32, + height: u32, + sstr: usize, + scratch: &[u16], + v_n_scratch: &mut [u32], +) -> Option<()> { + let quad_rows = height.div_ceil(2) as usize; + if scratch.len() < sstr * (quad_rows + 1) { + return None; + } + let v_n_width = width.div_ceil(2) as usize + 2; + if v_n_scratch.len() < v_n_width { + return None; + } + v_n_scratch[..v_n_width].fill(0); + Some(()) +} + fn sample_mask(bit: u32) -> u32 { 1 << (4 + bit) } @@ -354,15 +374,7 @@ pub(super) fn decode_cleanup_and_magnitude_sign_phase( vlc_buffer: &mut Vec, observer: &mut impl HtDecodeObserver, ) -> Option<()> { - let quad_rows = height.div_ceil(2) as usize; - if scratch.len() < sstr * (quad_rows + 1) { - return None; - } - let v_n_width = width.div_ceil(2) as usize + 2; - if v_n_scratch.len() < v_n_width { - return None; - } - v_n_scratch[..v_n_width].fill(0); + prepare_cleanup_scratch(width, height, sstr, scratch, v_n_scratch)?; let mut mel = MelDecoder::new(coded_data, lcup, scup); let consumable = (width as usize) @@ -465,15 +477,7 @@ pub(super) fn decode_cleanup_and_magnitude_sign_to_f32 Result { let previous_row_len = (width.div_ceil(4) as usize) .checked_add(8) .ok_or(ValidationError::ImageTooLarge)?; + #[cfg(feature = "simd")] + let vlc_bytes = flat_vlc_capacity(width, height); + #[cfg(not(feature = "simd"))] + let vlc_bytes = 0; Ok(HtWorkspaceLayout { coefficients: coefficient_len, cleanup_symbols: cleanup_len, @@ -50,7 +56,7 @@ fn workspace_layout(width: u32, height: u32) -> Result { sigma_symbols: sigma_len, previous_row_symbols: previous_row_len, magsgn_bytes: flat_magsgn_capacity(coefficient_len)?, - vlc_bytes: flat_vlc_capacity(width, height), + vlc_bytes, }) } diff --git a/crates/j2k-native/src/j2c/ht_block_decode/readers.rs b/crates/j2k-native/src/j2c/ht_block_decode/readers.rs index 9cc51fd3..6f93505b 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/readers.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/readers.rs @@ -590,6 +590,10 @@ pub(super) fn read_u32_pair(values: &[u16], index: usize) -> u32 { mod tests; /// Largest cleanup suffix (MEL + VLC) in bytes, Clause 7.1.4. +#[cfg_attr( + not(any(test, feature = "simd")), + expect(dead_code, reason = "only SIMD HT decoding sizes flat VLC bits") +)] pub(super) const MAX_CLEANUP_SUFFIX: usize = 4079; /// Padding after the unstuffed VLC bytes: one top-up window past the data, @@ -599,6 +603,10 @@ const FLAT_VLC_PADDING: usize = 16; /// Bytes of VLC data a `width` x `height` block can consume: a quad pair takes /// at most 30 bits (two seven-bit codewords and a 16-bit U-VLC pair), and the /// pairs of a quad row cover four columns. +#[cfg_attr( + not(any(test, feature = "simd")), + expect(dead_code, reason = "only SIMD HT decoding sizes flat VLC bits") +)] pub(super) fn flat_vlc_consumable(width: u32, height: u32) -> usize { let pairs = (width.div_ceil(4) as usize).saturating_mul(height.div_ceil(2) as usize); pairs @@ -609,6 +617,10 @@ pub(super) fn flat_vlc_consumable(width: u32, height: u32) -> usize { } /// Capacity of the flat VLC buffer of a `width` x `height` block. +#[cfg_attr( + not(any(test, feature = "simd")), + expect(dead_code, reason = "only SIMD HT decoding reserves flat VLC bits") +)] pub(super) fn flat_vlc_capacity(width: u32, height: u32) -> usize { flat_vlc_consumable(width, height) + FLAT_VLC_PADDING } diff --git a/crates/j2k-native/src/j2c/ht_block_decode/tests.rs b/crates/j2k-native/src/j2c/ht_block_decode/tests.rs index b884451e..1f5464f2 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/tests.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/tests.rs @@ -597,6 +597,10 @@ fn decode_combined_validated_with_scratch_reuses_zeroed_buffers() { let first_capacities = scratch.capacities_for_test(); assert!(first_capacities.cleanup > 0); assert!(first_capacities.v_n > 0); + #[cfg(feature = "simd")] + assert!(first_capacities.vlc > 0); + #[cfg(not(feature = "simd"))] + assert_eq!(first_capacities.vlc, 0); scratch.poison_for_test(); decoded.fill(0); diff --git a/crates/j2k-native/src/j2c/idwt/banded.rs b/crates/j2k-native/src/j2c/idwt/banded.rs index 3906251e..177ed7c1 100644 --- a/crates/j2k-native/src/j2c/idwt/banded.rs +++ b/crates/j2k-native/src/j2c/idwt/banded.rs @@ -28,6 +28,7 @@ use super::horizontal::filter_row; use super::interleave::interleave_row; use super::model::{IDWTInput, IDWTTempOutput}; use super::try_grow_level_buffer; +use super::vertical::irreversible_row_scale; #[cfg(feature = "parallel")] use crate::error::{bail, DecodingError}; use crate::math::{self, dispatch, f32x8, Level, Simd, SIMD_WIDTH}; @@ -444,11 +445,15 @@ pub(super) fn filter_2d_banded_from_bands( try_grow_level_buffer(band_scratch, scratch_len)?; dispatch!(Level::new(), simd => run_strips( simd, - level, + level.width, target, parallel_layout.strip_len, &mut band_scratch[..scratch_len], per_worker, + #[inline(always)] + |simd, first_row, output, worker_scratch| { + stream_strip(simd, level, first_row, output, worker_scratch); + }, )); } Ok(()) @@ -458,11 +463,12 @@ pub(super) fn filter_2d_banded_from_bands( #[inline(always)] fn run_strips( simd: S, - level: Level2d<'_>, + width: usize, target: &mut [f32], strip_len: usize, scratch: &mut [f32], per_worker: usize, + strip_kernel: impl Fn(S, usize, &mut [f32], &mut [f32]) + Sync, ) { use rayon::prelude::*; use std::sync::Mutex; @@ -483,15 +489,15 @@ fn run_strips( #[inline(always)] || { while let Some((strip, output)) = next() { - let first_row = strip * strip_len / level.width; - stream_strip(simd, level, first_row, output, worker_scratch); + let first_row = strip * strip_len / width; + strip_kernel(simd, first_row, output, worker_scratch); } }, ); #[cfg(not(feature = "simd"))] while let Some((strip, output)) = next() { - let first_row = strip * strip_len / level.width; - stream_strip(simd, level, first_row, output, worker_scratch); + let first_row = strip * strip_len / width; + strip_kernel(simd, first_row, output, worker_scratch); } }); } @@ -569,13 +575,24 @@ fn filter_levels_pair( let steps = lifting_steps(outer_level.transform); let scratch = PairScratch::new(inner_level.width, width, layout, steps)?; try_grow_level_buffer(band_scratch, scratch.len)?; - dispatch!(Level::new(), simd => run_pair_strips( + dispatch!(Level::new(), simd => run_strips( simd, - [inner_level, outer_level], + outer_level.width, target, layout.strip_len, &mut band_scratch[..scratch.len], - [scratch.per_worker, scratch.rows_len], + scratch.per_worker, + #[inline(always)] + |simd, first_row, output, worker_scratch| { + pair_strip( + simd, + [inner_level, outer_level], + first_row, + output, + worker_scratch, + scratch.rows_len, + ); + }, )); Ok(()) } @@ -629,45 +646,6 @@ const fn pair_ll_rows(strip_rows: usize, count: usize) -> usize { strip_rows.div_ceil(2) + count + 3 } -#[cfg(feature = "parallel")] -#[inline(always)] -fn run_pair_strips( - simd: S, - levels: [Level2d<'_>; 2], - target: &mut [f32], - strip_len: usize, - scratch: &mut [f32], - [per_worker, rows_len]: [usize; 2], -) { - use rayon::prelude::*; - use std::sync::Mutex; - - let strips = Mutex::new(target.chunks_mut(strip_len).enumerate()); - let next = || strips.lock().ok().and_then(|mut strips| strips.next()); - let width = levels[1].width; - scratch - .par_chunks_mut(per_worker) - .with_max_len(1) - .for_each(|worker_scratch| { - // As in `run_strips`. - #[cfg(feature = "simd")] - simd.vectorize( - #[inline(always)] - || { - while let Some((strip, output)) = next() { - let first_row = strip * strip_len / width; - pair_strip(simd, levels, first_row, output, worker_scratch, rows_len); - } - }, - ); - #[cfg(not(feature = "simd"))] - while let Some((strip, output)) = next() { - let first_row = strip * strip_len / width; - pair_strip(simd, levels, first_row, output, worker_scratch, rows_len); - } - }); -} - /// Reconstruct rows `first_row..` of `outer` into `output`, after the rows /// of `inner` they read. #[cfg(feature = "parallel")] @@ -773,40 +751,11 @@ fn synthesize_level_row( low: &mut [f32], high: &mut [f32], ) { - let row_scale = if level.transform == WaveletTransform::Irreversible97 { - if index % 2 == level.y0 % 2 { - dwt::DWT97_KAPPA_F32 - } else { - level.high_pass - } - } else { - 1.0 - }; - let source = if index % 2 == level.even.start_row % 2 { - level.even.row(index) - } else { - level.odd.row(index) - }; - match source { - Some((first, second)) => { - synthesize_row( - simd, - level.row_geometry(), - first, - second, - row, - low, - high, - row_scale, - ); - } - // Lifting maps an all-zero row to zeros. - None => row.fill(0.0), - } + synthesize_level_columns(simd, level, index, row, 0..level.width, low, high); } -/// [`synthesize_level_row`] restricted to `columns`, one of the level's -/// [`column_strips`], into `out`. +/// Synthesize level row `index` over `columns` (the whole row, or one of the +/// level's [`column_strips`]) into `out`, with 9-7I row scaling. #[expect( clippy::inline_always, reason = "specialized with the streaming kernels" @@ -821,17 +770,11 @@ fn synthesize_level_columns( low: &mut [f32], high: &mut [f32], ) { - if columns.len() == level.width { - synthesize_level_row(simd, level, index, out, low, high); - return; - } - // Strips are only made of even-origin levels (`column_strips`). let row_scale = if level.transform == WaveletTransform::Reversible53 { 1.0 - } else if index % 2 == level.y0 % 2 { - dwt::DWT97_KAPPA_F32 } else { - level.high_pass + let y0_parity = u32::from(!level.y0.is_multiple_of(2)); + irreversible_row_scale(y0_parity, index.is_multiple_of(2), level.high_pass) }; let source = if index % 2 == level.even.start_row % 2 { level.even.row(index) @@ -839,17 +782,34 @@ fn synthesize_level_columns( level.odd.row(index) }; match source { - Some((low_source, high_source)) => synthesize_row_columns( - simd, - level, - low_source, - high_source, - out, - columns, - low, - high, - row_scale, - ), + Some((low_source, high_source)) => { + if columns.len() == level.width { + synthesize_row( + simd, + level.row_geometry(), + low_source, + high_source, + out, + low, + high, + row_scale, + ); + } else { + // Strips are only made of even-origin levels (`column_strips`). + synthesize_row_columns( + simd, + level, + low_source, + high_source, + out, + columns, + low, + high, + row_scale, + ); + } + } + // Lifting maps an all-zero row to zeros. None => out.fill(0.0), } } diff --git a/crates/j2k-native/src/j2c/mod.rs b/crates/j2k-native/src/j2c/mod.rs index 9501b6dd..97a33564 100644 --- a/crates/j2k-native/src/j2c/mod.rs +++ b/crates/j2k-native/src/j2c/mod.rs @@ -64,10 +64,11 @@ pub(crate) use decode::{ }; pub use decode::{CpuDecodeParallelism, DecoderContext, DecoderWorkspace, DecoderWorkspaceStats}; pub use recode::Reversible53CoefficientImage; +pub(crate) use roi::output_grid_offset; #[cfg(feature = "std")] pub(crate) use segment::directory::PacketDirectory; pub(crate) use segment::MAX_BITPLANE_COUNT; -pub(crate) use tile::ParsedTiles; +pub(crate) use tile::{tile_rect, ParsedTiles}; pub(crate) struct ParsedCodestream<'a> { pub(crate) header: Header<'a>, diff --git a/crates/j2k-native/src/j2c/tile.rs b/crates/j2k-native/src/j2c/tile.rs index 0aed2a51..d97d2d71 100644 --- a/crates/j2k-native/src/j2c/tile.rs +++ b/crates/j2k-native/src/j2c/tile.rs @@ -3,7 +3,9 @@ use alloc::vec::Vec; use super::build::{PrecinctData, SubBandType}; -use super::codestream::{markers, ComponentInfo, Header, ProgressionChange, ProgressionOrder}; +use super::codestream::{ + markers, ComponentInfo, Header, ProgressionChange, ProgressionOrder, SizeData, +}; use super::rect::IntRect; use crate::error::{bail, MarkerError, Result, TileError, ValidationError}; use crate::reader::BitReader; @@ -159,48 +161,51 @@ impl TilePart<'_> { } } +/// Reference-grid bounds of a tile, clipped to the image area. +pub(crate) fn tile_rect(size_data: &SizeData, x_coord: u32, y_coord: u32) -> IntRect { + // See B-7, B-8, B-9 and B-10. Saturating arithmetic: the results + // are clamped against the reference grid anyway, and a saturated + // intermediate already exceeds every in-grid bound, so the clamp + // still produces the mathematically correct edge for crafted + // headers whose products overflow u32. + let x0 = u32::max( + size_data + .tile_x_offset + .saturating_add(x_coord.saturating_mul(size_data.tile_width)), + size_data.image_area_x_offset, + ); + let y0 = u32::max( + size_data + .tile_y_offset + .saturating_add(y_coord.saturating_mul(size_data.tile_height)), + size_data.image_area_y_offset, + ); + + // Note that `x1` and `y1` are exclusive. + let x1 = u32::min( + size_data.tile_x_offset.saturating_add( + x_coord + .saturating_add(1) + .saturating_mul(size_data.tile_width), + ), + size_data.reference_grid_width, + ); + let y1 = u32::min( + size_data.tile_y_offset.saturating_add( + y_coord + .saturating_add(1) + .saturating_mul(size_data.tile_height), + ), + size_data.reference_grid_height, + ); + + IntRect::from_ltrb(x0, y0, x1, y1) +} + impl Tile<'_> { fn new(idx: u32, header: &Header<'_>) -> Self { - let rect = { - let size_data = &header.size_data; - - let x_coord = size_data.tile_x_coord(idx); - let y_coord = size_data.tile_y_coord(idx); - - // See B-7, B-8, B-9 and B-10. Saturating arithmetic: the results - // are clamped against the reference grid anyway, and a saturated - // intermediate already exceeds every in-grid bound, so the clamp - // still produces the mathematically correct edge for crafted - // headers whose products overflow u32. - let x0 = u32::max( - size_data - .tile_x_offset - .saturating_add(x_coord.saturating_mul(size_data.tile_width)), - size_data.image_area_x_offset, - ); - let y0 = u32::max( - size_data - .tile_y_offset - .saturating_add(y_coord.saturating_mul(size_data.tile_height)), - size_data.image_area_y_offset, - ); - - // Note that `x1` and `y1` are exclusive. - let x1 = u32::min( - size_data - .tile_x_offset - .saturating_add((x_coord + 1).saturating_mul(size_data.tile_width)), - size_data.reference_grid_width, - ); - let y1 = u32::min( - size_data - .tile_y_offset - .saturating_add((y_coord + 1).saturating_mul(size_data.tile_height)), - size_data.reference_grid_height, - ); - - IntRect::from_ltrb(x0, y0, x1, y1) - }; + let size = &header.size_data; + let rect = tile_rect(size, size.tile_x_coord(idx), size.tile_y_coord(idx)); Tile { #[cfg(feature = "std")] diff --git a/crates/j2k-native/src/scalar.rs b/crates/j2k-native/src/scalar.rs index a7dafb0d..4087aa80 100644 --- a/crates/j2k-native/src/scalar.rs +++ b/crates/j2k-native/src/scalar.rs @@ -4,13 +4,12 @@ use alloc::vec::Vec; use crate::error::bail; use crate::{ - add_roi_shift_to_bitplanes, apply_roi_maxshift_inverse_i64, checked_decode_byte_len3, j2c, - profile, DecodingError, EncodedHtJ2kCodeBlock, EncodedJ2kCodeBlock, - HtCleanupEncodeDistribution, HtCodeBlockDecodeJob, HtCodeBlockDecodePhaseLimit, - J2kCodeBlockDecodeJob, J2kCodeBlockSegment, J2kCodeBlockStyle, J2kForwardDwt53Level, - J2kForwardDwt53Output, J2kForwardDwt97Level, J2kForwardDwt97Output, J2kPacketizationEncodeJob, - J2kSubBandDecodeJob, J2kSubBandType, J2kTier1TokenSegment, Result, ValidationError, - MAX_CLASSIC_DECODE_BITPLANES, MAX_DEINTERLEAVE_REFERENCE_BIT_DEPTH, + add_roi_shift_to_bitplanes, checked_decode_byte_len3, j2c, profile, DecodingError, + EncodedHtJ2kCodeBlock, EncodedJ2kCodeBlock, HtCleanupEncodeDistribution, HtCodeBlockDecodeJob, + HtCodeBlockDecodePhaseLimit, J2kCodeBlockDecodeJob, J2kCodeBlockSegment, J2kCodeBlockStyle, + J2kForwardDwt53Level, J2kForwardDwt53Output, J2kForwardDwt97Level, J2kForwardDwt97Output, + J2kPacketizationEncodeJob, J2kSubBandDecodeJob, J2kSubBandType, J2kTier1TokenSegment, Result, + ValidationError, MAX_CLASSIC_DECODE_BITPLANES, MAX_DEINTERLEAVE_REFERENCE_BIT_DEPTH, }; mod classic_decode; diff --git a/crates/j2k-native/src/scalar/classic_decode.rs b/crates/j2k-native/src/scalar/classic_decode.rs index 4e5d7ed1..7b8f3673 100644 --- a/crates/j2k-native/src/scalar/classic_decode.rs +++ b/crates/j2k-native/src/scalar/classic_decode.rs @@ -1,9 +1,9 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use super::{ - add_roi_shift_to_bitplanes, apply_roi_maxshift_inverse_i64, bail, - internal_j2k_code_block_style, internal_j2k_sub_band_type, j2c, profile, DecodingError, - J2kCodeBlockDecodeJob, J2kSubBandDecodeJob, Result, MAX_CLASSIC_DECODE_BITPLANES, + add_roi_shift_to_bitplanes, bail, internal_j2k_code_block_style, internal_j2k_sub_band_type, + j2c, profile, DecodingError, J2kCodeBlockDecodeJob, J2kSubBandDecodeJob, Result, + MAX_CLASSIC_DECODE_BITPLANES, }; /// Adapter scalar classic J2K decoder helper for backend experimentation. @@ -135,10 +135,6 @@ pub(super) fn checked_code_block_output_layout( Ok(CodeBlockOutputLayout { stride }) } -#[expect( - clippy::cast_precision_loss, - reason = "the public scalar adapter intentionally emits f32 coefficients" -)] fn write_j2k_code_block_output( decode_context: &j2c::bitplane::BitPlaneDecodeContext, job: J2kCodeBlockDecodeJob<'_>, @@ -148,7 +144,6 @@ fn write_j2k_code_block_output( ) { let midpoint = decode_context.midpoint_reconstructor(job.number_of_coding_passes, job.roi_shift); - let exact = !irreversible_midpoint && job.roi_shift == 0 && midpoint.reversible_is_exact(); for (row_idx, coeff_row) in decode_context .coefficient_rows() .enumerate() @@ -156,25 +151,12 @@ fn write_j2k_code_block_output( { let row_start = row_idx * job.output_stride; let output_row = &mut output[row_start..row_start + layout.stride]; - if irreversible_midpoint { - midpoint.dequantize_row(coeff_row, output_row, job.dequantization_step); - continue; - } - if exact { - j2c::bitplane::MidpointReconstructor::scale_exact_reversible_row( - coeff_row, - output_row, - job.dequantization_step, - ); - continue; - } - for (coefficient, sample) in coeff_row.iter().zip(output_row.iter_mut()) { - *sample = apply_roi_maxshift_inverse_i64( - midpoint.reconstruct_reversible(*coefficient), - job.roi_shift, - ) as f32 - * job.dequantization_step; - } + midpoint.reconstruct_row( + coeff_row, + output_row, + job.dequantization_step, + irreversible_midpoint, + ); } } diff --git a/crates/j2k/src/decode/output.rs b/crates/j2k/src/decode/output.rs index 814dabd1..00bdf045 100644 --- a/crates/j2k/src/decode/output.rs +++ b/crates/j2k/src/decode/output.rs @@ -6,6 +6,10 @@ mod native; mod u16; mod u8; +use crate::backend::ColorSpace; +use crate::J2kError; +use j2k_core::PixelFormat; + pub(crate) use native::{ decode_native_components_with_native_context, native_components_route, native_region_components_route, NativeRoute, @@ -14,9 +18,110 @@ use native::{planes_pack_floor, region_rows_preferred}; pub(super) use u16::{ region_rows_preferred_u16, rows_match_native_u16, u16_rows_pack_floor, - write_components_u16_output, write_plane_rows_u16, write_u16_output, U16RowSink, + write_components_u16_output, write_plane_rows_u16, write_u16_output, }; pub(super) use u8::{ can_decode_u8_directly, expand_rgb8_to_rgba8, write_components_u8_output, write_plane_rows_u8, - write_u8_output, U8RowSink, + write_u8_output, }; + +/// Shared destination and error plumbing for the 8- and 16-bit component +/// row sinks. Its pixel format selects the matching plane writer. The 16-bit +/// path matches [`write_u16_output`] through [`rows_match_native_u16`], and +/// the first write error prevents every later write. +pub(in crate::decode) struct ComponentRowOutput<'o> { + color_space: &'o ColorSpace, + has_alpha: bool, + out: &'o mut [u8], + stride: usize, + fmt: PixelFormat, + error: Option, +} + +impl<'o> ComponentRowOutput<'o> { + /// `out` must already hold the region at `stride` in `fmt`. + pub(in crate::decode) fn new( + color_space: &'o ColorSpace, + has_alpha: bool, + out: &'o mut [u8], + stride: usize, + fmt: PixelFormat, + ) -> Self { + Self { + color_space, + has_alpha, + out, + stride, + fmt, + error: None, + } + } + + pub(in crate::decode) fn finish(self) -> Result<(), J2kError> { + self.error.map_or(Ok(()), Err) + } +} + +pub(in crate::decode) type U8RowSink<'o> = ComponentRowOutput<'o>; +pub(in crate::decode) type U16RowSink<'o> = ComponentRowOutput<'o>; + +impl j2k_native::ComponentRowSink for ComponentRowOutput<'_> { + fn write_rows(&mut self, x: u32, y: u32, planes: &[j2k_native::ComponentPlane<'_>]) { + if self.error.is_some() { + return; + } + let dims = planes + .first() + .map_or((0, 0), j2k_native::ComponentPlane::dimensions); + let offset = (y as usize) + .checked_mul(self.stride) + .and_then(|row| row.checked_add((x as usize).checked_mul(self.fmt.bytes_per_pixel())?)); + let result = match offset.and_then(|offset| self.out.get_mut(offset..)) { + Some(out) => match self.fmt { + PixelFormat::Gray8 | PixelFormat::Rgb8 | PixelFormat::Rgba8 => u8::write_planes_u8( + self.color_space, + self.has_alpha, + planes, + dims, + out, + self.stride, + self.fmt, + ), + PixelFormat::Gray16 | PixelFormat::Rgb16 | PixelFormat::Rgba16 => { + u16::write_planes_u16( + self.color_space, + self.has_alpha, + planes, + dims, + out, + self.stride, + self.fmt, + ) + } + _ => Err(J2kError::internal_backend( + "component row sink has unsupported pixel format", + )), + }, + None => Err(J2kError::DimensionOverflow { + width: x, + height: y, + }), + }; + self.error = result.err(); + } + + fn rgb8_region(&mut self, width: u32, height: u32) -> Option<&mut [u8]> { + // `write_rows` packs three 8-bit planes as `write_rgb_component_rows_u8` + // does, so a tight RGB8 buffer can take them packed by the decoder. + let row_bytes = (width as usize).checked_mul(3)?; + let tight_rgb8 = self.error.is_none() + && matches!(self.color_space, ColorSpace::RGB) + && !self.has_alpha + && self.fmt == PixelFormat::Rgb8 + && self.stride == row_bytes; + if !tight_rgb8 { + return None; + } + self.out.get_mut(..row_bytes.checked_mul(height as usize)?) + } +} diff --git a/crates/j2k/src/decode/output/u16.rs b/crates/j2k/src/decode/output/u16.rs index a6cc9a84..cfe580d9 100644 --- a/crates/j2k/src/decode/output/u16.rs +++ b/crates/j2k/src/decode/output/u16.rs @@ -27,7 +27,7 @@ pub(in crate::decode) fn write_components_u16_output( } /// Whether a region decode of `image` to `fmt` may stream its rows through -/// [`U16RowSink`]: they carry the samples `decode_native_region` packs (see +/// [`super::U16RowSink`]: they carry the samples `decode_native_region` packs (see /// `Image::streamed_rows_native_precision`: every component has one /// precision of at most 24 bits), which `U16Conversion` converts to the /// bytes [`write_u16_output`] writes from those packed samples, and @@ -69,7 +69,7 @@ pub(in crate::decode) fn region_rows_preferred_u16( } /// Write `planes`, each `dims` in size, to `out` from its first byte. -fn write_planes_u16( +pub(super) fn write_planes_u16( color_space: &ColorSpace, has_alpha: bool, planes: &[j2k_native::ComponentPlane<'_>], @@ -134,74 +134,6 @@ pub(in crate::decode) fn write_plane_rows_u16( } } -/// Writes a region decode's rectangles of component rows to a 16-bit -/// output as [`write_components_u16_output`] writes whole planes, which for -/// an image that [`rows_match_native_u16`] accepts is what -/// [`write_u16_output`] writes from its native samples. The first error -/// stops every later write and is returned by [`Self::finish`]. -pub(in crate::decode) struct U16RowSink<'o> { - color_space: &'o ColorSpace, - has_alpha: bool, - out: &'o mut [u8], - stride: usize, - fmt: PixelFormat, - error: Option, -} - -impl<'o> U16RowSink<'o> { - /// `out` must already hold the region at `stride` in `fmt`. - pub(in crate::decode) fn new( - color_space: &'o ColorSpace, - has_alpha: bool, - out: &'o mut [u8], - stride: usize, - fmt: PixelFormat, - ) -> Self { - Self { - color_space, - has_alpha, - out, - stride, - fmt, - error: None, - } - } - - pub(in crate::decode) fn finish(self) -> Result<(), J2kError> { - self.error.map_or(Ok(()), Err) - } -} - -impl j2k_native::ComponentRowSink for U16RowSink<'_> { - fn write_rows(&mut self, x: u32, y: u32, planes: &[j2k_native::ComponentPlane<'_>]) { - if self.error.is_some() { - return; - } - let dims = planes - .first() - .map_or((0, 0), j2k_native::ComponentPlane::dimensions); - let offset = (y as usize) - .checked_mul(self.stride) - .and_then(|row| row.checked_add((x as usize).checked_mul(self.fmt.bytes_per_pixel())?)); - let result = match offset.and_then(|offset| self.out.get_mut(offset..)) { - Some(out) => write_planes_u16( - self.color_space, - self.has_alpha, - planes, - dims, - out, - self.stride, - self.fmt, - ), - None => Err(J2kError::DimensionOverflow { - width: x, - height: y, - }), - }; - self.error = result.err(); - } -} - /// Rows `rows` of `plane`, `width` samples each, written from the first row /// of `out` on. fn write_component_rows_u16( diff --git a/crates/j2k/src/decode/output/u8.rs b/crates/j2k/src/decode/output/u8.rs index e210bbf4..6364d1b9 100644 --- a/crates/j2k/src/decode/output/u8.rs +++ b/crates/j2k/src/decode/output/u8.rs @@ -41,7 +41,7 @@ pub(in crate::decode) fn write_components_u8_output( } /// Write `planes`, each `dims` in size, to `out` from its first byte. -fn write_planes_u8( +pub(super) fn write_planes_u8( color_space: &ColorSpace, has_alpha: bool, planes: &[j2k_native::ComponentPlane<'_>], @@ -106,87 +106,6 @@ pub(in crate::decode) fn write_plane_rows_u8( } } -/// Writes a region decode's rectangles of component rows to an 8-bit -/// output as [`write_components_u8_output`] writes whole planes. The first -/// error stops every later write and is returned by [`Self::finish`]. -pub(in crate::decode) struct U8RowSink<'o> { - color_space: &'o ColorSpace, - has_alpha: bool, - out: &'o mut [u8], - stride: usize, - fmt: PixelFormat, - error: Option, -} - -impl<'o> U8RowSink<'o> { - /// `out` must already hold the region at `stride` in `fmt`. - pub(in crate::decode) fn new( - color_space: &'o ColorSpace, - has_alpha: bool, - out: &'o mut [u8], - stride: usize, - fmt: PixelFormat, - ) -> Self { - Self { - color_space, - has_alpha, - out, - stride, - fmt, - error: None, - } - } - - pub(in crate::decode) fn finish(self) -> Result<(), J2kError> { - self.error.map_or(Ok(()), Err) - } -} - -impl j2k_native::ComponentRowSink for U8RowSink<'_> { - fn write_rows(&mut self, x: u32, y: u32, planes: &[j2k_native::ComponentPlane<'_>]) { - if self.error.is_some() { - return; - } - let dims = planes - .first() - .map_or((0, 0), j2k_native::ComponentPlane::dimensions); - let offset = (y as usize) - .checked_mul(self.stride) - .and_then(|row| row.checked_add((x as usize).checked_mul(self.fmt.bytes_per_pixel())?)); - let result = match offset.and_then(|offset| self.out.get_mut(offset..)) { - Some(out) => write_planes_u8( - self.color_space, - self.has_alpha, - planes, - dims, - out, - self.stride, - self.fmt, - ), - None => Err(J2kError::DimensionOverflow { - width: x, - height: y, - }), - }; - self.error = result.err(); - } - - fn rgb8_region(&mut self, width: u32, height: u32) -> Option<&mut [u8]> { - // `write_rows` packs three 8-bit planes as `write_rgb_component_rows_u8` - // does, so a tight RGB8 buffer can take them packed by the decoder. - let row_bytes = (width as usize).checked_mul(3)?; - let tight_rgb8 = self.error.is_none() - && matches!(self.color_space, ColorSpace::RGB) - && !self.has_alpha - && self.fmt == PixelFormat::Rgb8 - && self.stride == row_bytes; - if !tight_rgb8 { - return None; - } - self.out.get_mut(..row_bytes.checked_mul(height as usize)?) - } -} - pub(super) fn component_sample_count(dims: (u32, u32)) -> Result { (dims.0 as usize) .checked_mul(dims.1 as usize)