From c9f98959600de1a48e394d5ef190d4571fb69f52 Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 18 Oct 2026 22:58:06 -0400 Subject: [PATCH] perf(native): inline VLC symbol decoding and pass refill state by value Entropy decoding is most of a decode on both routes, and the HP coefficient loop is about 60% of it. That loop called each per-symbol VLC helper out of line, so every symbol reloaded and stored the bit reader's fields, returned its Result through memory, and re-checked its table indices. The helpers were 21% of main-thread samples on a 2048x2048 RGB q8 image. decode_index, decode_abs_level, decode_first_index, decode_run_cached, and read_bits are now forced inline, and the bit-serial run decoder they fall back to near a packet end is cold and out of line. The out-of-line cache refill and the end-of-packet error constructor take and return plain values instead of the reader, so calling them no longer forces it to memory. Decoding logic, error variants, and error positions are unchanged. Single-image entropy decoding is 1.04-1.13x faster, and batch tile preparation 1.06-1.29x faster, on an M4 Pro, an M4, and an M5 Max. --- crates/jxr-native/src/entropy/bit_reader.rs | 67 ++++++++++++------- crates/jxr-native/src/entropy/coefficients.rs | 27 ++++++-- 2 files changed, 66 insertions(+), 28 deletions(-) diff --git a/crates/jxr-native/src/entropy/bit_reader.rs b/crates/jxr-native/src/entropy/bit_reader.rs index 0853a94..ed5b794 100644 --- a/crates/jxr-native/src/entropy/bit_reader.rs +++ b/crates/jxr-native/src/entropy/bit_reader.rs @@ -67,7 +67,11 @@ impl<'a> PacketBitReader<'a> { } /// Reads up to 64 bits, most-significant bit first. - #[inline] + #[expect( + clippy::inline_always, + reason = "a call per read made flexbit-heavy decoding about 2% slower" + )] + #[inline(always)] pub fn read_bits(&mut self, count: u8) -> Result { // A refilled cache holds at least 57 bits unless the packet ends first. if count > 57 { @@ -140,13 +144,13 @@ impl<'a> PacketBitReader<'a> { } /// Builds the error a bit-serial reader reports after consuming `consumed` more bits. - #[cold] + #[inline] pub(crate) fn unexpected_end_after(&self, consumed: usize) -> EntropyError { - EntropyError::UnexpectedEnd { - bit_position: self.bit_position.saturating_add(consumed), - requested_bits: 1, - bit_length: self.bit_length, - } + unexpected_end( + self.bit_position.saturating_add(consumed), + 1, + self.bit_length, + ) } /// Consumes `count <= cached` bits, where `count` is at most 57. @@ -177,39 +181,42 @@ impl<'a> PacketBitReader<'a> { Ok((high << 32) | low) } - #[cold] - #[inline(never)] + #[inline] fn end_error(&self, count: u8) -> EntropyError { - EntropyError::UnexpectedEnd { - bit_position: self.bit_position, - requested_bits: count, - bit_length: self.bit_length, - } + unexpected_end(self.bit_position, count, self.bit_length) } /// Reloads the cache with the unread bits at `bit_position`. + #[inline] + fn refill(&mut self) { + (self.cache, self.cached) = Self::window(self.bytes, self.bit_position, self.bit_length); + } + + /// Returns the cache and its bit count for the unread bits at `bit_position`. /// /// A byte-aligned 64-bit window holds at least 57 bits after the in-byte /// offset. Bytes beyond the backing slice read as zero, and bits beyond - /// `bit_length` are cleared. + /// `bit_length` are cleared. It takes and returns values rather than the + /// reader so that calling it does not force a register-held reader to memory. #[inline(never)] - fn refill(&mut self) { - let byte = self.bit_position / 8; - let word = match self.bytes.get(byte..byte + 8) { + fn window(bytes: &[u8], bit_position: usize, bit_length: usize) -> (u64, u32) { + let byte = bit_position / 8; + let word = match bytes.get(byte..byte + 8) { Some(chunk) => { u64::from_be_bytes(chunk.try_into().expect("slice is exactly eight bytes")) } - None => Self::padded_word(self.bytes, byte), + None => Self::padded_word(bytes, byte), }; - let offset = self.bit_position % 8; - let available = (64 - offset).min(self.bits_remaining()); + let offset = bit_position % 8; + let available = (64 - offset).min(bit_length - bit_position); // `available` is at least 57 unless the packet ends, and never zero // when any bit remains. - self.cached = u32::try_from(available).expect("window holds at most 64 bits"); - self.cache = match available { + let cached = u32::try_from(available).expect("window holds at most 64 bits"); + let cache = match available { 0 => 0, bits => (word << offset) & (u64::MAX << (64 - bits)), }; + (cache, cached) } #[cold] @@ -222,6 +229,20 @@ impl<'a> PacketBitReader<'a> { } } +#[cold] +#[inline(never)] +const fn unexpected_end( + bit_position: usize, + requested_bits: u8, + bit_length: usize, +) -> EntropyError { + EntropyError::UnexpectedEnd { + bit_position, + requested_bits, + bit_length, + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/jxr-native/src/entropy/coefficients.rs b/crates/jxr-native/src/entropy/coefficients.rs index 171d3da..829b52d 100644 --- a/crates/jxr-native/src/entropy/coefficients.rs +++ b/crates/jxr-native/src/entropy/coefficients.rs @@ -359,7 +359,11 @@ fn advance_location(location: u8, run: u8) -> Result { Ok(next) } -#[inline] +#[expect( + clippy::inline_always, + reason = "a call per symbol made HP and LP block decoding up to 12% slower" +)] +#[inline(always)] fn decode_abs_level( reader: &mut PacketBitReader<'_>, state: &mut AdaptiveVlc, @@ -397,7 +401,11 @@ fn decode_abs_level( .ok_or(EntropyError::CoefficientOverflow) } -#[inline] +#[expect( + clippy::inline_always, + reason = "a call per symbol made HP and LP block decoding up to 12% slower" +)] +#[inline(always)] fn decode_first_index( reader: &mut PacketBitReader<'_>, state: &mut AdaptiveVlc, @@ -411,7 +419,11 @@ fn decode_first_index( Ok(symbol) } -#[inline] +#[expect( + clippy::inline_always, + reason = "a call per symbol made HP and LP block decoding up to 12% slower" +)] +#[inline(always)] fn decode_index( reader: &mut PacketBitReader<'_>, location: u8, @@ -429,7 +441,11 @@ fn decode_index( } /// [`decode_run`] through [`RUN_LOOKUP`], falling back near the packet end. -#[inline] +#[expect( + clippy::inline_always, + reason = "a call per symbol made HP and LP block decoding up to 12% slower" +)] +#[inline(always)] fn decode_run_cached(reader: &mut PacketBitReader<'_>, max_run: u8) -> Result { if let Some(row) = RUN_LOOKUP .get(usize::from(max_run)) @@ -443,7 +459,8 @@ fn decode_run_cached(reader: &mut PacketBitReader<'_>, max_run: u8) -> Result, max_run: u8) -> Result { if max_run == 0 || max_run > 14 { return Err(EntropyError::InvalidParameter {