From 10ec41d35a682ccbabc2feb9471703505efa4ebf Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Wed, 30 Sep 2026 09:44:10 +1000 Subject: [PATCH] feat(msgpack): export check_msgpack_structure as the shared structural walk (LAB-2735) The header-only MessagePack walk that ByteStorage::retrieve runs as its pre-scan is now public API, so SDKs that decode untrusted MessagePack themselves can call one shared walk instead of keeping copies that drift. The algorithm is unchanged. It needs no optional feature, takes the depth bound from the caller, and returns the bare reason for a rejection. --- README.md | 6 ++-- SECURITY.md | 7 +++++ src/byte_storage.rs | 9 ++++-- src/lib.rs | 6 ++-- src/msgpack_bounds.rs | 67 +++++++++++++++++++++++++++++++------------ 5 files changed, 70 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 02647ed..cc41776 100644 --- a/README.md +++ b/README.md @@ -256,7 +256,9 @@ headers declaring more elements or bytes than the input can back, the reserved marker `0xc1` and truncated input are all rejected before decoding, without allocating in proportion to any declared length. A rejection is `ByteStorageError::DeserializationFailed` with the message prefix -`decode pre-scan: `. See [`SECURITY.md`](SECURITY.md#envelope-decode-bounds). +`decode pre-scan: `. The same walk is public as +`check_msgpack_structure(bytes, max_depth)` for callers that decode untrusted +MessagePack themselves. See [`SECURITY.md`](SECURITY.md#envelope-decode-bounds). @@ -269,7 +271,7 @@ cachekit-core/ ├── src/ │ ├── lib.rs # Public API exports │ ├── byte_storage.rs # LZ4 + xxHash3 storage envelope -│ ├── msgpack_bounds.rs # Structural pre-scan run before the envelope decode +│ ├── msgpack_bounds.rs # Structural pre-scan (public check_msgpack_structure), run before the envelope decode │ ├── checksum.rs # Standalone xxHash3 checksum/verify primitive (feature = "checksum") │ ├── metrics.rs # Operation timing & statistics │ │ diff --git a/SECURITY.md b/SECURITY.md index 11eed3c..f183ae5 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -152,6 +152,13 @@ through `retrieve`, and asserts the pre-scan's message prefix, not merely that the call fails. As with the size bound above, a caller that deserializes `StorageEnvelope` directly bypasses the pre-scan and must impose its own. +The walk itself is public as `check_msgpack_structure(bytes, max_depth)`, so a +caller that decodes untrusted MessagePack outside `ByteStorage` can apply the +same rules at its own depth bound. It needs no optional feature. It returns the +bare reason for a rejection, with no prefix, and it does not enforce the +protocol's `32..=1024` range on `max_depth`: choosing the bound is the caller's +job. + ### Dependencies Security-critical dependencies are audited via `cargo-deny`: diff --git a/src/byte_storage.rs b/src/byte_storage.rs index aa7ba9c..974c915 100644 --- a/src/byte_storage.rs +++ b/src/byte_storage.rs @@ -325,14 +325,17 @@ impl Default for ByteStorage { } } +/// Nesting bound for the envelope decode. The protocol requires 32..=1024; 100 +/// matches cachekit-rs and cachekit-ts. A legitimate envelope nests 2 deep. +#[cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))] +const MAX_DEPTH: usize = 100; + /// Decode untrusted envelope bytes: structural pre-scan first, then the typed /// decode. Serde's derive skips an unknown map key with `IgnoredAny`, which /// recurses, so the depth bound has to hold before `rmp_serde` sees the bytes. #[cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))] fn decode_envelope(envelope_bytes: &[u8]) -> Result { - use crate::msgpack_bounds::{check_msgpack_structure, MAX_DEPTH}; - - check_msgpack_structure(envelope_bytes, MAX_DEPTH).map_err(|what| { + crate::check_msgpack_structure(envelope_bytes, MAX_DEPTH).map_err(|what| { ByteStorageError::DeserializationFailed(format!("decode pre-scan: {what}")) })?; rmp_serde::from_slice(envelope_bytes) diff --git a/src/lib.rs b/src/lib.rs index 7f97dcf..fc9ddcd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -73,10 +73,12 @@ pub use checksum::{checksum, verify_checksum}; // Core byte storage layer pub mod byte_storage; -#[cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))] -mod msgpack_bounds; pub use byte_storage::{ByteStorage, StorageEnvelope}; +// Structural pre-scan for untrusted MessagePack (no optional dependency) +mod msgpack_bounds; +pub use msgpack_bounds::check_msgpack_structure; + // Encryption module (feature-gated) #[cfg(feature = "encryption")] pub mod encryption; diff --git a/src/msgpack_bounds.rs b/src/msgpack_bounds.rs index 7a89b9d..527675d 100644 --- a/src/msgpack_bounds.rs +++ b/src/msgpack_bounds.rs @@ -1,27 +1,34 @@ //! Structural pre-scan for untrusted MessagePack. //! -//! `ByteStorage::retrieve` runs this over the envelope bytes before -//! `rmp_serde` materialises a `StorageEnvelope` (protocol `spec/wire-format.md` -//! → Retrieve Flow, step 2; the bounds are `spec/interop-mode.md` → Decode -//! bounds, pinned by `tests/vectors/decode-bounds.json`). The opcode table -//! matches cachekit-py's `check_msgpack_structure` and cachekit-rs's -//! `check_structure`. Unlike cachekit-py's walk, this one counts an empty -//! collection as a nesting level, which is how the spec defines depth; -//! cachekit-rs bounds depth in `rmp_serde` rather than in its walk. - -/// Nesting bound for the envelope decode. The protocol requires 32..=1024; 100 -/// matches cachekit-rs and cachekit-ts. A legitimate envelope nests 2 deep. -pub(crate) const MAX_DEPTH: usize = 100; - -/// Header-only walk over one MessagePack document: str/bin/ext payloads are -/// skipped by offset, never read, and nothing is allocated beyond one `u64` -/// per open collection (at most `max_depth`). +//! This is the one shared structural walk. `ByteStorage::retrieve` runs it over +//! the envelope bytes before `rmp_serde` materialises a `StorageEnvelope` +//! (protocol `spec/wire-format.md` → Retrieve Flow, step 2), and SDK bindings +//! call it through the [`crate::check_msgpack_structure`] re-export before they +//! decode untrusted values. The bounds are `spec/interop-mode.md` → Decode +//! bounds, pinned by `tests/vectors/decode-bounds.json`. Depth counts every +//! collection header, an empty one included, which is how the spec defines it. +//! +//! The walk needs no optional dependency, so it is available under every +//! feature set. + +/// Checks that `bytes` hold one MessagePack document whose declared structure +/// the input can actually back, before any decoder sees them. +/// +/// A header-only walk: str/bin/ext payloads are skipped by offset, never read, +/// and nothing is decoded. It allocates one `u64` per open non-empty +/// collection, so at most `max_depth` of them (8 KiB at a depth of 1024), and +/// nothing proportional to the input or to any declared length. /// /// Trailing bytes after the root element are left to the decoder. /// +/// `max_depth` is the caller's bound. The protocol (`spec/interop-mode.md` → +/// Decode bounds) requires one in `32..=1024`; this function does not enforce +/// that range and applies whatever value it is given. +/// /// # Errors /// -/// Names the violated bound, before any decoder pre-allocates a container, for: +/// Returns the violated bound as a bare reason with no prefix, so each caller +/// adds its own. It rejects: /// - nesting deeper than `max_depth`, counting every array or map header on /// the path (an empty one included); /// - a header declaring more payload bytes than the input holds; @@ -30,7 +37,27 @@ pub(crate) const MAX_DEPTH: usize = 100; /// total container pre-allocation is bounded by the input length rather /// than by `depth × declared length`; /// - the reserved marker `0xc1`, and input that ends mid-document. -pub(crate) fn check_msgpack_structure(bytes: &[u8], max_depth: usize) -> Result<(), String> { +/// +/// # Examples +/// +/// ``` +/// use cachekit_core::check_msgpack_structure; +/// +/// // [[[]]]: three levels, because the empty innermost array counts as one. +/// let doc = [0x91, 0x91, 0x90]; +/// assert_eq!(check_msgpack_structure(&doc, 3), Ok(())); +/// assert_eq!( +/// check_msgpack_structure(&doc, 2), +/// Err("nests deeper than 2 levels".to_owned()) +/// ); +/// +/// // An array32 header claiming 2^32 - 1 elements, in five bytes. +/// assert_eq!( +/// check_msgpack_structure(&[0xdd, 0xff, 0xff, 0xff, 0xff], 100), +/// Err("declares more elements than the input can back".to_owned()) +/// ); +/// ``` +pub fn check_msgpack_structure(bytes: &[u8], max_depth: usize) -> Result<(), String> { fn be(bytes: &[u8], pos: usize, width: usize) -> Result { let end = pos .checked_add(width) @@ -105,6 +132,9 @@ pub(crate) fn check_msgpack_structure(bytes: &[u8], max_depth: usize) -> Result< mod tests { use super::*; + /// The envelope's bound; any bound in the protocol's range would do here. + const MAX_DEPTH: usize = 100; + fn check(bytes: &[u8]) -> Result<(), String> { check_msgpack_structure(bytes, MAX_DEPTH) } @@ -228,6 +258,7 @@ mod tests { ); } + #[cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))] #[test] fn every_real_envelope_and_every_strict_prefix_of_one() { // The walk admits what writers emit, and nothing that ends early: