diff --git a/README.md b/README.md index 02647ed..cc41776 100644 --- a/README.md +++ b/README.md @@ -256,7 +256,9 @@ headers declaring more elements or bytes than the input can back, the reserved marker `0xc1` and truncated input are all rejected before decoding, without allocating in proportion to any declared length. A rejection is `ByteStorageError::DeserializationFailed` with the message prefix -`decode pre-scan: `. See [`SECURITY.md`](SECURITY.md#envelope-decode-bounds). +`decode pre-scan: `. The same walk is public as +`check_msgpack_structure(bytes, max_depth)` for callers that decode untrusted +MessagePack themselves. See [`SECURITY.md`](SECURITY.md#envelope-decode-bounds). @@ -269,7 +271,7 @@ cachekit-core/ ├── src/ │ ├── lib.rs # Public API exports │ ├── byte_storage.rs # LZ4 + xxHash3 storage envelope -│ ├── msgpack_bounds.rs # Structural pre-scan run before the envelope decode +│ ├── msgpack_bounds.rs # Structural pre-scan (public check_msgpack_structure), run before the envelope decode │ ├── checksum.rs # Standalone xxHash3 checksum/verify primitive (feature = "checksum") │ ├── metrics.rs # Operation timing & statistics │ │ diff --git a/SECURITY.md b/SECURITY.md index 11eed3c..f183ae5 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -152,6 +152,13 @@ through `retrieve`, and asserts the pre-scan's message prefix, not merely that the call fails. As with the size bound above, a caller that deserializes `StorageEnvelope` directly bypasses the pre-scan and must impose its own. +The walk itself is public as `check_msgpack_structure(bytes, max_depth)`, so a +caller that decodes untrusted MessagePack outside `ByteStorage` can apply the +same rules at its own depth bound. It needs no optional feature. It returns the +bare reason for a rejection, with no prefix, and it does not enforce the +protocol's `32..=1024` range on `max_depth`: choosing the bound is the caller's +job. + ### Dependencies Security-critical dependencies are audited via `cargo-deny`: diff --git a/src/byte_storage.rs b/src/byte_storage.rs index aa7ba9c..974c915 100644 --- a/src/byte_storage.rs +++ b/src/byte_storage.rs @@ -325,14 +325,17 @@ impl Default for ByteStorage { } } +/// Nesting bound for the envelope decode. The protocol requires 32..=1024; 100 +/// matches cachekit-rs and cachekit-ts. A legitimate envelope nests 2 deep. +#[cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))] +const MAX_DEPTH: usize = 100; + /// Decode untrusted envelope bytes: structural pre-scan first, then the typed /// decode. Serde's derive skips an unknown map key with `IgnoredAny`, which /// recurses, so the depth bound has to hold before `rmp_serde` sees the bytes. #[cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))] fn decode_envelope(envelope_bytes: &[u8]) -> Result { - use crate::msgpack_bounds::{check_msgpack_structure, MAX_DEPTH}; - - check_msgpack_structure(envelope_bytes, MAX_DEPTH).map_err(|what| { + crate::check_msgpack_structure(envelope_bytes, MAX_DEPTH).map_err(|what| { ByteStorageError::DeserializationFailed(format!("decode pre-scan: {what}")) })?; rmp_serde::from_slice(envelope_bytes) diff --git a/src/lib.rs b/src/lib.rs index 7f97dcf..fc9ddcd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -73,10 +73,12 @@ pub use checksum::{checksum, verify_checksum}; // Core byte storage layer pub mod byte_storage; -#[cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))] -mod msgpack_bounds; pub use byte_storage::{ByteStorage, StorageEnvelope}; +// Structural pre-scan for untrusted MessagePack (no optional dependency) +mod msgpack_bounds; +pub use msgpack_bounds::check_msgpack_structure; + // Encryption module (feature-gated) #[cfg(feature = "encryption")] pub mod encryption; diff --git a/src/msgpack_bounds.rs b/src/msgpack_bounds.rs index 7a89b9d..527675d 100644 --- a/src/msgpack_bounds.rs +++ b/src/msgpack_bounds.rs @@ -1,27 +1,34 @@ //! Structural pre-scan for untrusted MessagePack. //! -//! `ByteStorage::retrieve` runs this over the envelope bytes before -//! `rmp_serde` materialises a `StorageEnvelope` (protocol `spec/wire-format.md` -//! → Retrieve Flow, step 2; the bounds are `spec/interop-mode.md` → Decode -//! bounds, pinned by `tests/vectors/decode-bounds.json`). The opcode table -//! matches cachekit-py's `check_msgpack_structure` and cachekit-rs's -//! `check_structure`. Unlike cachekit-py's walk, this one counts an empty -//! collection as a nesting level, which is how the spec defines depth; -//! cachekit-rs bounds depth in `rmp_serde` rather than in its walk. - -/// Nesting bound for the envelope decode. The protocol requires 32..=1024; 100 -/// matches cachekit-rs and cachekit-ts. A legitimate envelope nests 2 deep. -pub(crate) const MAX_DEPTH: usize = 100; - -/// Header-only walk over one MessagePack document: str/bin/ext payloads are -/// skipped by offset, never read, and nothing is allocated beyond one `u64` -/// per open collection (at most `max_depth`). +//! This is the one shared structural walk. `ByteStorage::retrieve` runs it over +//! the envelope bytes before `rmp_serde` materialises a `StorageEnvelope` +//! (protocol `spec/wire-format.md` → Retrieve Flow, step 2), and SDK bindings +//! call it through the [`crate::check_msgpack_structure`] re-export before they +//! decode untrusted values. The bounds are `spec/interop-mode.md` → Decode +//! bounds, pinned by `tests/vectors/decode-bounds.json`. Depth counts every +//! collection header, an empty one included, which is how the spec defines it. +//! +//! The walk needs no optional dependency, so it is available under every +//! feature set. + +/// Checks that `bytes` hold one MessagePack document whose declared structure +/// the input can actually back, before any decoder sees them. +/// +/// A header-only walk: str/bin/ext payloads are skipped by offset, never read, +/// and nothing is decoded. It allocates one `u64` per open non-empty +/// collection, so at most `max_depth` of them (8 KiB at a depth of 1024), and +/// nothing proportional to the input or to any declared length. /// /// Trailing bytes after the root element are left to the decoder. /// +/// `max_depth` is the caller's bound. The protocol (`spec/interop-mode.md` → +/// Decode bounds) requires one in `32..=1024`; this function does not enforce +/// that range and applies whatever value it is given. +/// /// # Errors /// -/// Names the violated bound, before any decoder pre-allocates a container, for: +/// Returns the violated bound as a bare reason with no prefix, so each caller +/// adds its own. It rejects: /// - nesting deeper than `max_depth`, counting every array or map header on /// the path (an empty one included); /// - a header declaring more payload bytes than the input holds; @@ -30,7 +37,27 @@ pub(crate) const MAX_DEPTH: usize = 100; /// total container pre-allocation is bounded by the input length rather /// than by `depth × declared length`; /// - the reserved marker `0xc1`, and input that ends mid-document. -pub(crate) fn check_msgpack_structure(bytes: &[u8], max_depth: usize) -> Result<(), String> { +/// +/// # Examples +/// +/// ``` +/// use cachekit_core::check_msgpack_structure; +/// +/// // [[[]]]: three levels, because the empty innermost array counts as one. +/// let doc = [0x91, 0x91, 0x90]; +/// assert_eq!(check_msgpack_structure(&doc, 3), Ok(())); +/// assert_eq!( +/// check_msgpack_structure(&doc, 2), +/// Err("nests deeper than 2 levels".to_owned()) +/// ); +/// +/// // An array32 header claiming 2^32 - 1 elements, in five bytes. +/// assert_eq!( +/// check_msgpack_structure(&[0xdd, 0xff, 0xff, 0xff, 0xff], 100), +/// Err("declares more elements than the input can back".to_owned()) +/// ); +/// ``` +pub fn check_msgpack_structure(bytes: &[u8], max_depth: usize) -> Result<(), String> { fn be(bytes: &[u8], pos: usize, width: usize) -> Result { let end = pos .checked_add(width) @@ -105,6 +132,9 @@ pub(crate) fn check_msgpack_structure(bytes: &[u8], max_depth: usize) -> Result< mod tests { use super::*; + /// The envelope's bound; any bound in the protocol's range would do here. + const MAX_DEPTH: usize = 100; + fn check(bytes: &[u8]) -> Result<(), String> { check_msgpack_structure(bytes, MAX_DEPTH) } @@ -228,6 +258,7 @@ mod tests { ); } + #[cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))] #[test] fn every_real_envelope_and_every_strict_prefix_of_one() { // The walk admits what writers emit, and nothing that ends early: