From e4c962a3274135a68baf6f0cc7829d21e7b8fe9f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 20:32:38 +0000 Subject: [PATCH] feat(dig-node): page the chunk_lens prologue for >2048-chunk reads (dig-download 0.17) (#2230) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A trailing prologue-only continuation frame now carries byte-offset 0 and no chunk_index, so dig-download 0.17's establish-probe + rewind guards accept it; bump the reader 0.15->0.17 (ChunkLensAssembler) and drop the local chunk_lens_pages mirror for dig_nat::split_chunk_lens_pages (#1686). SPEC §5.1 updated; 0.99.9->0.100.0. Refs #2230, #1668, #1686. Co-Authored-By: Claude --- Cargo.lock | 6 +- Cargo.toml | 2 +- SPEC.md | 20 ++- crates/dig-node-core/Cargo.toml | 4 +- crates/dig-node-core/src/lib.rs | 29 ++++ crates/dig-node-core/src/peer.rs | 149 +++++++++++++++++- .../src/seams/content/range_frame.rs | 53 ++----- 7 files changed, 213 insertions(+), 50 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 646c627e..29a33d2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2036,9 +2036,9 @@ dependencies = [ [[package]] name = "dig-download" -version = "0.15.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e9677cf1c1125d8fbcd35a274dc16ad25c4bec037436935add8f89619ba19e2" +checksum = "e30c060c23125e04564fb20590bfb058468126324b4fde079652b9d7478ff65a" dependencies = [ "async-trait", "dig-constants 0.8.0", @@ -2295,7 +2295,7 @@ dependencies = [ [[package]] name = "dig-node-service" -version = "0.99.10" +version = "0.100.0" dependencies = [ "async-trait", "axum", diff --git a/Cargo.toml b/Cargo.toml index 4ae45067..b75088cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ edition = "2021" # the ROOT manifest (`[workspace.package].version`), so it MUST be set here for a # release to fire (§3.6). The library crates (dig-node-core/dig-runtime/dig-wallet) # keep their own independent versions — only the released binary tracks the workspace version. -version = "0.99.10" +version = "0.100.0" # Release hardening, matching digstore: keep integer-overflow checks ON in release. # The node parses untrusted serialized input and does offset/length arithmetic over diff --git a/SPEC.md b/SPEC.md index 95178d6b..d18adecc 100644 --- a/SPEC.md +++ b/SPEC.md @@ -929,7 +929,7 @@ windows itself. The envelope MUST therefore describe both the WHOLE resource and | `next_offset` | every window | the next window's offset, or **`null`** on the last one | | `root` | every window | the generation root the window was served against | | `inclusion_proof` | every window | base64 whole-resource Merkle inclusion proof | -| `chunk_lens` | first window (`offset == 0`) | per-chunk ciphertext lengths of the WHOLE resource | +| `chunk_lens` | prologue (once per stream, PAGED) | per-chunk ciphertext lengths of the WHOLE resource | | `source` | node profile | `"local"` or `"remote"` — where this node served it from | This table is normative and MUST agree field-for-field with `ChunkObject` in docs.dig.net's @@ -959,9 +959,21 @@ isolation and MUST hold the complete resource before verifying. Every window car that whichever window a client happens to receive first can supply it — not so that windows can be verified independently. -`chunk_lens` is the ONE field that rides the first window only. It describes how to split the -REASSEMBLED resource, which a client cannot act on until it holds every window; a client that -begins mid-resource therefore cannot decrypt a multi-chunk resource and MUST fetch window 0. +`chunk_lens` is a PROLOGUE field: it describes how to split the REASSEMBLED resource, so a client +cannot act on it until it holds every window, and a client that begins mid-resource cannot decrypt a +multi-chunk resource and MUST fetch from the start. It is sent once per stream and MUST NOT be +repeated. On the peer length-prefixed frame stream it is **PAGED**: a layout exceeding +`dig_nat::MAX_CHUNK_LENS_PER_FRAME` (2048) entries cannot state itself on one frame, so it is split +into pages of at most 2048 entries each, and every frame carrying a page is stamped with the +`chunk_lens_offset` at which its page begins. When the requested bytes are exhausted before the +layout is fully sent, the remaining pages ride trailing **prologue-only continuation frames** — a +frame with a zero-length data payload that carries byte-`offset = 0` (NOT the ascending byte cursor, +which by then equals the resource length and would trip a reader's `offset >= max_len` establish +guard) and NO `chunk_index` (it begins no chunk, and a stale index would trip the reader's +ascending-index rewind guard). A prologue-only frame does NOT terminate the stream; the stream is +complete only once the bytes are exhausted AND every prologue page has been sent. (The single-frame +JSON-RPC `dig.fetchRange` response is not framing-bound and carries the whole layout on its one +frame.) **Window size.** A window is at most **3 MiB** of ciphertext. This node currently IGNORES the `length` request parameter and always serves a full window (or the remainder), where diff --git a/crates/dig-node-core/Cargo.toml b/crates/dig-node-core/Cargo.toml index 45ea2a01..dc419ad8 100644 --- a/crates/dig-node-core/Cargo.toml +++ b/crates/dig-node-core/Cargo.toml @@ -265,7 +265,7 @@ dig-pex = "0.1" # `seams/dig_peer/module_anchor.rs` (`ChainAnchoredModuleVerifier`). NOTE the `testkit` feature is # deliberately NOT enabled: it is what makes the fail-OPEN `AcceptAnyModuleAnchor` nameable, and this # crate's anchor gate is the reshare path's ONLY root of trust. -dig-download = "0.15" +dig-download = "0.17" # -- The shared peer client (#1283/#1576) ------------------------------------------------------------- # `DigPeer` — the ONE DIG Network peer client: peer_id-pinned mTLS over the full NAT ladder plus typed # RPC. Depended on DIRECTLY (not only transitively through dig-download) because dig-node supplies the @@ -344,7 +344,7 @@ rcgen = "0.13" # # Pinned by the `the_fail_open_anchor_verifier_is_not_reachable_from_a_production_build` test, which # fails if `testkit` ever appears on the production entry. -dig-download = { version = "0.15", features = ["testkit"] } +dig-download = { version = "0.17", features = ["testkit"] } # Captures the peer-facing serve's real emitted tracing records into an in-memory buffer, so the # serve-observability tests (#1595) assert what an operator would actually see in the node log — # and that no payload byte or proof ever reaches it. diff --git a/crates/dig-node-core/src/lib.rs b/crates/dig-node-core/src/lib.rs index 62179983..ef6b8f99 100644 --- a/crates/dig-node-core/src/lib.rs +++ b/crates/dig-node-core/src/lib.rs @@ -4376,6 +4376,35 @@ pub(crate) mod test_support { (Arc::new(resp), chunk_lens) } + /// A REAL served resource with `chunk_count` uniform `chunk_len`-byte chunks, committed under a + /// single-leaf generation root with its genuine digstore inclusion proof. + /// + /// It exists to exercise the PAGED prologue: a `chunk_count` above + /// [`dig_nat::MAX_CHUNK_LENS_PER_FRAME`] cannot state its `chunk_lens` on one frame, so the serve + /// path must split the layout across several frames and the reader must reassemble it. The chunks + /// are deliberately tiny — the point is the ENTRY COUNT of the layout, not the byte volume, so the + /// fixture stays small enough to build thousands of chunks cheaply. + pub(crate) fn many_chunk_served_resource( + chunk_count: usize, + chunk_len: usize, + ) -> (Arc, Vec) { + use digstore_core::merkle::{resource_leaf, MerkleTree}; + + let total = chunk_count * chunk_len; + // Byte i is `i mod 251` (a prime, so the pattern never aligns with a chunk boundary): a + // mis-ordered or dropped chunk changes the bytes, unlike a constant fill. + let ciphertext: Vec = (0..total).map(|i| (i % 251) as u8).collect(); + let tree = MerkleTree::from_leaves(vec![resource_leaf(&ciphertext)]); + let resp = ContentResponse { + merkle_proof: tree.prove(0).expect("single-leaf proof"), + roothash: tree.root(), + chunk_lens: std::iter::repeat_n(chunk_len as u32, chunk_count).collect(), + ciphertext, + }; + let chunk_lens = resp.chunk_lens.iter().map(|&l| u64::from(l)).collect(); + (Arc::new(resp), chunk_lens) + } + /// Seed `resource` into `node`'s memoized serve cache so [`Node::fetch_range_frame`] serves it, /// and return the `(store_id, root, retrieval_key)` hex triple that names it. pub(crate) fn seed_served_resource( diff --git a/crates/dig-node-core/src/peer.rs b/crates/dig-node-core/src/peer.rs index 089421bf..88cbf462 100644 --- a/crates/dig-node-core/src/peer.rs +++ b/crates/dig-node-core/src/peer.rs @@ -1855,7 +1855,14 @@ async fn stream_range_frames( loop { let take = range_frame::FRAME_PAYLOAD.min(end_of_span.saturating_sub(off)); let window = bytes[off..off + take].to_vec(); - let frame = framer.next_frame(off as u64, window); + // A trailing prologue-only continuation frame carries NO data payload — only the next + // `chunk_lens` page. It must be stamped with byte-offset 0, NOT the ascending cursor `off` + // (which by now equals the resource length): the 0.17 reader's establish probe reads a 1-byte + // span and rejects any frame whose `offset >= max_len`, so a page-only frame at `off == total` + // would be refused before its layout could be reassembled. A zero-length window is never a real + // byte position, so anchoring it at 0 loses nothing. + let frame_start = if take == 0 { 0 } else { off as u64 }; + let frame = framer.next_frame(frame_start, window); // This frame is the FINAL one only when the bytes are done AND the prologue is fully sent. // The page this frame just took is already accounted for, so `prologue_pending` here answers // "are there pages still to come AFTER this frame". @@ -5313,6 +5320,146 @@ pub(crate) mod tests { ); } + /// **Proves the PAGED-prologue producer contract at the REAL wire (#2230).** A resource with more + /// than [`dig_nat::MAX_CHUNK_LENS_PER_FRAME`] chunks cannot state its whole `chunk_lens` on one + /// frame, so the serve path pages it: the layout rides several frames, and when the requested bytes + /// run out before the layout is fully sent, the remaining pages travel on trailing PROLOGUE-ONLY + /// frames (zero data payload). + /// + /// The two properties a conforming 0.17 reader depends on, asserted on the bytes actually written: + /// + /// * every prologue-only frame is stamped with byte-`offset 0` — NOT the ascending cursor, which by + /// then equals the resource length and would trip the reader's `offset >= max_len` establish + /// guard; + /// * a prologue-only frame carries NO `chunk_index` — it begins no chunk, and a stale index would + /// trip the reader's ascending-index rewind guard. + /// + /// The stream must also NOT early-terminate: the full 2,049-entry layout is delivered across the + /// frames, so a paged read reassembles it in full. + #[tokio::test] + async fn a_paged_prologue_rides_offset_zero_frames_with_no_chunk_index_over_the_real_wire() { + // 2,049 chunks → one entry past the 2,048/page ceiling → exactly two prologue pages. + let chunk_count = dig_nat::MAX_CHUNK_LENS_PER_FRAME + 1; + let (resource, served_layout) = + crate::test_support::many_chunk_served_resource(chunk_count, 8); + + let (node, _td) = crate::test_support::test_node_for_peer_surface(); + let (store, root, rk) = crate::test_support::seed_served_resource(&node, resource); + let responder = NodeResponder::without_pool(node); + // A one-byte probe: the bytes run out on the first frame, forcing the second prologue page onto + // a trailing data-less frame — the exact shape the offset-0 fix governs. + let req = json!({ + "store_id": store, "root": root, "retrieval_key": rk, + "offset": 0, "length": 1, + }); + + let (mut client, mut server) = tokio::io::duplex(256 * 1024); + let served = tokio::spawn(async move { + responder + .stream_range(req, &test_caller(), &mut server) + .await + }); + served.await.expect("serve task join").expect("served"); + + let mut frames = Vec::new(); + while let Some(frame) = read_framed(&mut client) + .await + .expect("no I/O error reading the frame stream") + { + frames.push(frame); + } + + // A prologue-only frame carries a `chunk_lens` page but zero data bytes (`bytes` is base64, so + // an empty payload serializes to the empty string). + let prologue_only: Vec<&Value> = frames + .iter() + .filter(|f| f["bytes"].as_str() == Some("") && f.get("chunk_lens").is_some()) + .collect(); + assert!( + !prologue_only.is_empty(), + "a 2,049-chunk layout past the request span must trail a prologue-only frame: {frames:?}" + ); + for frame in &prologue_only { + assert_eq!( + frame["offset"], + json!(0), + "a prologue-only frame must be stamped offset 0, not the ascending cursor: {frame:?}" + ); + assert!( + frame.get("chunk_index").is_none(), + "a prologue-only frame begins no chunk, so it carries no chunk_index: {frame:?}" + ); + assert!( + frame["chunk_lens_offset"].is_u64(), + "a prologue-only frame carries a located page: {frame:?}" + ); + } + + // The stream did not early-terminate: reassembling every page's entries yields the whole layout. + let mut reassembled: Vec = Vec::new(); + for frame in &frames { + if let Some(page) = frame["chunk_lens"].as_array() { + reassembled.extend(page.iter().map(|v| v.as_u64().expect("chunk_lens entry"))); + } + } + assert_eq!( + reassembled, served_layout, + "the paged prologue must reassemble to the full served layout, byte-for-byte" + ); + assert!( + frames + .iter() + .filter(|f| f.get("chunk_lens").is_some()) + .count() + >= 2, + "a 2,049-entry layout must span at least two prologue pages: {frames:?}" + ); + } + + /// **The primary end-to-end proof (#2230): the paged-prologue PRODUCER against the SHIPPED 0.17 + /// reader.** Drives the production [`NodeResponder::stream_range`] over a real `tokio::io::duplex` + /// and feeds the raw wire bytes into `dig_download::assemble_range_stream` — the actual reassembler + /// a downloading peer runs — with the `max_len: 1` establish probe dig-download sends on every + /// download. + /// + /// Pre-fix this FAILS for the right reason: the trailing prologue page rides a frame stamped with + /// the ascending cursor (== the resource length), which the reader rejects as + /// `offset >= max_len`. After the fix the reader reassembles the full 2,049-entry layout with no + /// paged-prologue or offset error. + #[tokio::test] + async fn the_paged_prologue_producer_reads_end_to_end_through_the_shipped_reader() { + let chunk_count = dig_nat::MAX_CHUNK_LENS_PER_FRAME + 1; + let (resource, served_layout) = + crate::test_support::many_chunk_served_resource(chunk_count, 8); + + let (node, _td) = crate::test_support::test_node_for_peer_surface(); + let (store, root, rk) = crate::test_support::seed_served_resource(&node, resource); + let responder = NodeResponder::without_pool(node); + let req = json!({ + "store_id": store, "root": root, "retrieval_key": rk, + "offset": 0, "length": 1, + }); + + let (mut client, mut server) = tokio::io::duplex(256 * 1024); + let served = tokio::spawn(async move { + responder + .stream_range(req, &test_caller(), &mut server) + .await + }); + served.await.expect("serve task join").expect("served"); + + // The 1-byte establish probe dig-download's orchestrator sends on every download. + let (_bytes, meta) = dig_download::assemble_range_stream(&mut client, 1) + .await + .expect("the shipped 0.17 reader reassembles the paged prologue with no offset error"); + + assert_eq!( + meta.chunk_lens, + Some(served_layout), + "the reassembled layout must equal the full 2,049-entry served layout" + ); + } + #[tokio::test] async fn an_inbound_fetch_range_for_content_we_do_not_hold_logs_the_refusal() { // The ambiguity #1595 closes: a request the node cannot answer must say so in the log, so diff --git a/crates/dig-node-core/src/seams/content/range_frame.rs b/crates/dig-node-core/src/seams/content/range_frame.rs index 8ba093a9..caa4e957 100644 --- a/crates/dig-node-core/src/seams/content/range_frame.rs +++ b/crates/dig-node-core/src/seams/content/range_frame.rs @@ -104,42 +104,6 @@ pub(crate) struct RangeVerification<'a> { pub inclusion_proof: Option<&'a str>, } -/// Split `chunk_lens` into the pages a paged prologue is made of: page-aligned offsets, each page -/// exactly [`dig_nat::MAX_CHUNK_LENS_PER_FRAME`] entries except a possibly-short tail. -/// -/// This mirrors `dig_nat::RangeFrame::split_chunk_lens_pages`, which is the normative split and the -/// reassembler's own mirror. It is reproduced here ONLY because that helper landed in dig-nat 0.14, -/// while this node is held at 0.13 by dig-download and dig-gossip (see the dependency rationale in -/// `Cargo.toml`); the moment the tree reaches 0.14 this function MUST be deleted in favour of it — -/// tracked as DIG-Network/dig_ecosystem#1686 (the dig-gossip + dig-peer-selector cascade onto dig-nat -/// ^0.14 / dig-dht ^0.8), so the mirror's removal is traceable from here rather than depending on -/// someone remembering why it exists, -/// because #1640 was precisely two sides of one rule maintained separately. The page SIZE is read from -/// dig-nat either way, so the one number that matters cannot drift. -/// -/// The shape is what the reassembler requires, and each requirement excludes a whole class rather -/// than one observed misbehaviour: -/// -/// * no page is EMPTY — an empty page fills nothing, so accepting one lets a sender stream frames -/// forever without ever completing the prologue; -/// * every page except the tail is exactly full — a short page anywhere but the end leaves a gap no -/// page-aligned page can ever fill, so it is refused on arrival rather than surfacing later as an -/// unexplained incompleteness; -/// * an empty ARRAY yields no pages at all, which is a complete prologue for a resource with no chunk -/// table rather than a stream that can never finish. -fn chunk_lens_pages(chunk_lens: &[u64]) -> Vec<(u64, Vec)> { - chunk_lens - .chunks(dig_nat::MAX_CHUNK_LENS_PER_FRAME) - .enumerate() - .map(|(page, entries)| { - ( - (page * dig_nat::MAX_CHUNK_LENS_PER_FRAME) as u64, - entries.to_vec(), - ) - }) - .collect() -} - /// The absolute chunk index that byte `offset` begins, or `None` when `offset` is not on a chunk /// boundary (or lies past the resource). /// @@ -186,7 +150,11 @@ impl<'a> RangeStreamFramer<'a> { let pending_pages = if skip_layout { VecDeque::new() } else { - chunk_lens_pages(verification.chunk_lens).into() + // The normative split lives in dig-nat, published together with its reassembler mirror so + // encode + decode cannot drift (#1640/#1686): page-aligned offsets, each page exactly + // `MAX_CHUNK_LENS_PER_FRAME` entries except a possibly-short tail; an empty layout yields no + // pages, which is a complete prologue for a resource with no chunk table. + dig_nat::RangeFrame::split_chunk_lens_pages(verification.chunk_lens).into() }; RangeStreamFramer { verification, @@ -205,6 +173,7 @@ impl<'a> RangeStreamFramer<'a> { /// afterwards and applies `with_complete` itself. Setting `complete` on a frame that still owes /// pages would stop a conforming reader before the layout it needs to DECRYPT ever arrives. pub(crate) fn next_frame(&mut self, start: u64, bytes: Vec) -> dig_nat::RangeFrame { + let has_payload = !bytes.is_empty(); let mut frame = dig_nat::RangeFrame::data(start, bytes); // The identity set rides EVERY frame. `chunk_count` is the resource's TOTAL entry count, so a @@ -229,8 +198,14 @@ impl<'a> RangeStreamFramer<'a> { frame.total_length = Some(self.verification.total_length); frame.chunk_count = Some(chunk_count); } - if let Some(index) = chunk_index_at(self.verification.chunk_lens, start) { - frame = frame.with_chunk_index(index); + // A chunk index identifies which chunk this frame's PAYLOAD begins. A zero-payload frame (a + // trailing prologue-only continuation) begins no chunk, so it must carry no index: the 0.17 + // reader's rewind guard rejects a frame whose `chunk_index < highest_chunk_index`, and a + // page-only frame stamped with chunk 0 after real chunks have gone out would trip exactly that. + if has_payload { + if let Some(index) = chunk_index_at(self.verification.chunk_lens, start) { + frame = frame.with_chunk_index(index); + } } if self.skip_layout {