feat(dig-node): page the chunk_lens prologue for >2048-chunk reads (dig-download 0.17) (#2230)

Cargo.toml

@@ -32,7 +32,7 @@ edition = "2021"
 # the ROOT manifest (`[workspace.package].version`), so it MUST be set here for a
 # release to fire (§3.6). The library crates (dig-node-core/dig-runtime/dig-wallet)
 # keep their own independent versions — only the released binary tracks the workspace version.
-version = "0.99.10"
+version = "0.100.0"
 
 # Release hardening, matching digstore: keep integer-overflow checks ON in release.
 # The node parses untrusted serialized input and does offset/length arithmetic over

SPEC.md

@@ -929,7 +929,7 @@ windows itself. The envelope MUST therefore describe both the WHOLE resource and
 | `next_offset` | every window | the next window's offset, or **`null`** on the last one |
 | `root` | every window | the generation root the window was served against |
 | `inclusion_proof` | every window | base64 whole-resource Merkle inclusion proof |
-| `chunk_lens` | first window (`offset == 0`) | per-chunk ciphertext lengths of the WHOLE resource |
+| `chunk_lens` | prologue (once per stream, PAGED) | per-chunk ciphertext lengths of the WHOLE resource |
 | `source` | node profile | `"local"` or `"remote"` — where this node served it from |
 
 This table is normative and MUST agree field-for-field with `ChunkObject` in docs.dig.net's
@@ -959,9 +959,21 @@ isolation and MUST hold the complete resource before verifying. Every window car
 that whichever window a client happens to receive first can supply it — not so that windows can be
 verified independently.
 
-`chunk_lens` is the ONE field that rides the first window only. It describes how to split the
-REASSEMBLED resource, which a client cannot act on until it holds every window; a client that
-begins mid-resource therefore cannot decrypt a multi-chunk resource and MUST fetch window 0.
+`chunk_lens` is a PROLOGUE field: it describes how to split the REASSEMBLED resource, so a client
+cannot act on it until it holds every window, and a client that begins mid-resource cannot decrypt a
+multi-chunk resource and MUST fetch from the start. It is sent once per stream and MUST NOT be
+repeated. On the peer length-prefixed frame stream it is **PAGED**: a layout exceeding
+`dig_nat::MAX_CHUNK_LENS_PER_FRAME` (2048) entries cannot state itself on one frame, so it is split
+into pages of at most 2048 entries each, and every frame carrying a page is stamped with the
+`chunk_lens_offset` at which its page begins. When the requested bytes are exhausted before the
+layout is fully sent, the remaining pages ride trailing **prologue-only continuation frames** — a
+frame with a zero-length data payload that carries byte-`offset = 0` (NOT the ascending byte cursor,
+which by then equals the resource length and would trip a reader's `offset >= max_len` establish
+guard) and NO `chunk_index` (it begins no chunk, and a stale index would trip the reader's
+ascending-index rewind guard). A prologue-only frame does NOT terminate the stream; the stream is
+complete only once the bytes are exhausted AND every prologue page has been sent. (The single-frame
+JSON-RPC `dig.fetchRange` response is not framing-bound and carries the whole layout on its one
+frame.)
 
 **Window size.** A window is at most **3 MiB** of ciphertext. This node currently IGNORES the
 `length` request parameter and always serves a full window (or the remainder), where

crates/dig-node-core/Cargo.toml

@@ -265,7 +265,7 @@ dig-pex = "0.1"
 # `seams/dig_peer/module_anchor.rs` (`ChainAnchoredModuleVerifier`). NOTE the `testkit` feature is
 # deliberately NOT enabled: it is what makes the fail-OPEN `AcceptAnyModuleAnchor` nameable, and this
 # crate's anchor gate is the reshare path's ONLY root of trust.
-dig-download = "0.15"
+dig-download = "0.17"
 # -- The shared peer client (#1283/#1576) -------------------------------------------------------------
 # `DigPeer` — the ONE DIG Network peer client: peer_id-pinned mTLS over the full NAT ladder plus typed
 # RPC. Depended on DIRECTLY (not only transitively through dig-download) because dig-node supplies the
@@ -344,7 +344,7 @@ rcgen = "0.13"
 #
 # Pinned by the `the_fail_open_anchor_verifier_is_not_reachable_from_a_production_build` test, which
 # fails if `testkit` ever appears on the production entry.
-dig-download = { version = "0.15", features = ["testkit"] }
+dig-download = { version = "0.17", features = ["testkit"] }
 # Captures the peer-facing serve's real emitted tracing records into an in-memory buffer, so the
 # serve-observability tests (#1595) assert what an operator would actually see in the node log —
 # and that no payload byte or proof ever reaches it.

crates/dig-node-core/src/lib.rs

@@ -4376,6 +4376,35 @@ pub(crate) mod test_support {
         (Arc::new(resp), chunk_lens)
     }
 
+    /// A REAL served resource with `chunk_count` uniform `chunk_len`-byte chunks, committed under a
+    /// single-leaf generation root with its genuine digstore inclusion proof.
+    ///
+    /// It exists to exercise the PAGED prologue: a `chunk_count` above
+    /// [`dig_nat::MAX_CHUNK_LENS_PER_FRAME`] cannot state its `chunk_lens` on one frame, so the serve
+    /// path must split the layout across several frames and the reader must reassemble it. The chunks
+    /// are deliberately tiny — the point is the ENTRY COUNT of the layout, not the byte volume, so the
+    /// fixture stays small enough to build thousands of chunks cheaply.
+    pub(crate) fn many_chunk_served_resource(
+        chunk_count: usize,
+        chunk_len: usize,
+    ) -> (Arc<ContentResponse>, Vec<u64>) {
+        use digstore_core::merkle::{resource_leaf, MerkleTree};
+
+        let total = chunk_count * chunk_len;
+        // Byte i is `i mod 251` (a prime, so the pattern never aligns with a chunk boundary): a
+        // mis-ordered or dropped chunk changes the bytes, unlike a constant fill.
+        let ciphertext: Vec<u8> = (0..total).map(|i| (i % 251) as u8).collect();
+        let tree = MerkleTree::from_leaves(vec![resource_leaf(&ciphertext)]);
+        let resp = ContentResponse {
+            merkle_proof: tree.prove(0).expect("single-leaf proof"),
+            roothash: tree.root(),
+            chunk_lens: std::iter::repeat_n(chunk_len as u32, chunk_count).collect(),
+            ciphertext,
+        };
+        let chunk_lens = resp.chunk_lens.iter().map(|&l| u64::from(l)).collect();
+        (Arc::new(resp), chunk_lens)
+    }
+
     /// Seed `resource` into `node`'s memoized serve cache so [`Node::fetch_range_frame`] serves it,
     /// and return the `(store_id, root, retrieval_key)` hex triple that names it.
     pub(crate) fn seed_served_resource(

crates/dig-node-core/src/peer.rs

@@ -1855,7 +1855,14 @@ async fn stream_range_frames(
     loop {
         let take = range_frame::FRAME_PAYLOAD.min(end_of_span.saturating_sub(off));
         let window = bytes[off..off + take].to_vec();
-        let frame = framer.next_frame(off as u64, window);
+        // A trailing prologue-only continuation frame carries NO data payload — only the next
+        // `chunk_lens` page. It must be stamped with byte-offset 0, NOT the ascending cursor `off`
+        // (which by now equals the resource length): the 0.17 reader's establish probe reads a 1-byte
+        // span and rejects any frame whose `offset >= max_len`, so a page-only frame at `off == total`
+        // would be refused before its layout could be reassembled. A zero-length window is never a real
+        // byte position, so anchoring it at 0 loses nothing.
+        let frame_start = if take == 0 { 0 } else { off as u64 };
+        let frame = framer.next_frame(frame_start, window);
         // This frame is the FINAL one only when the bytes are done AND the prologue is fully sent.
         // The page this frame just took is already accounted for, so `prologue_pending` here answers
         // "are there pages still to come AFTER this frame".
@@ -5313,6 +5320,146 @@ pub(crate) mod tests {
         );
     }
 
+    /// **Proves the PAGED-prologue producer contract at the REAL wire (#2230).** A resource with more
+    /// than [`dig_nat::MAX_CHUNK_LENS_PER_FRAME`] chunks cannot state its whole `chunk_lens` on one
+    /// frame, so the serve path pages it: the layout rides several frames, and when the requested bytes
+    /// run out before the layout is fully sent, the remaining pages travel on trailing PROLOGUE-ONLY
+    /// frames (zero data payload).
+    ///
+    /// The two properties a conforming 0.17 reader depends on, asserted on the bytes actually written:
+    ///
+    /// * every prologue-only frame is stamped with byte-`offset 0` — NOT the ascending cursor, which by
+    ///   then equals the resource length and would trip the reader's `offset >= max_len` establish
+    ///   guard;
+    /// * a prologue-only frame carries NO `chunk_index` — it begins no chunk, and a stale index would
+    ///   trip the reader's ascending-index rewind guard.
+    ///
+    /// The stream must also NOT early-terminate: the full 2,049-entry layout is delivered across the
+    /// frames, so a paged read reassembles it in full.
+    #[tokio::test]
+    async fn a_paged_prologue_rides_offset_zero_frames_with_no_chunk_index_over_the_real_wire() {
+        // 2,049 chunks → one entry past the 2,048/page ceiling → exactly two prologue pages.
+        let chunk_count = dig_nat::MAX_CHUNK_LENS_PER_FRAME + 1;
+        let (resource, served_layout) =
+            crate::test_support::many_chunk_served_resource(chunk_count, 8);
+
+        let (node, _td) = crate::test_support::test_node_for_peer_surface();
+        let (store, root, rk) = crate::test_support::seed_served_resource(&node, resource);
+        let responder = NodeResponder::without_pool(node);
+        // A one-byte probe: the bytes run out on the first frame, forcing the second prologue page onto
+        // a trailing data-less frame — the exact shape the offset-0 fix governs.
+        let req = json!({
+            "store_id": store, "root": root, "retrieval_key": rk,
+            "offset": 0, "length": 1,
+        });
+
+        let (mut client, mut server) = tokio::io::duplex(256 * 1024);
+        let served = tokio::spawn(async move {
+            responder
+                .stream_range(req, &test_caller(), &mut server)
+                .await
+        });
+        served.await.expect("serve task join").expect("served");
+
+        let mut frames = Vec::new();
+        while let Some(frame) = read_framed(&mut client)
+            .await
+            .expect("no I/O error reading the frame stream")
+        {
+            frames.push(frame);
+        }
+
+        // A prologue-only frame carries a `chunk_lens` page but zero data bytes (`bytes` is base64, so
+        // an empty payload serializes to the empty string).
+        let prologue_only: Vec<&Value> = frames
+            .iter()
+            .filter(|f| f["bytes"].as_str() == Some("") && f.get("chunk_lens").is_some())
+            .collect();
+        assert!(
+            !prologue_only.is_empty(),
+            "a 2,049-chunk layout past the request span must trail a prologue-only frame: {frames:?}"
+        );
+        for frame in &prologue_only {
+            assert_eq!(
+                frame["offset"],
+                json!(0),
+                "a prologue-only frame must be stamped offset 0, not the ascending cursor: {frame:?}"
+            );
+            assert!(
+                frame.get("chunk_index").is_none(),
+                "a prologue-only frame begins no chunk, so it carries no chunk_index: {frame:?}"
+            );
+            assert!(
+                frame["chunk_lens_offset"].is_u64(),
+                "a prologue-only frame carries a located page: {frame:?}"
+            );
+        }
+
+        // The stream did not early-terminate: reassembling every page's entries yields the whole layout.
+        let mut reassembled: Vec<u64> = Vec::new();
+        for frame in &frames {
+            if let Some(page) = frame["chunk_lens"].as_array() {
+                reassembled.extend(page.iter().map(|v| v.as_u64().expect("chunk_lens entry")));
+            }
+        }
+        assert_eq!(
+            reassembled, served_layout,
+            "the paged prologue must reassemble to the full served layout, byte-for-byte"
+        );
+        assert!(
+            frames
+                .iter()
+                .filter(|f| f.get("chunk_lens").is_some())
+                .count()
+                >= 2,
+            "a 2,049-entry layout must span at least two prologue pages: {frames:?}"
+        );
+    }
+
+    /// **The primary end-to-end proof (#2230): the paged-prologue PRODUCER against the SHIPPED 0.17
+    /// reader.** Drives the production [`NodeResponder::stream_range`] over a real `tokio::io::duplex`
+    /// and feeds the raw wire bytes into `dig_download::assemble_range_stream` — the actual reassembler
+    /// a downloading peer runs — with the `max_len: 1` establish probe dig-download sends on every
+    /// download.
+    ///
+    /// Pre-fix this FAILS for the right reason: the trailing prologue page rides a frame stamped with
+    /// the ascending cursor (== the resource length), which the reader rejects as
+    /// `offset >= max_len`. After the fix the reader reassembles the full 2,049-entry layout with no
+    /// paged-prologue or offset error.
+    #[tokio::test]
+    async fn the_paged_prologue_producer_reads_end_to_end_through_the_shipped_reader() {
+        let chunk_count = dig_nat::MAX_CHUNK_LENS_PER_FRAME + 1;
+        let (resource, served_layout) =
+            crate::test_support::many_chunk_served_resource(chunk_count, 8);
+
+        let (node, _td) = crate::test_support::test_node_for_peer_surface();
+        let (store, root, rk) = crate::test_support::seed_served_resource(&node, resource);
+        let responder = NodeResponder::without_pool(node);
+        let req = json!({
+            "store_id": store, "root": root, "retrieval_key": rk,
+            "offset": 0, "length": 1,
+        });
+
+        let (mut client, mut server) = tokio::io::duplex(256 * 1024);
+        let served = tokio::spawn(async move {
+            responder
+                .stream_range(req, &test_caller(), &mut server)
+                .await
+        });
+        served.await.expect("serve task join").expect("served");
+
+        // The 1-byte establish probe dig-download's orchestrator sends on every download.
+        let (_bytes, meta) = dig_download::assemble_range_stream(&mut client, 1)
+            .await
+            .expect("the shipped 0.17 reader reassembles the paged prologue with no offset error");
+
+        assert_eq!(
+            meta.chunk_lens,
+            Some(served_layout),
+            "the reassembled layout must equal the full 2,049-entry served layout"
+        );
+    }
+
     #[tokio::test]
     async fn an_inbound_fetch_range_for_content_we_do_not_hold_logs_the_refusal() {
         // The ambiguity #1595 closes: a request the node cannot answer must say so in the log, so

crates/dig-node-core/src/seams/content/range_frame.rs

@@ -104,42 +104,6 @@ pub(crate) struct RangeVerification<'a> {
     pub inclusion_proof: Option<&'a str>,
 }
 
-/// Split `chunk_lens` into the pages a paged prologue is made of: page-aligned offsets, each page
-/// exactly [`dig_nat::MAX_CHUNK_LENS_PER_FRAME`] entries except a possibly-short tail.
-///
-/// This mirrors `dig_nat::RangeFrame::split_chunk_lens_pages`, which is the normative split and the
-/// reassembler's own mirror. It is reproduced here ONLY because that helper landed in dig-nat 0.14,
-/// while this node is held at 0.13 by dig-download and dig-gossip (see the dependency rationale in
-/// `Cargo.toml`); the moment the tree reaches 0.14 this function MUST be deleted in favour of it —
-/// tracked as DIG-Network/dig_ecosystem#1686 (the dig-gossip + dig-peer-selector cascade onto dig-nat
-/// ^0.14 / dig-dht ^0.8), so the mirror's removal is traceable from here rather than depending on
-/// someone remembering why it exists,
-/// because #1640 was precisely two sides of one rule maintained separately. The page SIZE is read from
-/// dig-nat either way, so the one number that matters cannot drift.
-///
-/// The shape is what the reassembler requires, and each requirement excludes a whole class rather
-/// than one observed misbehaviour:
-///
-/// * no page is EMPTY — an empty page fills nothing, so accepting one lets a sender stream frames
-///   forever without ever completing the prologue;
-/// * every page except the tail is exactly full — a short page anywhere but the end leaves a gap no
-///   page-aligned page can ever fill, so it is refused on arrival rather than surfacing later as an
-///   unexplained incompleteness;
-/// * an empty ARRAY yields no pages at all, which is a complete prologue for a resource with no chunk
-///   table rather than a stream that can never finish.
-fn chunk_lens_pages(chunk_lens: &[u64]) -> Vec<(u64, Vec<u64>)> {
-    chunk_lens
-        .chunks(dig_nat::MAX_CHUNK_LENS_PER_FRAME)
-        .enumerate()
-        .map(|(page, entries)| {
-            (
-                (page * dig_nat::MAX_CHUNK_LENS_PER_FRAME) as u64,
-                entries.to_vec(),
-            )
-        })
-        .collect()
-}
-
 /// The absolute chunk index that byte `offset` begins, or `None` when `offset` is not on a chunk
 /// boundary (or lies past the resource).
 ///
@@ -186,7 +150,11 @@ impl<'a> RangeStreamFramer<'a> {
         let pending_pages = if skip_layout {
             VecDeque::new()
         } else {
-            chunk_lens_pages(verification.chunk_lens).into()
+            // The normative split lives in dig-nat, published together with its reassembler mirror so
+            // encode + decode cannot drift (#1640/#1686): page-aligned offsets, each page exactly
+            // `MAX_CHUNK_LENS_PER_FRAME` entries except a possibly-short tail; an empty layout yields no
+            // pages, which is a complete prologue for a resource with no chunk table.
+            dig_nat::RangeFrame::split_chunk_lens_pages(verification.chunk_lens).into()
         };
         RangeStreamFramer {
             verification,
@@ -205,6 +173,7 @@ impl<'a> RangeStreamFramer<'a> {
     /// afterwards and applies `with_complete` itself. Setting `complete` on a frame that still owes
     /// pages would stop a conforming reader before the layout it needs to DECRYPT ever arrives.
     pub(crate) fn next_frame(&mut self, start: u64, bytes: Vec<u8>) -> dig_nat::RangeFrame {
+        let has_payload = !bytes.is_empty();
         let mut frame = dig_nat::RangeFrame::data(start, bytes);
 
         // The identity set rides EVERY frame. `chunk_count` is the resource's TOTAL entry count, so a
@@ -229,8 +198,14 @@ impl<'a> RangeStreamFramer<'a> {
             frame.total_length = Some(self.verification.total_length);
             frame.chunk_count = Some(chunk_count);
         }
-        if let Some(index) = chunk_index_at(self.verification.chunk_lens, start) {
-            frame = frame.with_chunk_index(index);
+        // A chunk index identifies which chunk this frame's PAYLOAD begins. A zero-payload frame (a
+        // trailing prologue-only continuation) begins no chunk, so it must carry no index: the 0.17
+        // reader's rewind guard rejects a frame whose `chunk_index < highest_chunk_index`, and a
+        // page-only frame stamped with chunk 0 after real chunks have gone out would trip exactly that.
+        if has_payload {
+            if let Some(index) = chunk_index_at(self.verification.chunk_lens, start) {
+                frame = frame.with_chunk_index(index);
+            }
         }
 
         if self.skip_layout {