From 25da38c480b668a42c7951b792f522efd0c81968 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:33:07 -0300 Subject: [PATCH 01/33] docs: add early aggregation start design spec --- ...26-07-01-early-aggregation-start-design.md | 167 ++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md diff --git a/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md b/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md new file mode 100644 index 00000000..58dc8e90 --- /dev/null +++ b/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md @@ -0,0 +1,167 @@ +# Early Aggregation Start + +**Date:** 2026-07-01 +**Status:** Approved + +## Motivation + +Committee-signature aggregation (leanVM XMSS proofs) is the dominant cost in the +attestation pipeline. Today the aggregation session starts exactly at the +interval-2 tick, even when all expected signatures arrived earlier. Starting the +session up to 400 ms early, when enough signatures are already present, gives the +expensive proofs more wall-clock runway before block building consumes them at +interval 4. + +## Current behavior + +- The blockchain actor ticks at 800 ms interval boundaries. At interval 2 (if + `is_aggregator`), `start_aggregation_session` snapshots the current slot's + gossip signatures (`snapshot_current_slot_aggregation_inputs`) and spawns an + off-thread worker. A deadline timer cancels the session's token after + `AGGREGATION_DEADLINE` (750 ms). +- Gossip signatures accumulate in the store per attestation-data group as + `NewAttestation` messages arrive (plus self-delivery of own validators' + attestations at interval 1). Only aggregators store them. +- Each produced aggregate is applied to the store and published to gossip + immediately in the `AggregateProduced` handler. +- Subnet assignment is `vid % attestation_committee_count`. The set of subnets a + node subscribes to (own-validator subnets, plus `--aggregate-subnet-ids` for + aggregators, with a fallback to subnet 0 for validator-less aggregators) is + computed inside `build_swarm` in the p2p crate; the blockchain actor does not + know it. + +## New behavior + +### Early trigger + +Aggregator-only, fires at most once per slot. + +- **Window:** wall clock in `[T2 - 400 ms, T2)`, where + `T2 = genesis_ms + slot * MILLISECONDS_PER_SLOT + 2 * MILLISECONDS_PER_INTERVAL` + is the current slot's interval-2 boundary. +- **Condition:** some current-slot attestation-data group has gossip signatures + from at least 2/3 of the expected validators: + `group_count * 3 >= expected * 2`, with `expected > 0`. `group_count` is the + number of stored gossip signatures in that group (one per validator). + `expected` = number of head-state validators whose + `vid % attestation_committee_count` is in this node's aggregation subnet set. + Note that at most one group per slot can reach 2/3 (each validator signs once + per slot), so the per-group condition can hold for at most one group. +- **Check sites:** + 1. After each gossip-signature insert in the `NewAttestation` handler, while + the wall clock is inside the window. + 2. A one-shot `EarlyAggregationCheck { slot }` self-message scheduled at the + interval-1 tick via `send_after(400 ms)` (interval-1 tick + 400 ms = + T2 - 400 ms), covering the case where the threshold was already met before + the window opened. If the interval-1 tick is skipped (overrun), the + per-insert checks still apply. +- **Once-per-slot guard:** a session with `session_id == slot` already exists in + `current_aggregation` (the field persists after the worker finishes; it is + only replaced at the next session start). + +### Action on trigger + +Call the existing `start_aggregation_session(slot, ctx)` — the full current-slot +snapshot, not just the group that hit the threshold. This is the slot's one and +only session; it merely starts early. The existing prior-session cancel+join in +`start_aggregation_session` handles a still-running previous-slot worker +identically in both paths. + +### Interval-2 tick + +If `current_aggregation` already holds this slot's session (running or +finished), skip starting a new one. Otherwise start normally — the unchanged +fallback for slots where the threshold is never met. + +`emit_agg_start_new_coverage` moves from the interval-2 tick into +`start_aggregation_session`, so the coverage report reflects the store state the +session actually snapshotted in both paths. + +### Publish alignment + +Aggregates must not reach gossip before interval 2 (mirrors the block prebuild +pattern: build early, publish aligned to the boundary). + +- `AggregateProduced` handler: apply the output to the store immediately + (nothing local consumes `new_aggregated_payloads` before interval 4, so early + application is unobservable locally). If the wall clock is `>= T2` for the + session's slot, publish immediately (unchanged). If `< T2`, push the + `SignedAggregatedAttestation` into a pending-publish buffer on the actor. +- When starting an *early* session, schedule a `FlushAggregatePublishes` + self-message via `send_after(T2 - now)`. Its handler publishes and drains the + buffer. This is the primary flush mechanism (not the interval-2 tick, which + can be skipped on overrun); the handler is idempotent (draining an empty + buffer is a no-op). +- The buffer is bounded in practice by the number of data groups in one slot. + Entries are never carried across slots: the flush timer fires at T2 of the + same slot that buffered them. + +### Deadline change + +`AGGREGATION_DEADLINE`: 750 ms → 800 ms, still measured from session start. + +- Early session started at `T2 - x` (x ≤ 400): deadline at `T2 - x + 800`, + i.e. at most `T2 + 400`. +- Normal session started at `T2`: deadline lands exactly on the interval-3 + boundary. This removes the previous 50 ms publish/propagation margin — + accepted trade-off (the deadline only stops *new* jobs from starting; a job + mid-proof finishes and publishes slightly after). + +### Subnet-set plumbing + +The subscription-subnet computation is hoisted out of `build_swarm` into a +shared pure helper (in the p2p crate, e.g. +`compute_subscription_subnets(validator_ids, committee_count, is_aggregator, explicit_ids) -> HashSet`). +`main.rs` calls it once and passes the resulting set to both: + +- the p2p config (which subscribes to exactly this set, behavior unchanged), and +- the `BlockChainServer` (new field, used for the `expected` count). + +The set is frozen at startup, matching the existing hot-standby aggregator +model (runtime toggles do not resubscribe subnets). + +## Failure modes and edge cases + +- **Snapshot returns `None` after trigger** (should not happen — the trigger + requires signatures to be present): no session is created, + `current_aggregation` keeps the prior slot's id, and interval 2 retries + normally. No signatures are lost. +- **Threshold never met:** interval 2 starts the session exactly as today. +- **Signatures arriving after the early snapshot:** never aggregated that slot. + Accepted trade-off (explicitly chosen over a top-up session). +- **Wall-clock vs monotonic drift:** checks use `unix_now_ms()` like the rest of + the tick logic; the idempotency-guard pattern already tolerates drift. The + worst case of drift is a slightly mistimed early start or flush, never a + correctness issue. +- **Non-aggregators:** never store gossip signatures, and all check sites are + additionally guarded by `AggregatorController::is_enabled()` read at check + time. + +## Observability + +- Counter `lean_aggregation_early_starts_total`: early sessions started. +- Histogram for early-start lead time (`T2 - start`, seconds, buckets covering + 0–400 ms). +- Existing session logs gain an `early: bool` field so devnet log analysis can + attribute publish-time shifts. + +## Testing + +- Unit tests for the pure trigger decision (window bounds, threshold math, + `expected` derivation from subnet set + validator count, once-per-slot guard). +- Unit tests for `compute_subscription_subnets` (validator subnets, explicit + ids, fallback-to-0, non-aggregator). +- Unit test for the publish-delay decision (before/after T2). +- Existing aggregation tests must pass unchanged (normal interval-2 path is the + fallback and keeps its semantics; only the deadline constant changes). +- Devnet validation via `test-branch.sh`: compare aggregate publish offsets + (per-slot duty timing method) and `lean_aggregation_early_starts_total`. + +## Non-goals + +- No gossip resubscription or topic changes. +- No change to the fork-choice attestation pipeline (`new_attestations` / + `known_attestations`). +- No top-up/second session per slot. +- No persistence of the pending-publish buffer (in-memory actor state, same as + the rest of the aggregation pipeline). From 5e4cfbf6b61d9926509f3ce1723835456e90ec92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:42:29 -0300 Subject: [PATCH 02/33] docs: add early aggregation start implementation plan --- .../2026-07-01-early-aggregation-start.md | 932 ++++++++++++++++++ ...26-07-01-early-aggregation-start-design.md | 21 +- 2 files changed, 945 insertions(+), 8 deletions(-) create mode 100644 docs/superpowers/plans/2026-07-01-early-aggregation-start.md diff --git a/docs/superpowers/plans/2026-07-01-early-aggregation-start.md b/docs/superpowers/plans/2026-07-01-early-aggregation-start.md new file mode 100644 index 00000000..69d193f9 --- /dev/null +++ b/docs/superpowers/plans/2026-07-01-early-aggregation-start.md @@ -0,0 +1,932 @@ +# Early Aggregation Start Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Start the committee-signature aggregation session up to 400 ms before the interval-2 boundary when a single attestation-data group already holds 2/3 of the signatures expected from this node's aggregation subnets, while holding back any early-produced aggregates from gossip until the boundary. + +**Architecture:** The blockchain actor (`BlockChainServer`, GenServer pattern) gains an early-trigger check invoked from two sites: after every stored gossip signature (`NewAttestation` handler) and once via a timer fired at the window opening (`T2 − 400 ms`). The trigger calls the existing `start_aggregation_session`, which becomes the single session-start path for both early and normal (interval-2 tick) starts; the tick skips if the slot's session already exists. Aggregates produced before T2 are buffered on the actor and flushed by a `send_after` timer at T2. + +**Tech Stack:** Rust (edition 2024), spawned-concurrency actors (`send_after` self-messages), prometheus metrics via `ethlambda_metrics`. + +**Spec:** `docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md` + +**Key existing code:** +- `crates/blockchain/src/lib.rs` — actor, tick loop (`on_tick`), `start_aggregation_session` (~line 366), handlers (`NewAttestation` ~line 1034, `AggregateProduced` ~line 1050, `AggregationDone` ~line 1079, `AggregationDeadline` ~line 1099) +- `crates/blockchain/src/aggregation.rs` — session types, snapshot builders, worker, `AGGREGATION_DEADLINE` +- `crates/blockchain/src/metrics.rs` — prometheus registration patterns +- `crates/storage/src/store.rs` — `GossipSignatureBuffer` (~line 348), `Store` methods (~line 1451), test mod (~line 1555) with `make_att_data(slot)`, `make_att_data_for_target(slot, root)`, `make_dummy_sig()` helpers +- `crates/net/p2p/src/lib.rs` — `build_swarm` subnet subscription (~lines 304–337), test mod (~line 790) +- `bin/ethlambda/src/main.rs` — `BlockChain::spawn` (~line 213), `build_swarm(SwarmConfig {...})` (~line 230) + +**Timing model (4 s slots, 5 × 800 ms intervals):** + +``` +slot start T1=+800ms T2=+1600ms T3=+2400ms T4=+3200ms + |------------|------------|------------|------------| + ^ attest ^ aggregate ^ safe tgt ^ accept/build + [T2-400, T2) = early window + ^ timer check (scheduled at T1 + 400ms) + ^..^..^ per-insert checks as gossip sigs arrive +``` + +--- + +### Task 1: Store query — max gossip group count for a slot + +The early threshold needs "largest signature count among current-slot data groups" without cloning signatures (the existing `iter_gossip_signatures` snapshot clones every signature; signatures are ~3 KB each). + +**Files:** +- Modify: `crates/storage/src/store.rs` (buffer method after `snapshot()` ~line 465; `Store` method next to `gossip_signatures_count()` ~line 1451; test in existing `mod tests`) + +- [ ] **Step 1: Write the failing test** + +Add to the `// ============ GossipSignatureBuffer Tests ============` section of `mod tests` in `crates/storage/src/store.rs` (near `gossip_buffer_fifo_eviction`, ~line 2423): + +```rust +#[test] +fn gossip_buffer_max_group_count_for_slot() { + let mut buf = GossipSignatureBuffer::new(100); + assert_eq!(buf.max_group_count_for_slot(1), 0); + + // Slot 1, group A: 3 validators. + for vid in 0..3 { + let data = make_att_data(1); + buf.insert(HashedAttestationData::new(data), vid, make_dummy_sig()); + } + // Slot 1, group B (same slot, different target root): 2 validators. + for vid in 3..5 { + let data = make_att_data_for_target(1, root(99)); + buf.insert(HashedAttestationData::new(data), vid, make_dummy_sig()); + } + // Slot 2: 5 validators. + for vid in 0..5 { + let data = make_att_data(2); + buf.insert(HashedAttestationData::new(data), vid, make_dummy_sig()); + } + + assert_eq!(buf.max_group_count_for_slot(1), 3); + assert_eq!(buf.max_group_count_for_slot(2), 5); + assert_eq!(buf.max_group_count_for_slot(3), 0); +} +``` + +Note: `make_att_data_for_target` is defined further down in the test mod (~line 2319); Rust test mods don't care about definition order. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cargo test -p ethlambda-storage max_group_count_for_slot` +Expected: FAIL to compile with "no method named `max_group_count_for_slot`" + +- [ ] **Step 3: Implement the buffer method and Store wrapper** + +In `impl GossipSignatureBuffer`, after `snapshot()` (~line 465): + +```rust +/// Largest signature count among data groups whose attestation slot is `slot`. +fn max_group_count_for_slot(&self, slot: u64) -> usize { + self.data + .values() + .filter(|entry| entry.data.slot == slot) + .map(|entry| entry.signatures.len()) + .max() + .unwrap_or(0) +} +``` + +In `impl Store`, next to `gossip_signatures_count()` (~line 1451): + +```rust +/// Largest per-group signature count among gossip groups voting for `slot`. +/// +/// One lock, no signature clones — cheap enough to call per gossip insert. +/// Drives the early-aggregation threshold check. +pub fn max_gossip_group_count_for_slot(&self, slot: u64) -> usize { + let gossip = self.gossip_signatures.lock().unwrap(); + gossip.max_group_count_for_slot(slot) +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cargo test -p ethlambda-storage max_group_count_for_slot` +Expected: PASS (1 test) + +- [ ] **Step 5: Commit** + +```bash +git add crates/storage/src/store.rs +git commit -m "feat(storage): add max gossip group count query for early aggregation" +``` + +--- + +### Task 2: p2p — extract `compute_subscription_subnets` helper + +Single source of truth for the subnet set, callable from `main.rs` (Task 4) without touching `SwarmConfig`. + +**Files:** +- Modify: `crates/net/p2p/src/lib.rs` (subnet logic in `build_swarm` ~lines 304–337; tests in existing `mod tests` ~line 790) + +- [ ] **Step 1: Write the failing tests** + +Add to `mod tests` in `crates/net/p2p/src/lib.rs` (if the test mod doesn't already have `HashSet` in scope via `use super::*;`, add `use std::collections::HashSet;`): + +```rust +#[test] +fn subscription_subnets_validators_only() { + // 3 % 4 = 3, 7 % 4 = 3, 10 % 4 = 2 + let subnets = compute_subscription_subnets(&[3, 7, 10], 4, false, None); + assert_eq!(subnets, HashSet::from([3, 2])); +} + +#[test] +fn subscription_subnets_aggregator_unions_explicit_ids() { + let subnets = compute_subscription_subnets(&[1], 4, true, Some(&[0, 2])); + assert_eq!(subnets, HashSet::from([1, 0, 2])); +} + +#[test] +fn subscription_subnets_validatorless_aggregator_falls_back_to_zero() { + let subnets = compute_subscription_subnets(&[], 4, true, None); + assert_eq!(subnets, HashSet::from([0])); +} + +#[test] +fn subscription_subnets_validatorless_non_aggregator_is_empty() { + let subnets = compute_subscription_subnets(&[], 4, false, None); + assert!(subnets.is_empty()); +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test -p ethlambda-p2p subscription_subnets` +Expected: FAIL to compile with "cannot find function `compute_subscription_subnets`" + +- [ ] **Step 3: Implement the helper and use it in `build_swarm`** + +Add near `build_swarm` (above it, ~line 185): + +```rust +/// Compute the set of attestation subnets this node subscribes to (and, when +/// aggregating, aggregates over), per leanSpec (`src/lean_spec/__main__.py`): +/// every validator subscribes to its own subnet (`vid % committee_count`) for +/// mesh health; aggregators additionally subscribe to explicit +/// `aggregate_subnet_ids` and fall back to subnet 0 when the set would +/// otherwise be empty. +/// +/// Evaluated once at startup — runtime aggregator toggles do not resubscribe +/// (hot-standby model); see the invariant note on [`SwarmConfig`]. +pub fn compute_subscription_subnets( + validator_ids: &[u64], + attestation_committee_count: u64, + is_aggregator: bool, + aggregate_subnet_ids: Option<&[u64]>, +) -> HashSet { + let mut subnets: HashSet = validator_ids + .iter() + .map(|vid| vid % attestation_committee_count) + .collect(); + if is_aggregator { + if let Some(explicit_ids) = aggregate_subnet_ids { + subnets.extend(explicit_ids); + } + if subnets.is_empty() { + subnets.insert(0); + } + } + subnets +} +``` + +Then replace the inline logic in `build_swarm` (~lines 304–329). The metric must keep reflecting validator-only subnets, so that part stays: + +```rust + // Subscribe to attestation subnets — see `compute_subscription_subnets`. + // The committee metric should reflect validator membership only, not + // aggregator-only subscriptions. + let metric_subnet = config + .validator_ids + .iter() + .map(|vid| vid % config.attestation_committee_count) + .min() + .unwrap_or(0); + metrics::set_attestation_committee_subnet(metric_subnet); + + let subscription_subnets = compute_subscription_subnets( + &config.validator_ids, + config.attestation_committee_count, + config.is_aggregator, + config.aggregate_subnet_ids.as_deref(), + ); +``` + +(The `let mut attestation_topics ...` loop over `&subscription_subnets` at ~line 331 stays unchanged.) + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cargo test -p ethlambda-p2p subscription_subnets` +Expected: PASS (4 tests) + +Run: `cargo build -p ethlambda-p2p` +Expected: clean build (no unused warnings; the old inline block is fully replaced) + +- [ ] **Step 5: Commit** + +```bash +git add crates/net/p2p/src/lib.rs +git commit -m "refactor(p2p): extract subscription subnet computation into pure helper" +``` + +--- + +### Task 3: Blockchain — pure early-aggregation helpers + +Window math and threshold predicate, unit-testable without an actor. + +**Files:** +- Modify: `crates/blockchain/src/aggregation.rs` (new consts + functions after `PRIOR_WORKER_JOIN_TIMEOUT` ~line 37; new `#[cfg(test)] mod tests` at end of file) + +- [ ] **Step 1: Write the failing tests** + +`crates/blockchain/src/aggregation.rs` has no test mod yet. Add at the end of the file: + +```rust +#[cfg(test)] +mod tests { + use super::*; + use crate::{MILLISECONDS_PER_INTERVAL, MILLISECONDS_PER_SLOT}; + + const GENESIS_MS: u64 = 1_000_000; + + /// Interval-2 boundary of `slot`, relative to the test genesis. + fn t2(slot: u64) -> u64 { + GENESIS_MS + slot * MILLISECONDS_PER_SLOT + 2 * MILLISECONDS_PER_INTERVAL + } + + #[test] + fn interval2_boundary_matches_slot_layout() { + assert_eq!(interval2_boundary_ms(GENESIS_MS, 0), GENESIS_MS + 1_600); + assert_eq!(interval2_boundary_ms(GENESIS_MS, 5), GENESIS_MS + 5 * 4_000 + 1_600); + } + + #[test] + fn early_window_bounds() { + // Window is [T2 - EARLY_AGGREGATION_WINDOW_MS, T2). + let before_window = t2(5) - EARLY_AGGREGATION_WINDOW_MS - 1; + assert_eq!(early_aggregation_slot(before_window, GENESIS_MS), None); + let window_open = t2(5) - EARLY_AGGREGATION_WINDOW_MS; + assert_eq!(early_aggregation_slot(window_open, GENESIS_MS), Some(5)); + assert_eq!(early_aggregation_slot(t2(5) - 1, GENESIS_MS), Some(5)); + assert_eq!(early_aggregation_slot(t2(5), GENESIS_MS), None); + // Slot 0 has a window too. + assert_eq!(early_aggregation_slot(t2(0) - 1, GENESIS_MS), Some(0)); + } + + #[test] + fn early_window_before_genesis_is_none() { + assert_eq!(early_aggregation_slot(GENESIS_MS - 1, GENESIS_MS), None); + } + + #[test] + fn threshold_is_two_thirds_of_expected() { + // Zero expected: never trigger (validator-less edge, div-by-zero guard). + assert!(!early_threshold_met(0, 0)); + assert!(!early_threshold_met(5, 0)); + // Exact 2/3 triggers. + assert!(early_threshold_met(2, 3)); + assert!(!early_threshold_met(1, 3)); + // Ceiling behavior when expected isn't divisible by 3. + assert!(early_threshold_met(3, 4)); // 9 >= 8 + assert!(!early_threshold_met(2, 4)); // 6 < 8 + // Devnet-scale sanity: 33 validators, one subnet. + assert!(early_threshold_met(22, 33)); + assert!(!early_threshold_met(21, 33)); + } +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test -p ethlambda-blockchain --lib aggregation::tests` +Expected: FAIL to compile with "cannot find function `early_aggregation_slot`" (and the others) + +- [ ] **Step 3: Implement consts and functions** + +In `crates/blockchain/src/aggregation.rs`, after `PRIOR_WORKER_JOIN_TIMEOUT` (~line 37). Also add `use crate::{MILLISECONDS_PER_INTERVAL, MILLISECONDS_PER_SLOT};` to the file's imports (both constants are `pub` in `lib.rs`). + +```rust +/// Width of the early-aggregation window: a session may start at most this +/// long before the interval-2 boundary, provided the signature threshold is +/// met (see `early_threshold_met`). +pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 400; + +/// Wall-clock millisecond timestamp of `slot`'s interval-2 boundary (the +/// normal aggregation start). +pub(crate) fn interval2_boundary_ms(genesis_time_ms: u64, slot: u64) -> u64 { + genesis_time_ms + slot * MILLISECONDS_PER_SLOT + 2 * MILLISECONDS_PER_INTERVAL +} + +/// If `now_ms` falls inside some slot's early-aggregation window +/// (`[T2 - EARLY_AGGREGATION_WINDOW_MS, T2)` with `T2` that slot's interval-2 +/// boundary), return that slot. +pub(crate) fn early_aggregation_slot(now_ms: u64, genesis_time_ms: u64) -> Option { + let since_genesis = now_ms.checked_sub(genesis_time_ms)?; + let ms_into_slot = since_genesis % MILLISECONDS_PER_SLOT; + let t2_offset = 2 * MILLISECONDS_PER_INTERVAL; + let in_window = + ms_into_slot >= t2_offset - EARLY_AGGREGATION_WINDOW_MS && ms_into_slot < t2_offset; + in_window.then_some(since_genesis / MILLISECONDS_PER_SLOT) +} + +/// Early-start threshold: a single attestation-data group holds at least 2/3 +/// of the signatures expected from this node's aggregation subnets. At most +/// one group per slot can satisfy this (each validator signs once per slot). +pub(crate) fn early_threshold_met(max_group_count: usize, expected: usize) -> bool { + expected > 0 && max_group_count * 3 >= expected * 2 +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cargo test -p ethlambda-blockchain --lib aggregation::tests` +Expected: PASS (4 tests). Dead-code warnings for the not-yet-used functions are acceptable at this step only if the compiler emits them (`pub(crate)` items used from tests usually don't warn); they get consumed in Tasks 6–7. + +- [ ] **Step 5: Commit** + +```bash +git add crates/blockchain/src/aggregation.rs +git commit -m "feat(blockchain): add early-aggregation window and threshold helpers" +``` + +--- + +### Task 4: Plumbing — subnet set into the actor, expected-count field + +Compile-level wiring, no behavior change yet. + +**Files:** +- Modify: `crates/blockchain/src/lib.rs` (`BlockChain::spawn` ~line 80, `BlockChainServer` struct ~line 138) +- Modify: `bin/ethlambda/src/main.rs` (~lines 206–223) + +- [ ] **Step 1: Add spawn parameter and fields** + +In `crates/blockchain/src/lib.rs`, change `BlockChain::spawn` signature (~line 80) — new `aggregation_subnets` parameter after `attestation_committee_count`: + +```rust + pub fn spawn( + store: Store, + validator_keys: HashMap, + aggregator: AggregatorController, + attestation_committee_count: u64, + aggregation_subnets: HashSet, + gate_duties: bool, + proposer_config: ProposerConfig, + ) -> BlockChain { +``` + +Inside `spawn`, before the `let handle = BlockChainServer {` block (~line 101), compute the expected count (guard the modulo against a zero committee count): + +```rust + // Denominator of the early-aggregation 2/3 threshold: how many + // validators attest on subnets this node aggregates. Computed once — + // the validator registry is static and `head_state()` clones the full + // state, so this must not run per gossip insert. + let validator_count = store.head_state().validators.len() as u64; + let early_aggregation_expected_sigs = if attestation_committee_count == 0 { + 0 + } else { + (0..validator_count) + .filter(|vid| { + aggregation_subnets.contains(&(vid % attestation_committee_count)) + }) + .count() + }; +``` + +Add to the `BlockChainServer` struct literal in `spawn`: + +```rust + early_aggregation_expected_sigs, + pending_aggregate_publishes: Vec::new(), +``` + +Add the fields to the `BlockChainServer` struct definition (after `attestation_committee_count`, ~line 170): + +```rust + /// Number of validators whose subnet is one this node aggregates — the + /// denominator of the early-aggregation 2/3 threshold. Computed once at + /// spawn (the validator registry is static). + early_aggregation_expected_sigs: usize, + + /// Aggregates produced before the interval-2 boundary, held back so they + /// are not gossiped early. Flushed by `FlushAggregatePublishes` at the + /// boundary; only ever holds the current slot's aggregates. + pending_aggregate_publishes: Vec, +``` + +(`HashSet` and `SignedAggregatedAttestation` are already imported in `lib.rs`.) + +- [ ] **Step 2: Update `main.rs`** + +In `bin/ethlambda/src/main.rs`, add `compute_subscription_subnets` to the `ethlambda_p2p` import (line 40): + +```rust +use ethlambda_p2p::{ + Bootnode, P2P, PeerId, SwarmConfig, build_swarm, compute_subscription_subnets, parse_enrs, +}; +``` + +Before `BlockChain::spawn` (~line 213), compute the set with the same startup inputs `build_swarm` uses, and pass it (borrow `aggregate_subnet_ids` — it is moved into `SwarmConfig` further down): + +```rust + // Same startup inputs build_swarm uses — single source of truth is the + // shared helper. Frozen at startup like the gossip subscriptions + // themselves (hot-standby model). + let aggregation_subnets = compute_subscription_subnets( + &validator_ids, + attestation_committee_count, + options.is_aggregator, + options.aggregate_subnet_ids.as_deref(), + ); + + let blockchain = BlockChain::spawn( + store.clone(), + validator_keys, + aggregator.clone(), + attestation_committee_count, + aggregation_subnets, + !options.disable_duty_sync_gate, + ProposerConfig { + enable_proposer_aggregation: options.enable_proposer_aggregation, + max_attestations_per_block: options.max_attestations_per_block, + }, + ); +``` + +- [ ] **Step 3: Build the workspace** + +Run: `cargo build --workspace` +Expected: clean build. If `pending_aggregate_publishes` / `early_aggregation_expected_sigs` trigger dead-code warnings, silence is NOT needed — they are read in Tasks 6–7; if the build fails on `-D warnings` locally, proceed (only `make lint` enforces it) or add the fields in the task where they're first read. Do not annotate with `#[allow(dead_code)]`. + +Note: plain `cargo build` does not deny warnings, so this passes; the fields become live two tasks later, before `make lint` runs in Task 8. + +- [ ] **Step 4: Commit** + +```bash +git add crates/blockchain/src/lib.rs bin/ethlambda/src/main.rs +git commit -m "feat(blockchain): plumb aggregation subnet set and expected-signature count into actor" +``` + +--- + +### Task 5: Deadline change 750 → 800 ms + +**Files:** +- Modify: `crates/blockchain/src/aggregation.rs:28-33` + +- [ ] **Step 1: Change the constant and its comment** + +Replace: + +```rust +/// Soft deadline for committee-signature aggregation measured from the +/// interval-2 tick. After this much wall time elapses, the actor signals the +/// worker to stop via its cancellation token. The 50 ms budget before the next +/// interval (interval 3 at +800 ms) is reserved for publishing any late-arriving +/// aggregates and for gossip propagation margin. +pub(crate) const AGGREGATION_DEADLINE: Duration = Duration::from_millis(750); +``` + +with: + +```rust +/// Soft deadline for committee-signature aggregation measured from session +/// start. After this much wall time elapses, the actor signals the worker to +/// stop via its cancellation token. A session started exactly at interval 2 +/// gets the full interval (interval 3 is one interval later); a session +/// started early (see `early_aggregation_slot`) ends correspondingly earlier. +/// The deadline only stops new jobs from starting — a job mid-proof finishes +/// and publishes right after. +pub(crate) const AGGREGATION_DEADLINE: Duration = Duration::from_millis(800); +``` + +- [ ] **Step 2: Build and check references** + +Run: `cargo build -p ethlambda-blockchain && grep -rn "750" crates/blockchain/src/lib.rs` +Expected: clean build; the grep finds no stale `+750 ms` comment references (the doc comment on `start_aggregation_session` ~line 365 says "Schedule the `AggregationDeadline` self-message at +750 ms" — update it to "+`AGGREGATION_DEADLINE`"). + +- [ ] **Step 3: Commit** + +```bash +git add crates/blockchain/src/aggregation.rs crates/blockchain/src/lib.rs +git commit -m "feat(blockchain): extend aggregation deadline to a full interval" +``` + +--- + +### Task 6: Publish alignment — hold early aggregates until the interval-2 boundary + +**Files:** +- Modify: `crates/blockchain/src/aggregation.rs` (`AggregationSession` struct ~line 75, new message) +- Modify: `crates/blockchain/src/lib.rs` (imports ~line 16, `start_aggregation_session` ~line 366, `AggregateProduced` handler ~line 1050, `AggregationDone` handler ~line 1079, interval-2 tick arm ~line 312, new handler + helper) + +- [ ] **Step 1: Add the `early` flag and the flush message in `aggregation.rs`** + +Add field to `AggregationSession` (~line 75): + +```rust +pub(crate) struct AggregationSession { + /// Slot at which this session was started; used as a fencing id so we can + /// drop late-arriving messages from a prior session. + pub(crate) session_id: u64, + /// Whether the session started before the slot's interval-2 boundary via + /// the early-aggregation trigger. + pub(crate) early: bool, + /// Child of the actor cancellation token; fires either at the deadline or + /// when the actor itself is stopping. + pub(crate) cancel: CancellationToken, + /// Handle to the `spawn_blocking` worker. Held so `stopped()` / new-session + /// start can await completion. + pub(crate) worker: tokio::task::JoinHandle<()>, +} +``` + +Add next to `AggregationDeadline` (~line 112): + +```rust +/// Self-message scheduled when a session starts early; fires at the +/// interval-2 boundary and publishes any aggregates held back by the +/// publish-alignment rule (aggregates must not reach gossip before +/// interval 2). +pub(crate) struct FlushAggregatePublishes; +impl Message for FlushAggregatePublishes { + type Result = (); +} +``` + +- [ ] **Step 2: Rework `start_aggregation_session` in `lib.rs`** + +Extend the `use crate::aggregation::{...}` import (~line 16) with `FlushAggregatePublishes` (and, for Task 7, `EarlyAggregationCheck` will join it). + +In `start_aggregation_session` (~line 366): move the coverage emission here from the tick arm, and compute `early` + schedule the flush. After the prior-session join block and before the snapshot: + +```rust + coverage::emit_agg_start_new_coverage(&self.store, self.attestation_committee_count); +``` + +Replace the tail of the function (from `let session_id = slot;` through the final `self.current_aggregation = Some(...)`) with: + +```rust + let session_id = slot; + let genesis_time_ms = self.store.config().genesis_time * 1000; + let t2_ms = aggregation::interval2_boundary_ms(genesis_time_ms, slot); + let now_ms = unix_now_ms(); + let early = now_ms < t2_ms; + if early { + // Publish alignment: aggregates must not reach gossip before the + // interval-2 boundary. Aggregates produced before T2 are buffered + // in `pending_aggregate_publishes`; this timer flushes them at T2. + let lead = Duration::from_millis(t2_ms - now_ms); + metrics::inc_aggregation_early_starts(); + metrics::observe_aggregation_early_start_lead(lead); + info!( + %slot, + lead_ms = lead.as_millis() as u64, + "Starting aggregation session early" + ); + send_after(lead, ctx.clone(), FlushAggregatePublishes); + } + + // Independent token per session. Shutdown propagates via our + // #[stopped] hook which cancels any current session; the deadline + // timer cancels this specific session at +AGGREGATION_DEADLINE. + let cancel = CancellationToken::new(); + let actor_ref = ctx.actor_ref(); + + let worker_cancel = cancel.clone(); + let worker_actor = actor_ref.clone(); + let worker = tokio::task::spawn_blocking(move || { + run_aggregation_worker(snapshot, worker_actor, worker_cancel, session_id); + }); + + let _deadline_timer = send_after( + AGGREGATION_DEADLINE, + ctx.clone(), + AggregationDeadline { session_id }, + ); + + self.current_aggregation = Some(AggregationSession { + session_id, + early, + cancel, + worker, + }); +``` + +(`metrics::inc_aggregation_early_starts` / `observe_aggregation_early_start_lead` are added in Step 4 of this task.) + +Remove the emission from the interval-2 tick arm (~line 312), which becomes: + +```rust + // ==== interval 2 ==== + 2 => { + if is_aggregator { + self.start_aggregation_session(slot, ctx).await; + } else { + metrics::inc_aggregator_skipped_not_aggregator(); + } + } +``` + +(The `coverage::emit_agg_start_new_coverage` call and its old surrounding braces go away; the early-session skip guard lands in Task 7.) + +- [ ] **Step 3: Split publish out of `AggregateProduced` and add the flush handler** + +Add a helper method on `BlockChainServer` (near `on_gossip_attestation`, ~line 929): + +```rust + /// Publish an aggregated attestation to the aggregation gossip topic. + fn publish_aggregate(&self, aggregate: SignedAggregatedAttestation) { + if let Some(ref p2p) = self.p2p { + let _ = p2p + .publish_aggregated_attestation(aggregate) + .inspect_err(|err| error!(%err, "Failed to publish aggregated attestation")); + } + } +``` + +Rewrite the body of `impl Handler` (~line 1050) — the session-id fencing stays, then: + +```rust + aggregation::apply_aggregated_group(&mut self.store, &msg.output); + + let aggregate = SignedAggregatedAttestation { + data: msg.output.hashed.data().clone(), + proof: msg.output.proof, + }; + + // Publish alignment: hold back aggregates produced before this slot's + // interval-2 boundary; `FlushAggregatePublishes` publishes them at T2. + // (`session_id` is the session's slot.) + let genesis_time_ms = self.store.config().genesis_time * 1000; + let t2_ms = aggregation::interval2_boundary_ms(genesis_time_ms, msg.session_id); + if unix_now_ms() < t2_ms { + self.pending_aggregate_publishes.push(aggregate); + return; + } + self.publish_aggregate(aggregate); +``` + +Add the flush handler next to the other aggregation handlers (~line 1099): + +```rust +impl Handler for BlockChainServer { + async fn handle(&mut self, _msg: FlushAggregatePublishes, _ctx: &Context) { + let pending = std::mem::take(&mut self.pending_aggregate_publishes); + if pending.is_empty() { + return; + } + info!( + count = pending.len(), + "Publishing aggregates held back until the interval-2 boundary" + ); + for aggregate in pending { + self.publish_aggregate(aggregate); + } + } +} +``` + +Add the `early` field to the `AggregationDone` completion log (~line 1085): before the `info!`, compute + +```rust + let early = self + .current_aggregation + .as_ref() + .is_some_and(|s| s.session_id == msg.session_id && s.early); +``` + +and add `early,` to the `info!(...)` field list (after `cancelled = msg.cancelled,`). + +- [ ] **Step 4: Add the two metrics** + +In `crates/blockchain/src/metrics.rs` — counter in the `// --- Counters ---` section, histogram in `// --- Histograms ---`, public fns near their statics following file style: + +```rust +static LEAN_AGGREGATION_EARLY_STARTS_TOTAL: std::sync::LazyLock = + std::sync::LazyLock::new(|| { + register_int_counter!( + "lean_aggregation_early_starts_total", + "Aggregation sessions started before the interval-2 boundary" + ) + .unwrap() + }); + +pub fn inc_aggregation_early_starts() { + LEAN_AGGREGATION_EARLY_STARTS_TOTAL.inc(); +} +``` + +```rust +static LEAN_AGGREGATION_EARLY_START_LEAD_SECONDS: std::sync::LazyLock = + std::sync::LazyLock::new(|| { + register_histogram!( + "lean_aggregation_early_start_lead_seconds", + "How far before the interval-2 boundary an early aggregation session started", + vec![0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4] + ) + .unwrap() + }); + +pub fn observe_aggregation_early_start_lead(lead: Duration) { + LEAN_AGGREGATION_EARLY_START_LEAD_SECONDS.observe(lead.as_secs_f64()); +} +``` + +- [ ] **Step 5: Build and run existing tests** + +Run: `cargo build -p ethlambda-blockchain && cargo test -p ethlambda-blockchain --lib` +Expected: clean build, all unit tests pass. Behavior so far is identical for normal sessions (`early` is always false when started at interval 2, so no buffering, no flush timer, no metric increments). + +- [ ] **Step 6: Commit** + +```bash +git add crates/blockchain/src/aggregation.rs crates/blockchain/src/lib.rs crates/blockchain/src/metrics.rs +git commit -m "feat(blockchain): hold early-produced aggregates until the interval-2 boundary" +``` + +--- + +### Task 7: Early trigger — threshold checks, timer, and interval-2 skip + +**Files:** +- Modify: `crates/blockchain/src/aggregation.rs` (new message next to `FlushAggregatePublishes`) +- Modify: `crates/blockchain/src/lib.rs` (imports ~line 16, interval-1 arm ~line 290, interval-2 arm ~line 312, `NewAttestation` handler ~line 1034, new method + handler) + +- [ ] **Step 1: Add the check message in `aggregation.rs`** + +```rust +/// One-shot self-message scheduled at the interval-1 tick; fires when the +/// early-aggregation window opens (T2 - EARLY_AGGREGATION_WINDOW_MS) to run +/// the threshold check for signatures that all arrived before the window. +/// Arrivals inside the window are checked per insert instead. +pub(crate) struct EarlyAggregationCheck; +impl Message for EarlyAggregationCheck { + type Result = (); +} +``` + +- [ ] **Step 2: Add the trigger method on `BlockChainServer`** + +Extend the `use crate::aggregation::{...}` import with `EarlyAggregationCheck` and `EARLY_AGGREGATION_WINDOW_MS`. Add the method after `start_aggregation_session` (~line 418): + +```rust + /// Early-aggregation trigger: start the slot's session ahead of the + /// interval-2 tick when, inside the window `[T2 - EARLY_AGGREGATION_WINDOW_MS, T2)`, + /// a single attestation-data group already holds 2/3 of the signatures + /// expected from this node's aggregation subnets. Called after every + /// stored gossip signature and once at the window opening via + /// [`EarlyAggregationCheck`]. Fires at most once per slot: the started + /// session stays in `current_aggregation` (running or finished) until the + /// next session replaces it. + async fn maybe_start_early_aggregation(&mut self, ctx: &Context) { + if !self.aggregator.is_enabled() { + return; + } + let genesis_time_ms = self.store.config().genesis_time * 1000; + let Some(slot) = aggregation::early_aggregation_slot(unix_now_ms(), genesis_time_ms) + else { + return; + }; + if self + .current_aggregation + .as_ref() + .is_some_and(|session| session.session_id == slot) + { + return; + } + let max_group = self.store.max_gossip_group_count_for_slot(slot); + if !aggregation::early_threshold_met(max_group, self.early_aggregation_expected_sigs) { + return; + } + info!( + %slot, + max_group, + expected = self.early_aggregation_expected_sigs, + "Early-aggregation threshold met" + ); + self.start_aggregation_session(slot, ctx).await; + } +``` + +- [ ] **Step 3: Wire the three trigger sites** + +**(a) Interval-1 arm** (~line 290) — at the end of the `1 => { ... }` block, after the attestation-production `if/else`: + +```rust + // Schedule the early-aggregation window check. This tick is + // one interval before T2, so the timer fires right as the + // window opens at T2 - EARLY_AGGREGATION_WINDOW_MS. + if is_aggregator { + send_after( + Duration::from_millis( + MILLISECONDS_PER_INTERVAL - EARLY_AGGREGATION_WINDOW_MS, + ), + ctx.clone(), + EarlyAggregationCheck, + ); + } +``` + +**(b) Timer handler** — next to the other aggregation handlers: + +```rust +impl Handler for BlockChainServer { + async fn handle(&mut self, _msg: EarlyAggregationCheck, ctx: &Context) { + self.maybe_start_early_aggregation(ctx).await; + } +} +``` + +**(c) Per-insert check** — `impl Handler` (~line 1034) becomes: + +```rust +impl Handler for BlockChainServer { + async fn handle(&mut self, msg: NewAttestation, ctx: &Context) { + self.on_gossip_attestation(&msg.attestation); + self.maybe_start_early_aggregation(ctx).await; + } +} +``` + +(The method's internal guards make this a few comparisons outside the window; `max_gossip_group_count_for_slot` is one lock with no clones inside it.) + +- [ ] **Step 4: Skip the interval-2 start when the early session exists** + +The `2 => { ... }` arm becomes: + +```rust + // ==== interval 2 ==== + 2 => { + if is_aggregator { + // The early trigger may have already started this slot's + // session (running or finished) — it IS the slot's session, + // so don't start a second one. + let already_started = self + .current_aggregation + .as_ref() + .is_some_and(|session| session.session_id == slot); + if !already_started { + self.start_aggregation_session(slot, ctx).await; + } + } else { + metrics::inc_aggregator_skipped_not_aggregator(); + } + } +``` + +- [ ] **Step 5: Build and run all blockchain tests** + +Run: `cargo build --workspace && cargo test -p ethlambda-blockchain --lib` +Expected: clean build, all tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add crates/blockchain/src/aggregation.rs crates/blockchain/src/lib.rs +git commit -m "feat(blockchain): start aggregation early when 2/3 of subnet signatures arrived" +``` + +--- + +### Task 8: Full verification + +**Files:** none (verification only) + +- [ ] **Step 1: Format and lint** + +Run: `make fmt && make lint` +Expected: no diffs from fmt; clippy clean with `-D warnings` (this is where any dead-code stragglers from Task 4 would surface — fix by ensuring both new fields are read, which Tasks 6–7 guarantee). + +- [ ] **Step 2: Run the full test suite** + +Run: `make test` +Expected: all workspace tests + forkchoice spec tests pass. The spec tests exercise `on_block_without_verification` and the STF — untouched by this change. If fixtures are missing: `rm -rf leanSpec && make leanSpec/fixtures`. + +- [ ] **Step 3: Commit any fmt fallout** + +```bash +git add -u && git commit -m "chore: fmt" || true +``` + +(Skip the commit if the tree is clean.) + +- [ ] **Step 4: Devnet validation (manual, out of band)** + +Not part of the automated plan — run `.claude/skills/test-pr-devnet/scripts/test-branch.sh` on this branch and check: +- `lean_aggregation_early_starts_total` grows on aggregator nodes +- `lean_aggregation_early_start_lead_seconds` distribution sits in (0, 0.4] +- "Publishing aggregates held back until the interval-2 boundary" appears with count ≥ 1 +- Aggregate publish offsets (per-slot duty timing method) do not move earlier than interval 2 +- Finalization rate is not worse than the base image diff --git a/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md b/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md index 58dc8e90..0a2d51ca 100644 --- a/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md +++ b/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md @@ -43,8 +43,11 @@ Aggregator-only, fires at most once per slot. from at least 2/3 of the expected validators: `group_count * 3 >= expected * 2`, with `expected > 0`. `group_count` is the number of stored gossip signatures in that group (one per validator). - `expected` = number of head-state validators whose - `vid % attestation_committee_count` is in this node's aggregation subnet set. + `expected` = number of validators whose + `vid % attestation_committee_count` is in this node's aggregation subnet set, + computed once at actor spawn (the validator registry is static, and + `Store::head_state()` clones the full state so per-check reads would be + needlessly expensive). Note that at most one group per slot can reach 2/3 (each validator signs once per slot), so the per-group condition can hold for at most one group. - **Check sites:** @@ -110,12 +113,14 @@ pattern: build early, publish aligned to the boundary). ### Subnet-set plumbing The subscription-subnet computation is hoisted out of `build_swarm` into a -shared pure helper (in the p2p crate, e.g. -`compute_subscription_subnets(validator_ids, committee_count, is_aggregator, explicit_ids) -> HashSet`). -`main.rs` calls it once and passes the resulting set to both: - -- the p2p config (which subscribes to exactly this set, behavior unchanged), and -- the `BlockChainServer` (new field, used for the `expected` count). +shared pure helper in the p2p crate: +`compute_subscription_subnets(validator_ids, committee_count, is_aggregator, explicit_ids) -> HashSet`. +Both call sites use it with the same startup inputs, keeping a single source of +truth without churning `SwarmConfig`: + +- `build_swarm` (subscribes to exactly this set, behavior unchanged), and +- `main.rs`, which passes the resulting set to `BlockChain::spawn` for the + `expected` count. The set is frozen at startup, matching the existing hot-standby aggregator model (runtime toggles do not resubscribe subnets). From 2b782402760387307b2c13ffa15a38433af3098c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:47:09 -0300 Subject: [PATCH 03/33] docs: drop new-test steps from plan, add 4-node devnet validation task --- .../2026-07-01-early-aggregation-start.md | 248 +++++------------- 1 file changed, 67 insertions(+), 181 deletions(-) diff --git a/docs/superpowers/plans/2026-07-01-early-aggregation-start.md b/docs/superpowers/plans/2026-07-01-early-aggregation-start.md index 69d193f9..c5be585e 100644 --- a/docs/superpowers/plans/2026-07-01-early-aggregation-start.md +++ b/docs/superpowers/plans/2026-07-01-early-aggregation-start.md @@ -8,14 +8,16 @@ **Tech Stack:** Rust (edition 2024), spawned-concurrency actors (`send_after` self-messages), prometheus metrics via `ethlambda_metrics`. +**Verification policy (per user):** No new unit tests. Each task verifies by compiling (plus existing test suites where they already exist); final validation is a local 4-node devnet run (Task 9). + **Spec:** `docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md` **Key existing code:** - `crates/blockchain/src/lib.rs` — actor, tick loop (`on_tick`), `start_aggregation_session` (~line 366), handlers (`NewAttestation` ~line 1034, `AggregateProduced` ~line 1050, `AggregationDone` ~line 1079, `AggregationDeadline` ~line 1099) - `crates/blockchain/src/aggregation.rs` — session types, snapshot builders, worker, `AGGREGATION_DEADLINE` - `crates/blockchain/src/metrics.rs` — prometheus registration patterns -- `crates/storage/src/store.rs` — `GossipSignatureBuffer` (~line 348), `Store` methods (~line 1451), test mod (~line 1555) with `make_att_data(slot)`, `make_att_data_for_target(slot, root)`, `make_dummy_sig()` helpers -- `crates/net/p2p/src/lib.rs` — `build_swarm` subnet subscription (~lines 304–337), test mod (~line 790) +- `crates/storage/src/store.rs` — `GossipSignatureBuffer` (~line 348), `Store` methods (~line 1451) +- `crates/net/p2p/src/lib.rs` — `build_swarm` subnet subscription (~lines 304–337) - `bin/ethlambda/src/main.rs` — `BlockChain::spawn` (~line 213), `build_swarm(SwarmConfig {...})` (~line 230) **Timing model (4 s slots, 5 × 800 ms intervals):** @@ -36,48 +38,9 @@ slot start T1=+800ms T2=+1600ms T3=+2400ms T4=+3200ms The early threshold needs "largest signature count among current-slot data groups" without cloning signatures (the existing `iter_gossip_signatures` snapshot clones every signature; signatures are ~3 KB each). **Files:** -- Modify: `crates/storage/src/store.rs` (buffer method after `snapshot()` ~line 465; `Store` method next to `gossip_signatures_count()` ~line 1451; test in existing `mod tests`) - -- [ ] **Step 1: Write the failing test** - -Add to the `// ============ GossipSignatureBuffer Tests ============` section of `mod tests` in `crates/storage/src/store.rs` (near `gossip_buffer_fifo_eviction`, ~line 2423): - -```rust -#[test] -fn gossip_buffer_max_group_count_for_slot() { - let mut buf = GossipSignatureBuffer::new(100); - assert_eq!(buf.max_group_count_for_slot(1), 0); - - // Slot 1, group A: 3 validators. - for vid in 0..3 { - let data = make_att_data(1); - buf.insert(HashedAttestationData::new(data), vid, make_dummy_sig()); - } - // Slot 1, group B (same slot, different target root): 2 validators. - for vid in 3..5 { - let data = make_att_data_for_target(1, root(99)); - buf.insert(HashedAttestationData::new(data), vid, make_dummy_sig()); - } - // Slot 2: 5 validators. - for vid in 0..5 { - let data = make_att_data(2); - buf.insert(HashedAttestationData::new(data), vid, make_dummy_sig()); - } +- Modify: `crates/storage/src/store.rs` (buffer method after `snapshot()` ~line 465; `Store` method next to `gossip_signatures_count()` ~line 1451) - assert_eq!(buf.max_group_count_for_slot(1), 3); - assert_eq!(buf.max_group_count_for_slot(2), 5); - assert_eq!(buf.max_group_count_for_slot(3), 0); -} -``` - -Note: `make_att_data_for_target` is defined further down in the test mod (~line 2319); Rust test mods don't care about definition order. - -- [ ] **Step 2: Run test to verify it fails** - -Run: `cargo test -p ethlambda-storage max_group_count_for_slot` -Expected: FAIL to compile with "no method named `max_group_count_for_slot`" - -- [ ] **Step 3: Implement the buffer method and Store wrapper** +- [ ] **Step 1: Implement the buffer method and Store wrapper** In `impl GossipSignatureBuffer`, after `snapshot()` (~line 465): @@ -106,12 +69,12 @@ pub fn max_gossip_group_count_for_slot(&self, slot: u64) -> usize { } ``` -- [ ] **Step 4: Run test to verify it passes** +- [ ] **Step 2: Build** -Run: `cargo test -p ethlambda-storage max_group_count_for_slot` -Expected: PASS (1 test) +Run: `cargo build -p ethlambda-storage` +Expected: clean build (a dead-code warning on the buffer method is possible until Task 7 wires the caller; plain `cargo build` does not deny warnings). -- [ ] **Step 5: Commit** +- [ ] **Step 3: Commit** ```bash git add crates/storage/src/store.rs @@ -125,45 +88,9 @@ git commit -m "feat(storage): add max gossip group count query for early aggrega Single source of truth for the subnet set, callable from `main.rs` (Task 4) without touching `SwarmConfig`. **Files:** -- Modify: `crates/net/p2p/src/lib.rs` (subnet logic in `build_swarm` ~lines 304–337; tests in existing `mod tests` ~line 790) - -- [ ] **Step 1: Write the failing tests** +- Modify: `crates/net/p2p/src/lib.rs` (subnet logic in `build_swarm` ~lines 304–337) -Add to `mod tests` in `crates/net/p2p/src/lib.rs` (if the test mod doesn't already have `HashSet` in scope via `use super::*;`, add `use std::collections::HashSet;`): - -```rust -#[test] -fn subscription_subnets_validators_only() { - // 3 % 4 = 3, 7 % 4 = 3, 10 % 4 = 2 - let subnets = compute_subscription_subnets(&[3, 7, 10], 4, false, None); - assert_eq!(subnets, HashSet::from([3, 2])); -} - -#[test] -fn subscription_subnets_aggregator_unions_explicit_ids() { - let subnets = compute_subscription_subnets(&[1], 4, true, Some(&[0, 2])); - assert_eq!(subnets, HashSet::from([1, 0, 2])); -} - -#[test] -fn subscription_subnets_validatorless_aggregator_falls_back_to_zero() { - let subnets = compute_subscription_subnets(&[], 4, true, None); - assert_eq!(subnets, HashSet::from([0])); -} - -#[test] -fn subscription_subnets_validatorless_non_aggregator_is_empty() { - let subnets = compute_subscription_subnets(&[], 4, false, None); - assert!(subnets.is_empty()); -} -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cargo test -p ethlambda-p2p subscription_subnets` -Expected: FAIL to compile with "cannot find function `compute_subscription_subnets`" - -- [ ] **Step 3: Implement the helper and use it in `build_swarm`** +- [ ] **Step 1: Implement the helper and use it in `build_swarm`** Add near `build_swarm` (above it, ~line 185): @@ -223,15 +150,12 @@ Then replace the inline logic in `build_swarm` (~lines 304–329). The metric mu (The `let mut attestation_topics ...` loop over `&subscription_subnets` at ~line 331 stays unchanged.) -- [ ] **Step 4: Run tests to verify they pass** - -Run: `cargo test -p ethlambda-p2p subscription_subnets` -Expected: PASS (4 tests) +- [ ] **Step 2: Build and run existing p2p tests** -Run: `cargo build -p ethlambda-p2p` -Expected: clean build (no unused warnings; the old inline block is fully replaced) +Run: `cargo build -p ethlambda-p2p && cargo test -p ethlambda-p2p --lib` +Expected: clean build, existing tests pass (behavior of `build_swarm` is unchanged — same set, same subscriptions). -- [ ] **Step 5: Commit** +- [ ] **Step 3: Commit** ```bash git add crates/net/p2p/src/lib.rs @@ -242,76 +166,12 @@ git commit -m "refactor(p2p): extract subscription subnet computation into pure ### Task 3: Blockchain — pure early-aggregation helpers -Window math and threshold predicate, unit-testable without an actor. +Window math and threshold predicate. **Files:** -- Modify: `crates/blockchain/src/aggregation.rs` (new consts + functions after `PRIOR_WORKER_JOIN_TIMEOUT` ~line 37; new `#[cfg(test)] mod tests` at end of file) - -- [ ] **Step 1: Write the failing tests** +- Modify: `crates/blockchain/src/aggregation.rs` (new consts + functions after `PRIOR_WORKER_JOIN_TIMEOUT` ~line 37) -`crates/blockchain/src/aggregation.rs` has no test mod yet. Add at the end of the file: - -```rust -#[cfg(test)] -mod tests { - use super::*; - use crate::{MILLISECONDS_PER_INTERVAL, MILLISECONDS_PER_SLOT}; - - const GENESIS_MS: u64 = 1_000_000; - - /// Interval-2 boundary of `slot`, relative to the test genesis. - fn t2(slot: u64) -> u64 { - GENESIS_MS + slot * MILLISECONDS_PER_SLOT + 2 * MILLISECONDS_PER_INTERVAL - } - - #[test] - fn interval2_boundary_matches_slot_layout() { - assert_eq!(interval2_boundary_ms(GENESIS_MS, 0), GENESIS_MS + 1_600); - assert_eq!(interval2_boundary_ms(GENESIS_MS, 5), GENESIS_MS + 5 * 4_000 + 1_600); - } - - #[test] - fn early_window_bounds() { - // Window is [T2 - EARLY_AGGREGATION_WINDOW_MS, T2). - let before_window = t2(5) - EARLY_AGGREGATION_WINDOW_MS - 1; - assert_eq!(early_aggregation_slot(before_window, GENESIS_MS), None); - let window_open = t2(5) - EARLY_AGGREGATION_WINDOW_MS; - assert_eq!(early_aggregation_slot(window_open, GENESIS_MS), Some(5)); - assert_eq!(early_aggregation_slot(t2(5) - 1, GENESIS_MS), Some(5)); - assert_eq!(early_aggregation_slot(t2(5), GENESIS_MS), None); - // Slot 0 has a window too. - assert_eq!(early_aggregation_slot(t2(0) - 1, GENESIS_MS), Some(0)); - } - - #[test] - fn early_window_before_genesis_is_none() { - assert_eq!(early_aggregation_slot(GENESIS_MS - 1, GENESIS_MS), None); - } - - #[test] - fn threshold_is_two_thirds_of_expected() { - // Zero expected: never trigger (validator-less edge, div-by-zero guard). - assert!(!early_threshold_met(0, 0)); - assert!(!early_threshold_met(5, 0)); - // Exact 2/3 triggers. - assert!(early_threshold_met(2, 3)); - assert!(!early_threshold_met(1, 3)); - // Ceiling behavior when expected isn't divisible by 3. - assert!(early_threshold_met(3, 4)); // 9 >= 8 - assert!(!early_threshold_met(2, 4)); // 6 < 8 - // Devnet-scale sanity: 33 validators, one subnet. - assert!(early_threshold_met(22, 33)); - assert!(!early_threshold_met(21, 33)); - } -} -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cargo test -p ethlambda-blockchain --lib aggregation::tests` -Expected: FAIL to compile with "cannot find function `early_aggregation_slot`" (and the others) - -- [ ] **Step 3: Implement consts and functions** +- [ ] **Step 1: Implement consts and functions** In `crates/blockchain/src/aggregation.rs`, after `PRIOR_WORKER_JOIN_TIMEOUT` (~line 37). Also add `use crate::{MILLISECONDS_PER_INTERVAL, MILLISECONDS_PER_SLOT};` to the file's imports (both constants are `pub` in `lib.rs`). @@ -347,12 +207,12 @@ pub(crate) fn early_threshold_met(max_group_count: usize, expected: usize) -> bo } ``` -- [ ] **Step 4: Run tests to verify they pass** +- [ ] **Step 2: Build** -Run: `cargo test -p ethlambda-blockchain --lib aggregation::tests` -Expected: PASS (4 tests). Dead-code warnings for the not-yet-used functions are acceptable at this step only if the compiler emits them (`pub(crate)` items used from tests usually don't warn); they get consumed in Tasks 6–7. +Run: `cargo build -p ethlambda-blockchain` +Expected: clean build (dead-code warnings possible until Tasks 6–7 wire the callers). -- [ ] **Step 5: Commit** +- [ ] **Step 3: Commit** ```bash git add crates/blockchain/src/aggregation.rs @@ -467,9 +327,7 @@ Before `BlockChain::spawn` (~line 213), compute the set with the same startup in - [ ] **Step 3: Build the workspace** Run: `cargo build --workspace` -Expected: clean build. If `pending_aggregate_publishes` / `early_aggregation_expected_sigs` trigger dead-code warnings, silence is NOT needed — they are read in Tasks 6–7; if the build fails on `-D warnings` locally, proceed (only `make lint` enforces it) or add the fields in the task where they're first read. Do not annotate with `#[allow(dead_code)]`. - -Note: plain `cargo build` does not deny warnings, so this passes; the fields become live two tasks later, before `make lint` runs in Task 8. +Expected: clean build. Dead-code warnings on the two new fields are acceptable here (plain `cargo build` does not deny warnings); they become live in Tasks 6–7, before `make lint` runs in Task 8. Do not annotate with `#[allow(dead_code)]`. - [ ] **Step 4: Commit** @@ -484,6 +342,7 @@ git commit -m "feat(blockchain): plumb aggregation subnet set and expected-signa **Files:** - Modify: `crates/blockchain/src/aggregation.rs:28-33` +- Modify: `crates/blockchain/src/lib.rs` (stale `+750 ms` doc-comment reference on `start_aggregation_session`, ~line 365) - [ ] **Step 1: Change the constant and its comment** @@ -530,6 +389,7 @@ git commit -m "feat(blockchain): extend aggregation deadline to a full interval" **Files:** - Modify: `crates/blockchain/src/aggregation.rs` (`AggregationSession` struct ~line 75, new message) - Modify: `crates/blockchain/src/lib.rs` (imports ~line 16, `start_aggregation_session` ~line 366, `AggregateProduced` handler ~line 1050, `AggregationDone` handler ~line 1079, interval-2 tick arm ~line 312, new handler + helper) +- Modify: `crates/blockchain/src/metrics.rs` (new counter + histogram) - [ ] **Step 1: Add the `early` flag and the flush message in `aggregation.rs`** @@ -567,7 +427,7 @@ impl Message for FlushAggregatePublishes { - [ ] **Step 2: Rework `start_aggregation_session` in `lib.rs`** -Extend the `use crate::aggregation::{...}` import (~line 16) with `FlushAggregatePublishes` (and, for Task 7, `EarlyAggregationCheck` will join it). +Extend the `use crate::aggregation::{...}` import (~line 16) with `FlushAggregatePublishes`. In `start_aggregation_session` (~line 366): move the coverage emission here from the tick arm, and compute `early` + schedule the flush. After the prior-session join block and before the snapshot: @@ -747,7 +607,7 @@ pub fn observe_aggregation_early_start_lead(lead: Duration) { - [ ] **Step 5: Build and run existing tests** Run: `cargo build -p ethlambda-blockchain && cargo test -p ethlambda-blockchain --lib` -Expected: clean build, all unit tests pass. Behavior so far is identical for normal sessions (`early` is always false when started at interval 2, so no buffering, no flush timer, no metric increments). +Expected: clean build, existing unit tests pass. Behavior so far is identical for normal sessions (`early` is always false when started at interval 2, so no buffering, no flush timer, no metric increments). - [ ] **Step 6: Commit** @@ -886,10 +746,10 @@ The `2 => { ... }` arm becomes: } ``` -- [ ] **Step 5: Build and run all blockchain tests** +- [ ] **Step 5: Build and run existing blockchain tests** Run: `cargo build --workspace && cargo test -p ethlambda-blockchain --lib` -Expected: clean build, all tests pass. +Expected: clean build, existing tests pass. - [ ] **Step 6: Commit** @@ -900,19 +760,19 @@ git commit -m "feat(blockchain): start aggregation early when 2/3 of subnet sign --- -### Task 8: Full verification +### Task 8: Lint and existing test suites **Files:** none (verification only) - [ ] **Step 1: Format and lint** Run: `make fmt && make lint` -Expected: no diffs from fmt; clippy clean with `-D warnings` (this is where any dead-code stragglers from Task 4 would surface — fix by ensuring both new fields are read, which Tasks 6–7 guarantee). +Expected: no diffs from fmt; clippy clean with `-D warnings` (this is where any dead-code stragglers from Tasks 1–4 would surface — by now every added item has a caller). -- [ ] **Step 2: Run the full test suite** +- [ ] **Step 2: Run the existing test suites** Run: `make test` -Expected: all workspace tests + forkchoice spec tests pass. The spec tests exercise `on_block_without_verification` and the STF — untouched by this change. If fixtures are missing: `rm -rf leanSpec && make leanSpec/fixtures`. +Expected: all workspace tests + forkchoice spec tests pass (no new tests were added; this guards against regressions in the touched paths). If fixtures are missing: `rm -rf leanSpec && make leanSpec/fixtures`. - [ ] **Step 3: Commit any fmt fallout** @@ -922,11 +782,37 @@ git add -u && git commit -m "chore: fmt" || true (Skip the commit if the tree is clean.) -- [ ] **Step 4: Devnet validation (manual, out of band)** +--- + +### Task 9: Local 4-node devnet validation + +Run a local 4-node devnet on this branch and verify the early-start behavior end to end. Use the `devnet-runner` skill (`.claude/skills/devnet-runner/`) for the exact workflow; constraints that matter here: + +- **Release build only** — debug binaries stack-overflow in leanVM `rec_aggregation`. +- **Exactly one node with `--is-aggregator`** — multiple co-located aggregators stall finality via leanVM CPU contention. +- If using Docker Desktop, the VM needs 16+ GiB (leanVM peaks ~4–5 GiB per node); binary mode avoids this. + +- [ ] **Step 1: Start the devnet** + +4 nodes, fresh genesis, one aggregator. Let it run at least ~50 slots (~4 minutes). + +- [ ] **Step 2: Verify early-start behavior on the aggregator node** + +Check the aggregator node's logs and metrics: + +1. `"Early-aggregation threshold met"` and `"Starting aggregation session early"` appear for most slots (local gossip is fast; signatures land well inside the window). `lead_ms` values must be in `(0, 400]`. +2. `"Publishing aggregates held back until the interval-2 boundary"` appears with `count >= 1`. +3. `lean_aggregation_early_starts_total` grows (metrics port, default `:5054`, path `/metrics`). +4. `lean_aggregation_early_start_lead_seconds` observations sit in `(0, 0.4]`. +5. `"Committee signatures aggregated"` logs show `early=true` sessions and no `"Prior aggregation worker still running"` warnings. +6. No aggregate publish happens before its slot's interval-2 boundary: spot-check a few `"Starting aggregation session early"` slots and confirm the corresponding aggregated-attestation publish log lands at or after T2 (per-slot offset = `(log_epoch - GENESIS_TIME) mod 4 >= 1.6`). + +- [ ] **Step 3: Verify chain health** + +1. `lean_latest_finalized_slot` advances steadily on all 4 nodes (fresh local devnet should finalize nearly every justifiable slot). +2. Blocks carry attestations (`attestation_count > 0` in block logs). +3. No stalls, no panic/error-level logs attributable to aggregation. + +- [ ] **Step 4: Stop the devnet and report** -Not part of the automated plan — run `.claude/skills/test-pr-devnet/scripts/test-branch.sh` on this branch and check: -- `lean_aggregation_early_starts_total` grows on aggregator nodes -- `lean_aggregation_early_start_lead_seconds` distribution sits in (0, 0.4] -- "Publishing aggregates held back until the interval-2 boundary" appears with count ≥ 1 -- Aggregate publish offsets (per-slot duty timing method) do not move earlier than interval 2 -- Finalization rate is not worse than the base image +Stop the nodes; summarize observed early-start rate (early sessions / slots), lead-time distribution, and finalization progress. From ee03c4aa07dc9bdf4e40a04c0e6050eabd30f59e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:59:43 -0300 Subject: [PATCH 04/33] docs: apply plan-review fixes (stale publish drain, cached genesis_time_ms, spec corrections) --- .../2026-07-01-early-aggregation-start.md | 34 +++++++++++++------ ...26-07-01-early-aggregation-start-design.md | 30 ++++++++++------ 2 files changed, 44 insertions(+), 20 deletions(-) diff --git a/docs/superpowers/plans/2026-07-01-early-aggregation-start.md b/docs/superpowers/plans/2026-07-01-early-aggregation-start.md index c5be585e..f20dafd5 100644 --- a/docs/superpowers/plans/2026-07-01-early-aggregation-start.md +++ b/docs/superpowers/plans/2026-07-01-early-aggregation-start.md @@ -72,7 +72,7 @@ pub fn max_gossip_group_count_for_slot(&self, slot: u64) -> usize { - [ ] **Step 2: Build** Run: `cargo build -p ethlambda-storage` -Expected: clean build (a dead-code warning on the buffer method is possible until Task 7 wires the caller; plain `cargo build` does not deny warnings). +Expected: clean build, no warnings (the `pub` Store method keeps the private buffer method alive). - [ ] **Step 3: Commit** @@ -264,9 +264,10 @@ Inside `spawn`, before the `let handle = BlockChainServer {` block (~line 101), }; ``` -Add to the `BlockChainServer` struct literal in `spawn`: +Add to the `BlockChainServer` struct literal in `spawn` (`genesis_time` is already computed at ~line 90): ```rust + genesis_time_ms: genesis_time * 1000, early_aggregation_expected_sigs, pending_aggregate_publishes: Vec::new(), ``` @@ -274,6 +275,11 @@ Add to the `BlockChainServer` struct literal in `spawn`: Add the fields to the `BlockChainServer` struct definition (after `attestation_committee_count`, ~line 170): ```rust + /// Genesis time in milliseconds, cached at spawn. `store.config()` is an + /// uncached backend read, too heavy for the per-gossip-insert early + /// checks that need this. + genesis_time_ms: u64, + /// Number of validators whose subnet is one this node aggregates — the /// denominator of the early-aggregation 2/3 threshold. Computed once at /// spawn (the validator registry is static). @@ -285,7 +291,7 @@ Add the fields to the `BlockChainServer` struct definition (after `attestation_c pending_aggregate_publishes: Vec, ``` -(`HashSet` and `SignedAggregatedAttestation` are already imported in `lib.rs`.) +(`HashSet` and `SignedAggregatedAttestation` are already imported in `lib.rs`. Existing `config().genesis_time * 1000` call sites in `on_tick`/`handle_tick` stay as they are — only the new code paths use the cached field.) - [ ] **Step 2: Update `main.rs`** @@ -439,8 +445,18 @@ Replace the tail of the function (from `let session_id = slot;` through the fina ```rust let session_id = slot; - let genesis_time_ms = self.store.config().genesis_time * 1000; - let t2_ms = aggregation::interval2_boundary_ms(genesis_time_ms, slot); + // Any leftovers from a prior slot mean its flush never fired (a + // backwards wall-clock step, or a flush timer delayed past the next + // session). Drop them — late aggregates are dropped, same policy as + // signatures that miss the snapshot. + let stale = std::mem::take(&mut self.pending_aggregate_publishes); + if !stale.is_empty() { + warn!( + count = stale.len(), + "Dropping stale pending aggregate publishes" + ); + } + let t2_ms = aggregation::interval2_boundary_ms(self.genesis_time_ms, slot); let now_ms = unix_now_ms(); let early = now_ms < t2_ms; if early { @@ -529,8 +545,7 @@ Rewrite the body of `impl Handler` (~line 1050) — the sessi // Publish alignment: hold back aggregates produced before this slot's // interval-2 boundary; `FlushAggregatePublishes` publishes them at T2. // (`session_id` is the session's slot.) - let genesis_time_ms = self.store.config().genesis_time * 1000; - let t2_ms = aggregation::interval2_boundary_ms(genesis_time_ms, msg.session_id); + let t2_ms = aggregation::interval2_boundary_ms(self.genesis_time_ms, msg.session_id); if unix_now_ms() < t2_ms { self.pending_aggregate_publishes.push(aggregate); return; @@ -654,8 +669,7 @@ Extend the `use crate::aggregation::{...}` import with `EarlyAggregationCheck` a if !self.aggregator.is_enabled() { return; } - let genesis_time_ms = self.store.config().genesis_time * 1000; - let Some(slot) = aggregation::early_aggregation_slot(unix_now_ms(), genesis_time_ms) + let Some(slot) = aggregation::early_aggregation_slot(unix_now_ms(), self.genesis_time_ms) else { return; }; @@ -804,7 +818,7 @@ Check the aggregator node's logs and metrics: 2. `"Publishing aggregates held back until the interval-2 boundary"` appears with `count >= 1`. 3. `lean_aggregation_early_starts_total` grows (metrics port, default `:5054`, path `/metrics`). 4. `lean_aggregation_early_start_lead_seconds` observations sit in `(0, 0.4]`. -5. `"Committee signatures aggregated"` logs show `early=true` sessions and no `"Prior aggregation worker still running"` warnings. +5. `"Committee signatures aggregated"` logs show `early=true` sessions. `"Prior aggregation worker still running"` warnings must be rare (the early start shrinks the worst-case gap to the previous slot's worker from 3250 ms to 2800 ms, so occasional joins under slow proofs are expected — every slot would be a bug). 6. No aggregate publish happens before its slot's interval-2 boundary: spot-check a few `"Starting aggregation session early"` slots and confirm the corresponding aggregated-attestation publish log lands at or after T2 (per-slot offset = `(log_epoch - GENESIS_TIME) mod 4 >= 1.6`). - [ ] **Step 3: Verify chain health** diff --git a/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md b/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md index 0a2d51ca..082ef330 100644 --- a/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md +++ b/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md @@ -53,11 +53,13 @@ Aggregator-only, fires at most once per slot. - **Check sites:** 1. After each gossip-signature insert in the `NewAttestation` handler, while the wall clock is inside the window. - 2. A one-shot `EarlyAggregationCheck { slot }` self-message scheduled at the - interval-1 tick via `send_after(400 ms)` (interval-1 tick + 400 ms = - T2 - 400 ms), covering the case where the threshold was already met before - the window opened. If the interval-1 tick is skipped (overrun), the - per-insert checks still apply. + 2. A one-shot `EarlyAggregationCheck` self-message (no payload — the slot is + recomputed from the wall clock at fire time, so a late-fired timer can + never act on a stale slot) scheduled at the interval-1 tick via + `send_after(400 ms)` (interval-1 tick + 400 ms = T2 - 400 ms), covering + the case where the threshold was already met before the window opened. If + the interval-1 tick is skipped (overrun), the per-insert checks still + apply. - **Once-per-slot guard:** a session with `session_id == slot` already exists in `current_aggregation` (the field persists after the worker finishes; it is only replaced at the next session start). @@ -97,7 +99,11 @@ pattern: build early, publish aligned to the boundary). buffer is a no-op). - The buffer is bounded in practice by the number of data groups in one slot. Entries are never carried across slots: the flush timer fires at T2 of the - same slot that buffered them. + same slot that buffered them, and as a backstop `start_aggregation_session` + drains (drops, with a warning) any leftovers before installing a new session + — covering a backwards wall-clock step or a flush timer delayed past the + next session. Late aggregates are dropped, the same policy as signatures + that miss the snapshot. ### Deadline change @@ -127,10 +133,14 @@ model (runtime toggles do not resubscribe subnets). ## Failure modes and edge cases -- **Snapshot returns `None` after trigger** (should not happen — the trigger - requires signatures to be present): no session is created, - `current_aggregation` keeps the prior slot's id, and interval 2 retries - normally. No signatures are lost. +- **Snapshot returns `None` after trigger**: unreachable in practice — the + trigger requires stored signatures, and `on_gossip_attestation` only stores + signatures whose pubkeys resolved during verification, so the snapshot + always yields at least one job for them. If it did happen, no session is + created and `current_aggregation` is left `None` (the prior session was + already taken), so the once-per-slot guard would not hold and per-insert + checks would retry; interval 2 remains the fallback. No signatures are + lost. - **Threshold never met:** interval 2 starts the session exactly as today. - **Signatures arriving after the early snapshot:** never aggregated that slot. Accepted trade-off (explicitly chosen over a top-up session). From 8ae3b0728c4735ed6d80d46cca655ab3602f3076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 1 Jul 2026 18:02:13 -0300 Subject: [PATCH 05/33] feat(storage): add max gossip group count query for early aggregation --- crates/storage/src/store.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index c916bfca..8e344f9b 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -464,6 +464,16 @@ impl GossipSignatureBuffer { .collect() } + /// Largest signature count among data groups whose attestation slot is `slot`. + fn max_group_count_for_slot(&self, slot: u64) -> usize { + self.data + .values() + .filter(|entry| entry.data.slot == slot) + .map(|entry| entry.signatures.len()) + .max() + .unwrap_or(0) + } + /// Extract per-validator latest attestations from the raw signature pool. /// /// Mirrors `PayloadBuffer::extract_latest_attestations`: iterate data_roots @@ -1453,6 +1463,15 @@ impl Store { gossip.total_signatures() } + /// Largest per-group signature count among gossip groups voting for `slot`. + /// + /// One lock, no signature clones — cheap enough to call per gossip insert. + /// Drives the early-aggregation threshold check. + pub fn max_gossip_group_count_for_slot(&self, slot: u64) -> usize { + let gossip = self.gossip_signatures.lock().unwrap(); + gossip.max_group_count_for_slot(slot) + } + /// Estimated live data size in bytes for a table, as reported by the backend. pub fn estimate_table_bytes(&self, table: Table) -> u64 { self.backend.estimate_table_bytes(table) From 433ac14065e30d75acc581ca85871e4d23059f7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 1 Jul 2026 18:10:25 -0300 Subject: [PATCH 06/33] refactor(p2p): extract subscription subnet computation into pure helper --- crates/net/p2p/src/lib.rs | 63 ++++++++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 21 deletions(-) diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index de34e469..4308e130 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -182,6 +182,36 @@ pub struct BuiltSwarm { pub(crate) bootnode_addrs: HashMap, } +/// Compute the set of attestation subnets this node subscribes to (and, when +/// aggregating, aggregates over), per leanSpec (`src/lean_spec/__main__.py`): +/// every validator subscribes to its own subnet (`vid % committee_count`) for +/// mesh health; aggregators additionally subscribe to explicit +/// `aggregate_subnet_ids` and fall back to subnet 0 when the set would +/// otherwise be empty. +/// +/// Evaluated once at startup — runtime aggregator toggles do not resubscribe +/// (hot-standby model); see the invariant note on [`SwarmConfig`]. +pub fn compute_subscription_subnets( + validator_ids: &[u64], + attestation_committee_count: u64, + is_aggregator: bool, + aggregate_subnet_ids: Option<&[u64]>, +) -> HashSet { + let mut subnets: HashSet = validator_ids + .iter() + .map(|vid| vid % attestation_committee_count) + .collect(); + if is_aggregator { + if let Some(explicit_ids) = aggregate_subnet_ids { + subnets.extend(explicit_ids); + } + if subnets.is_empty() { + subnets.insert(0); + } + } + subnets +} + /// Build and configure the libp2p swarm, dial bootnodes, subscribe to topics. pub fn build_swarm( config: SwarmConfig, @@ -301,32 +331,23 @@ pub fn build_swarm( .subscribe(&aggregation_topic) .unwrap(); - // Subscribe to attestation subnets per leanSpec (`src/lean_spec/__main__.py`): - // every validator subscribes to its own subnet for mesh health; aggregators - // additionally subscribe to explicit `aggregate_subnet_ids` and fall back to - // subnet 0 when they have no validators of their own. - let validator_subnets: HashSet = config + // Subscribe to attestation subnets — see `compute_subscription_subnets`. + // The committee metric should reflect validator membership only, not + // aggregator-only subscriptions. + let metric_subnet = config .validator_ids .iter() .map(|vid| vid % config.attestation_committee_count) - .collect(); - - // The committee metric should reflect validator membership only, not - // aggregator-only subscriptions. - let metric_subnet = validator_subnets.iter().copied().min().unwrap_or(0); + .min() + .unwrap_or(0); metrics::set_attestation_committee_subnet(metric_subnet); - let mut subscription_subnets = validator_subnets; - if config.is_aggregator { - if let Some(ref explicit_ids) = config.aggregate_subnet_ids { - subscription_subnets.extend(explicit_ids); - } - // Fall back to subnet 0 only when the aggregator has no validators - // and no explicit subnets — otherwise leave the set as configured. - if subscription_subnets.is_empty() { - subscription_subnets.insert(0); - } - } + let subscription_subnets = compute_subscription_subnets( + &config.validator_ids, + config.attestation_committee_count, + config.is_aggregator, + config.aggregate_subnet_ids.as_deref(), + ); let mut attestation_topics: HashMap = HashMap::new(); for &subnet_id in &subscription_subnets { From 1791da9310c91ac24b5f1540afb6ed194396935c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 1 Jul 2026 18:33:22 -0300 Subject: [PATCH 07/33] feat(blockchain): add early-aggregation window and threshold helpers --- crates/blockchain/src/aggregation.rs | 32 +++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index f8387fcb..0eebd4e9 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -23,7 +23,7 @@ use spawned_concurrency::tasks::ActorRef; use tokio_util::sync::CancellationToken; use tracing::{info, warn}; -use crate::metrics; +use crate::{MILLISECONDS_PER_INTERVAL, MILLISECONDS_PER_SLOT, metrics}; /// Soft deadline for committee-signature aggregation measured from the /// interval-2 tick. After this much wall time elapses, the actor signals the @@ -36,6 +36,36 @@ pub(crate) const AGGREGATION_DEADLINE: Duration = Duration::from_millis(750); /// (mismatched timers, stuck proofs); we warn before blocking. pub(crate) const PRIOR_WORKER_JOIN_TIMEOUT: Duration = Duration::from_secs(2); +/// Width of the early-aggregation window: a session may start at most this +/// long before the interval-2 boundary, provided the signature threshold is +/// met (see `early_threshold_met`). +pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 400; + +/// Wall-clock millisecond timestamp of `slot`'s interval-2 boundary (the +/// normal aggregation start). +pub(crate) fn interval2_boundary_ms(genesis_time_ms: u64, slot: u64) -> u64 { + genesis_time_ms + slot * MILLISECONDS_PER_SLOT + 2 * MILLISECONDS_PER_INTERVAL +} + +/// If `now_ms` falls inside some slot's early-aggregation window +/// (`[T2 - EARLY_AGGREGATION_WINDOW_MS, T2)` with `T2` that slot's interval-2 +/// boundary), return that slot. +pub(crate) fn early_aggregation_slot(now_ms: u64, genesis_time_ms: u64) -> Option { + let since_genesis = now_ms.checked_sub(genesis_time_ms)?; + let ms_into_slot = since_genesis % MILLISECONDS_PER_SLOT; + let t2_offset = 2 * MILLISECONDS_PER_INTERVAL; + let in_window = + ms_into_slot >= t2_offset - EARLY_AGGREGATION_WINDOW_MS && ms_into_slot < t2_offset; + in_window.then_some(since_genesis / MILLISECONDS_PER_SLOT) +} + +/// Early-start threshold: a single attestation-data group holds at least 2/3 +/// of the signatures expected from this node's aggregation subnets. At most +/// one group per slot can satisfy this (each validator signs once per slot). +pub(crate) fn early_threshold_met(max_group_count: usize, expected: usize) -> bool { + expected > 0 && max_group_count * 3 >= expected * 2 +} + /// A single pre-prepared aggregation group. /// /// Built on the actor thread from a store snapshot; consumed by an off-thread From 07d7425521645e257e0fd3c080c87554c9b2a657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 1 Jul 2026 18:37:15 -0300 Subject: [PATCH 08/33] feat(blockchain): plumb aggregation subnet set and expected-signature count into actor --- bin/ethlambda/src/main.rs | 15 ++++++++++++++- crates/blockchain/src/lib.rs | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index 914c6c86..19db7eb5 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -37,7 +37,9 @@ use ethlambda_blockchain::MILLISECONDS_PER_SLOT; use ethlambda_blockchain::block_builder::ProposerConfig; use ethlambda_blockchain::key_manager::ValidatorKeyPair; use ethlambda_network_api::{InitBlockChain, InitP2P, ToBlockChainToP2PRef, ToP2PToBlockChainRef}; -use ethlambda_p2p::{Bootnode, P2P, PeerId, SwarmConfig, build_swarm, parse_enrs}; +use ethlambda_p2p::{ + Bootnode, P2P, PeerId, SwarmConfig, build_swarm, compute_subscription_subnets, parse_enrs, +}; use ethlambda_types::primitives::{H256, HashTreeRoot as _}; use ethlambda_types::{ aggregator::AggregatorController, @@ -210,11 +212,22 @@ async fn main() -> eyre::Result<()> { // and the API server (which exposes GET/POST admin endpoints). let aggregator = AggregatorController::new(options.is_aggregator); + // Same startup inputs build_swarm uses — single source of truth is the + // shared helper. Frozen at startup like the gossip subscriptions + // themselves (hot-standby model). + let aggregation_subnets = compute_subscription_subnets( + &validator_ids, + attestation_committee_count, + options.is_aggregator, + options.aggregate_subnet_ids.as_deref(), + ); + let blockchain = BlockChain::spawn( store.clone(), validator_keys, aggregator.clone(), attestation_committee_count, + aggregation_subnets, !options.disable_duty_sync_gate, ProposerConfig { enable_proposer_aggregation: options.enable_proposer_aggregation, diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 8be7c89f..9726a564 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -82,6 +82,7 @@ impl BlockChain { validator_keys: HashMap, aggregator: AggregatorController, attestation_committee_count: u64, + aggregation_subnets: HashSet, gate_duties: bool, proposer_config: ProposerConfig, ) -> BlockChain { @@ -98,6 +99,19 @@ impl BlockChain { (now_ms.saturating_sub(genesis_time * 1000) / MILLISECONDS_PER_SLOT) as u32; key_manager.advance_keys_to(current_slot); + // Denominator of the early-aggregation 2/3 threshold: how many + // validators attest on subnets this node aggregates. Computed once — + // the validator registry is static and `head_state()` clones the full + // state, so this must not run per gossip insert. + let validator_count = store.head_state().validators.len() as u64; + let early_aggregation_expected_sigs = if attestation_committee_count == 0 { + 0 + } else { + (0..validator_count) + .filter(|vid| aggregation_subnets.contains(&(vid % attestation_committee_count))) + .count() + }; + let handle = BlockChainServer { store, p2p: None, @@ -108,6 +122,9 @@ impl BlockChain { current_aggregation: None, last_tick_instant: None, attestation_committee_count, + genesis_time_ms: genesis_time * 1000, + early_aggregation_expected_sigs, + pending_aggregate_publishes: Vec::new(), proposer_config, pre_merge_coverage: None, sync_status: SyncStatusTracker::new(gate_duties), @@ -169,6 +186,21 @@ pub struct BlockChainServer { /// attestation aggregate coverage emission. attestation_committee_count: u64, + /// Genesis time in milliseconds, cached at spawn. `store.config()` is an + /// uncached backend read, too heavy for the per-gossip-insert early + /// checks that need this. + genesis_time_ms: u64, + + /// Number of validators whose subnet is one this node aggregates — the + /// denominator of the early-aggregation 2/3 threshold. Computed once at + /// spawn (the validator registry is static). + early_aggregation_expected_sigs: usize, + + /// Aggregates produced before the interval-2 boundary, held back so they + /// are not gossiped early. Flushed by `FlushAggregatePublishes` at the + /// boundary; only ever holds the current slot's aggregates. + pending_aggregate_publishes: Vec, + /// Proposer-side block-building policy proposer_config: ProposerConfig, From ff1af32749384b379035fe22b09bd3befe7aa6a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 10:05:34 -0300 Subject: [PATCH 09/33] docs(blockchain): note genesis_time_ms cache covers tick-path recomputations --- crates/blockchain/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 9726a564..ea5787f3 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -188,7 +188,9 @@ pub struct BlockChainServer { /// Genesis time in milliseconds, cached at spawn. `store.config()` is an /// uncached backend read, too heavy for the per-gossip-insert early - /// checks that need this. + /// checks that need this. The tick paths still recompute it from + /// `store.config()` — deliberately left alone; prefer this field in new + /// code. genesis_time_ms: u64, /// Number of validators whose subnet is one this node aggregates — the From 1466af03d0ede84b87a993c3d25c0facd33b2083 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 10:06:57 -0300 Subject: [PATCH 10/33] feat(blockchain): extend aggregation deadline to a full interval --- crates/blockchain/src/aggregation.rs | 14 ++++++++------ crates/blockchain/src/lib.rs | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index 0eebd4e9..caf1b51f 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -25,12 +25,14 @@ use tracing::{info, warn}; use crate::{MILLISECONDS_PER_INTERVAL, MILLISECONDS_PER_SLOT, metrics}; -/// Soft deadline for committee-signature aggregation measured from the -/// interval-2 tick. After this much wall time elapses, the actor signals the -/// worker to stop via its cancellation token. The 50 ms budget before the next -/// interval (interval 3 at +800 ms) is reserved for publishing any late-arriving -/// aggregates and for gossip propagation margin. -pub(crate) const AGGREGATION_DEADLINE: Duration = Duration::from_millis(750); +/// Soft deadline for committee-signature aggregation measured from session +/// start. After this much wall time elapses, the actor signals the worker to +/// stop via its cancellation token. A session started exactly at interval 2 +/// gets the full interval (interval 3 is one interval later); a session +/// started early (see `early_aggregation_slot`) ends correspondingly earlier. +/// The deadline only stops new jobs from starting — a job mid-proof finishes +/// and publishes right after. +pub(crate) const AGGREGATION_DEADLINE: Duration = Duration::from_millis(800); /// Upper bound we wait for a prior worker to exit if it is still running when /// the next session is about to start. Reached only in pathological cases /// (mismatched timers, stuck proofs); we warn before blocking. diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index ea5787f3..c01b6700 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -396,7 +396,7 @@ impl BlockChainServer { /// 1. If a prior session is still running (pathological), warn and join it. /// 2. Snapshot the aggregation inputs from the store. /// 3. Spawn a `spawn_blocking` worker that streams results back as messages. - /// 4. Schedule the `AggregationDeadline` self-message at +750 ms. + /// 4. Schedule the `AggregationDeadline` self-message at +`AGGREGATION_DEADLINE`. async fn start_aggregation_session(&mut self, slot: u64, ctx: &Context) { if let Some(prior) = self.current_aggregation.take() { prior.cancel.cancel(); From 5e7fed514bb591728c8160156f91caeaabd8ca6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 10:10:28 -0300 Subject: [PATCH 11/33] feat(blockchain): hold early-produced aggregates until the interval-2 boundary --- crates/blockchain/src/aggregation.rs | 12 ++++ crates/blockchain/src/lib.rs | 89 ++++++++++++++++++++++++---- crates/blockchain/src/metrics.rs | 29 +++++++++ 3 files changed, 117 insertions(+), 13 deletions(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index caf1b51f..2efb57cc 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -108,6 +108,9 @@ pub(crate) struct AggregationSession { /// Slot at which this session was started; used as a fencing id so we can /// drop late-arriving messages from a prior session. pub(crate) session_id: u64, + /// Whether the session started before the slot's interval-2 boundary via + /// the early-aggregation trigger. + pub(crate) early: bool, /// Child of the actor cancellation token; fires either at the deadline or /// when the actor itself is stopping. pub(crate) cancel: CancellationToken, @@ -148,6 +151,15 @@ impl Message for AggregationDeadline { type Result = (); } +/// Self-message scheduled when a session starts early; fires at the +/// interval-2 boundary and publishes any aggregates held back by the +/// publish-alignment rule (aggregates must not reach gossip before +/// interval 2). +pub(crate) struct FlushAggregatePublishes; +impl Message for FlushAggregatePublishes { + type Result = (); +} + /// Build a snapshot of everything needed to aggregate. Runs on the actor /// thread, touches the store, does no heavy cryptography. Returns `None` when /// there is nothing to aggregate so callers can avoid spawning an empty worker. diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index c01b6700..698a213b 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -15,7 +15,7 @@ use ethlambda_types::{ use crate::aggregation::{ AGGREGATION_DEADLINE, AggregateProduced, AggregationDeadline, AggregationDone, - AggregationSession, PRIOR_WORKER_JOIN_TIMEOUT, run_aggregation_worker, + AggregationSession, FlushAggregatePublishes, PRIOR_WORKER_JOIN_TIMEOUT, run_aggregation_worker, }; use crate::key_manager::ValidatorKeyPair; use crate::sync_status::SyncStatusTracker; @@ -345,10 +345,6 @@ impl BlockChainServer { // ==== interval 2 ==== 2 => { if is_aggregator { - coverage::emit_agg_start_new_coverage( - &self.store, - self.attestation_committee_count, - ); self.start_aggregation_session(slot, ctx).await; } else { metrics::inc_aggregator_skipped_not_aggregator(); @@ -417,6 +413,8 @@ impl BlockChainServer { } } + coverage::emit_agg_start_new_coverage(&self.store, self.attestation_committee_count); + let Some(snapshot) = aggregation::snapshot_current_slot_aggregation_inputs(&self.store, slot) else { @@ -425,6 +423,35 @@ impl BlockChainServer { }; let session_id = slot; + // Any leftovers from a prior slot mean its flush never fired (a + // backwards wall-clock step, or a flush timer delayed past the next + // session). Drop them — late aggregates are dropped, same policy as + // signatures that miss the snapshot. + let stale = std::mem::take(&mut self.pending_aggregate_publishes); + if !stale.is_empty() { + warn!( + count = stale.len(), + "Dropping stale pending aggregate publishes" + ); + } + let t2_ms = aggregation::interval2_boundary_ms(self.genesis_time_ms, slot); + let now_ms = unix_now_ms(); + let early = now_ms < t2_ms; + if early { + // Publish alignment: aggregates must not reach gossip before the + // interval-2 boundary. Aggregates produced before T2 are buffered + // in `pending_aggregate_publishes`; this timer flushes them at T2. + let lead = Duration::from_millis(t2_ms - now_ms); + metrics::inc_aggregation_early_starts(); + metrics::observe_aggregation_early_start_lead(lead); + info!( + %slot, + lead_ms = lead.as_millis() as u64, + "Starting aggregation session early" + ); + send_after(lead, ctx.clone(), FlushAggregatePublishes); + } + // Independent token per session. Shutdown propagates via our // #[stopped] hook which cancels any current session; the deadline // timer cancels this specific session at +AGGREGATION_DEADLINE. @@ -445,6 +472,7 @@ impl BlockChainServer { self.current_aggregation = Some(AggregationSession { session_id, + early, cancel, worker, }); @@ -960,6 +988,15 @@ impl BlockChainServer { } } + /// Publish an aggregated attestation to the aggregation gossip topic. + fn publish_aggregate(&self, aggregate: SignedAggregatedAttestation) { + if let Some(ref p2p) = self.p2p { + let _ = p2p + .publish_aggregated_attestation(aggregate) + .inspect_err(|err| error!(%err, "Failed to publish aggregated attestation")); + } + } + fn on_gossip_attestation(&mut self, attestation: &SignedAttestation) { // Read fresh here too: a gossip event can arrive between ticks, and // if the admin API just toggled, the first gossip after the toggle @@ -1098,14 +1135,35 @@ impl Handler for BlockChainServer { aggregation::apply_aggregated_group(&mut self.store, &msg.output); - if let Some(ref p2p) = self.p2p { - let aggregate = SignedAggregatedAttestation { - data: msg.output.hashed.data().clone(), - proof: msg.output.proof, - }; - let _ = p2p - .publish_aggregated_attestation(aggregate) - .inspect_err(|err| error!(%err, "Failed to publish aggregated attestation")); + let aggregate = SignedAggregatedAttestation { + data: msg.output.hashed.data().clone(), + proof: msg.output.proof, + }; + + // Publish alignment: hold back aggregates produced before this slot's + // interval-2 boundary; `FlushAggregatePublishes` publishes them at T2. + // (`session_id` is the session's slot.) + let t2_ms = aggregation::interval2_boundary_ms(self.genesis_time_ms, msg.session_id); + if unix_now_ms() < t2_ms { + self.pending_aggregate_publishes.push(aggregate); + return; + } + self.publish_aggregate(aggregate); + } +} + +impl Handler for BlockChainServer { + async fn handle(&mut self, _msg: FlushAggregatePublishes, _ctx: &Context) { + let pending = std::mem::take(&mut self.pending_aggregate_publishes); + if pending.is_empty() { + return; + } + info!( + count = pending.len(), + "Publishing aggregates held back until the interval-2 boundary" + ); + for aggregate in pending { + self.publish_aggregate(aggregate); } } } @@ -1116,6 +1174,10 @@ impl Handler for BlockChainServer { metrics::observe_committee_signatures_aggregation(msg.total_elapsed); let aggregation_elapsed = msg.total_elapsed; + let early = self + .current_aggregation + .as_ref() + .is_some_and(|s| s.session_id == msg.session_id && s.early); info!( ?aggregation_elapsed, session_id = msg.session_id, @@ -1124,6 +1186,7 @@ impl Handler for BlockChainServer { total_raw_sigs = msg.total_raw_sigs, total_children = msg.total_children, cancelled = msg.cancelled, + early, aggregation_deadline_ms = AGGREGATION_DEADLINE.as_millis() as u64, "Committee signatures aggregated" ); diff --git a/crates/blockchain/src/metrics.rs b/crates/blockchain/src/metrics.rs index 28e510f5..d178e22c 100644 --- a/crates/blockchain/src/metrics.rs +++ b/crates/blockchain/src/metrics.rs @@ -263,6 +263,19 @@ static LEAN_PQ_SIG_ATTESTATION_SIGNATURES_INVALID_TOTAL: std::sync::LazyLock = + std::sync::LazyLock::new(|| { + register_int_counter!( + "lean_aggregation_early_starts_total", + "Aggregation sessions started before the interval-2 boundary" + ) + .unwrap() + }); + +pub fn inc_aggregation_early_starts() { + LEAN_AGGREGATION_EARLY_STARTS_TOTAL.inc(); +} + // --- Histograms --- static LEAN_FORK_CHOICE_BLOCK_PROCESSING_TIME_SECONDS: std::sync::LazyLock = @@ -374,6 +387,20 @@ static LEAN_FORK_CHOICE_REORG_DEPTH: std::sync::LazyLock = .unwrap() }); +static LEAN_AGGREGATION_EARLY_START_LEAD_SECONDS: std::sync::LazyLock = + std::sync::LazyLock::new(|| { + register_histogram!( + "lean_aggregation_early_start_lead_seconds", + "How far before the interval-2 boundary an early aggregation session started", + vec![0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4] + ) + .unwrap() + }); + +pub fn observe_aggregation_early_start_lead(lead: Duration) { + LEAN_AGGREGATION_EARLY_START_LEAD_SECONDS.observe(lead.as_secs_f64()); +} + static LEAN_TICK_INTERVAL_DURATION_SECONDS: std::sync::LazyLock = std::sync::LazyLock::new(|| { register_histogram!( @@ -590,6 +617,7 @@ pub fn init() { std::sync::LazyLock::force(&LEAN_PQ_SIG_ATTESTATION_SIGNATURES_TOTAL); std::sync::LazyLock::force(&LEAN_PQ_SIG_ATTESTATION_SIGNATURES_VALID_TOTAL); std::sync::LazyLock::force(&LEAN_PQ_SIG_ATTESTATION_SIGNATURES_INVALID_TOTAL); + std::sync::LazyLock::force(&LEAN_AGGREGATION_EARLY_STARTS_TOTAL); // Histograms std::sync::LazyLock::force(&LEAN_FORK_CHOICE_BLOCK_PROCESSING_TIME_SECONDS); std::sync::LazyLock::force(&LEAN_ATTESTATION_VALIDATION_TIME_SECONDS); @@ -601,6 +629,7 @@ pub fn init() { std::sync::LazyLock::force(&LEAN_COMMITTEE_SIGNATURES_AGGREGATION_TIME_SECONDS); std::sync::LazyLock::force(&LEAN_AGGREGATED_PROOF_SIZE_BYTES); std::sync::LazyLock::force(&LEAN_FORK_CHOICE_REORG_DEPTH); + std::sync::LazyLock::force(&LEAN_AGGREGATION_EARLY_START_LEAD_SECONDS); std::sync::LazyLock::force(&LEAN_TICK_INTERVAL_DURATION_SECONDS); // Block production std::sync::LazyLock::force(&LEAN_BLOCK_AGGREGATED_PAYLOADS); From 88c40bcef468942f61e0b875ef49ba68a59ed810 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 10:21:19 -0300 Subject: [PATCH 12/33] polish(blockchain): move early-agg metric helpers to Public API section, add log identity fields --- crates/blockchain/src/lib.rs | 3 +++ crates/blockchain/src/metrics.rs | 16 ++++++++-------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 698a213b..f5714a44 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -430,6 +430,7 @@ impl BlockChainServer { let stale = std::mem::take(&mut self.pending_aggregate_publishes); if !stale.is_empty() { warn!( + %slot, count = stale.len(), "Dropping stale pending aggregate publishes" ); @@ -1158,7 +1159,9 @@ impl Handler for BlockChainServer { if pending.is_empty() { return; } + let session_id = self.current_aggregation.as_ref().map(|s| s.session_id); info!( + session_id, count = pending.len(), "Publishing aggregates held back until the interval-2 boundary" ); diff --git a/crates/blockchain/src/metrics.rs b/crates/blockchain/src/metrics.rs index d178e22c..11b63f2c 100644 --- a/crates/blockchain/src/metrics.rs +++ b/crates/blockchain/src/metrics.rs @@ -272,10 +272,6 @@ static LEAN_AGGREGATION_EARLY_STARTS_TOTAL: std::sync::LazyLock = .unwrap() }); -pub fn inc_aggregation_early_starts() { - LEAN_AGGREGATION_EARLY_STARTS_TOTAL.inc(); -} - // --- Histograms --- static LEAN_FORK_CHOICE_BLOCK_PROCESSING_TIME_SECONDS: std::sync::LazyLock = @@ -397,10 +393,6 @@ static LEAN_AGGREGATION_EARLY_START_LEAD_SECONDS: std::sync::LazyLock .unwrap() }); -pub fn observe_aggregation_early_start_lead(lead: Duration) { - LEAN_AGGREGATION_EARLY_START_LEAD_SECONDS.observe(lead.as_secs_f64()); -} - static LEAN_TICK_INTERVAL_DURATION_SECONDS: std::sync::LazyLock = std::sync::LazyLock::new(|| { register_histogram!( @@ -656,6 +648,14 @@ pub fn init() { // --- Public API --- +pub fn inc_aggregation_early_starts() { + LEAN_AGGREGATION_EARLY_STARTS_TOTAL.inc(); +} + +pub fn observe_aggregation_early_start_lead(lead: Duration) { + LEAN_AGGREGATION_EARLY_START_LEAD_SECONDS.observe(lead.as_secs_f64()); +} + pub fn update_head_slot(slot: u64) { LEAN_HEAD_SLOT.set(slot.try_into().unwrap()); } From 76ed220affd96b636925896c75469af7751aab7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 10:24:13 -0300 Subject: [PATCH 13/33] feat(blockchain): start aggregation early when 2/3 of subnet signatures arrived --- crates/blockchain/src/aggregation.rs | 9 ++++ crates/blockchain/src/lib.rs | 72 ++++++++++++++++++++++++++-- 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index 2efb57cc..46a11b03 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -160,6 +160,15 @@ impl Message for FlushAggregatePublishes { type Result = (); } +/// One-shot self-message scheduled at the interval-1 tick; fires when the +/// early-aggregation window opens (T2 - EARLY_AGGREGATION_WINDOW_MS) to run +/// the threshold check for signatures that all arrived before the window. +/// Arrivals inside the window are checked per insert instead. +pub(crate) struct EarlyAggregationCheck; +impl Message for EarlyAggregationCheck { + type Result = (); +} + /// Build a snapshot of everything needed to aggregate. Runs on the actor /// thread, touches the store, does no heavy cryptography. Returns `None` when /// there is nothing to aggregate so callers can avoid spawning an empty worker. diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index f5714a44..90e22ee5 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -15,7 +15,8 @@ use ethlambda_types::{ use crate::aggregation::{ AGGREGATION_DEADLINE, AggregateProduced, AggregationDeadline, AggregationDone, - AggregationSession, FlushAggregatePublishes, PRIOR_WORKER_JOIN_TIMEOUT, run_aggregation_worker, + AggregationSession, EARLY_AGGREGATION_WINDOW_MS, EarlyAggregationCheck, + FlushAggregatePublishes, PRIOR_WORKER_JOIN_TIMEOUT, run_aggregation_worker, }; use crate::key_manager::ValidatorKeyPair; use crate::sync_status::SyncStatusTracker; @@ -340,12 +341,34 @@ impl BlockChainServer { } else if !self.key_manager.validator_ids().is_empty() { info!(%slot, "Skipping attestations while syncing"); } + + // Schedule the early-aggregation window check. This tick is + // one interval before T2, so the timer fires right as the + // window opens at T2 - EARLY_AGGREGATION_WINDOW_MS. + if is_aggregator { + send_after( + Duration::from_millis( + MILLISECONDS_PER_INTERVAL - EARLY_AGGREGATION_WINDOW_MS, + ), + ctx.clone(), + EarlyAggregationCheck, + ); + } } // ==== interval 2 ==== 2 => { if is_aggregator { - self.start_aggregation_session(slot, ctx).await; + // The early trigger may have already started this slot's + // session (running or finished) — it IS the slot's session, + // so don't start a second one. + let already_started = self + .current_aggregation + .as_ref() + .is_some_and(|session| session.session_id == slot); + if !already_started { + self.start_aggregation_session(slot, ctx).await; + } } else { metrics::inc_aggregator_skipped_not_aggregator(); } @@ -479,6 +502,42 @@ impl BlockChainServer { }); } + /// Early-aggregation trigger: start the slot's session ahead of the + /// interval-2 tick when, inside the window `[T2 - EARLY_AGGREGATION_WINDOW_MS, T2)`, + /// a single attestation-data group already holds 2/3 of the signatures + /// expected from this node's aggregation subnets. Called after every + /// stored gossip signature and once at the window opening via + /// [`EarlyAggregationCheck`]. Fires at most once per slot: the started + /// session stays in `current_aggregation` (running or finished) until the + /// next session replaces it. + async fn maybe_start_early_aggregation(&mut self, ctx: &Context) { + if !self.aggregator.is_enabled() { + return; + } + let Some(slot) = aggregation::early_aggregation_slot(unix_now_ms(), self.genesis_time_ms) + else { + return; + }; + if self + .current_aggregation + .as_ref() + .is_some_and(|session| session.session_id == slot) + { + return; + } + let max_group = self.store.max_gossip_group_count_for_slot(slot); + if !aggregation::early_threshold_met(max_group, self.early_aggregation_expected_sigs) { + return; + } + info!( + %slot, + max_group, + expected = self.early_aggregation_expected_sigs, + "Early-aggregation threshold met" + ); + self.start_aggregation_session(slot, ctx).await; + } + /// Returns the validator ID if any of our validators is the proposer for this slot. fn get_our_proposer(&self, slot: u64) -> Option { let head_state = self.store.head_state(); @@ -1104,8 +1163,9 @@ impl Handler for BlockChainServer { } impl Handler for BlockChainServer { - async fn handle(&mut self, msg: NewAttestation, _ctx: &Context) { + async fn handle(&mut self, msg: NewAttestation, ctx: &Context) { self.on_gossip_attestation(&msg.attestation); + self.maybe_start_early_aggregation(ctx).await; } } @@ -1171,6 +1231,12 @@ impl Handler for BlockChainServer { } } +impl Handler for BlockChainServer { + async fn handle(&mut self, _msg: EarlyAggregationCheck, ctx: &Context) { + self.maybe_start_early_aggregation(ctx).await; + } +} + impl Handler for BlockChainServer { async fn handle(&mut self, msg: AggregationDone, _ctx: &Context) { aggregation::finalize_aggregation_session(&self.store); From d57248f03bc4dc53ccaf9e6293cb3eebf24626a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 10:31:58 -0300 Subject: [PATCH 14/33] docs(blockchain): note the once-per-slot latch hole for unresolvable pubkeys --- crates/blockchain/src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 90e22ee5..470c7ff4 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -509,7 +509,10 @@ impl BlockChainServer { /// stored gossip signature and once at the window opening via /// [`EarlyAggregationCheck`]. Fires at most once per slot: the started /// session stays in `current_aggregation` (running or finished) until the - /// next session replaces it. + /// next session replaces it. The latch has one hole: if the snapshot + /// yields no jobs (possible only when no signer's pubkey resolves, i.e. a + /// corrupted validator registry), no session is installed and the check + /// retries on later inserts — each retry is a no-op session attempt. async fn maybe_start_early_aggregation(&mut self, ctx: &Context) { if !self.aggregator.is_enabled() { return; From 235d97fa19f3a84ee3a60fdf1e2fb81a9e4f6aaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 10:50:43 -0300 Subject: [PATCH 15/33] docs: fix stale session-lifecycle comments flagged by final review --- crates/blockchain/src/aggregation.rs | 6 ++++-- crates/blockchain/src/lib.rs | 8 +++++--- ...2026-07-01-early-aggregation-start-design.md | 17 ++++++++--------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index 46a11b03..d5384290 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -1,7 +1,9 @@ //! Committee-signature aggregation: off-thread worker orchestration and the //! pure functions it runs. //! -//! The blockchain actor fires one aggregation session per interval 2 via +//! The blockchain actor fires one aggregation session per slot — at interval 2, +//! or up to [`EARLY_AGGREGATION_WINDOW_MS`] early when the 2/3 signature +//! threshold is met — via //! [`run_aggregation_worker`]. The actor stays on its message loop; the worker //! runs the expensive XMSS proofs on a `spawn_blocking` thread and streams //! results back as [`AggregateProduced`] / [`AggregationDone`] messages. @@ -142,7 +144,7 @@ impl Message for AggregationDone { type Result = (); } -/// Self-message scheduled via `send_after` at interval-2 start. Cancels the +/// Self-message scheduled via `send_after` at session start. Cancels the /// session's token so the worker stops starting new aggregations. pub(crate) struct AggregationDeadline { pub(crate) session_id: u64, diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 470c7ff4..edda6c2c 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -175,9 +175,11 @@ pub struct BlockChainServer { /// `--is-aggregator` flag at spawn. aggregator: AggregatorController, - /// In-flight committee-signature aggregation, if any. Present only while a - /// worker started at the most recent interval 2 is still running or until - /// the next interval 2 takes over. + /// The slot's one committee-signature aggregation session (started at + /// interval 2, or early via the 2/3 trigger). Deliberately persists after + /// the worker finishes — that persistence is the once-per-slot latch the + /// early trigger and the interval-2 skip both check — until the next + /// session start replaces it. current_aggregation: Option, /// Last tick instant for measuring interval duration. diff --git a/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md b/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md index 082ef330..e9ca5ddb 100644 --- a/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md +++ b/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md @@ -162,15 +162,14 @@ model (runtime toggles do not resubscribe subnets). ## Testing -- Unit tests for the pure trigger decision (window bounds, threshold math, - `expected` derivation from subnet set + validator count, once-per-slot guard). -- Unit tests for `compute_subscription_subnets` (validator subnets, explicit - ids, fallback-to-0, non-aggregator). -- Unit test for the publish-delay decision (before/after T2). -- Existing aggregation tests must pass unchanged (normal interval-2 path is the - fallback and keeps its semantics; only the deadline constant changes). -- Devnet validation via `test-branch.sh`: compare aggregate publish offsets - (per-slot duty timing method) and `lean_aggregation_early_starts_total`. +Per user decision, no new unit tests. Validation is: + +- Existing suites pass unchanged (normal interval-2 path is the fallback and + keeps its semantics; only the deadline constant changes). +- Local 4-node devnet run (1 aggregator, binary mode): early-start rate and + lead-time distribution from logs, held-back publish flushes landing at or + after the interval-2 boundary (per-slot offset), lockstep finalization, no + aggregation warnings/errors. ## Non-goals From d9fe68a0bebbab18418b808cb82e86418bd542b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 12:12:33 -0300 Subject: [PATCH 16/33] fix(blockchain): fence FlushAggregatePublishes by slot; guard subnet div-by-zero and window underflow Addresses PR #487 review feedback: - FlushAggregatePublishes now carries its session slot and drains only when it matches current_aggregation, matching AggregationDeadline's fencing. A flush timer delayed past the next session start can no longer publish the newer session's buffered aggregates before its interval-2 boundary. - compute_subscription_subnets returns an empty set for a zero committee count instead of dividing by zero (defensive; callers enforce >= 1). - const assert that EARLY_AGGREGATION_WINDOW_MS fits within one interval, making the no-underflow invariant self-enforcing. --- crates/blockchain/src/aggregation.rs | 18 ++++++++++++++++-- crates/blockchain/src/lib.rs | 20 ++++++++++++++++---- crates/net/p2p/src/lib.rs | 16 ++++++++++++---- 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index d5384290..7f77e43a 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -45,6 +45,16 @@ pub(crate) const PRIOR_WORKER_JOIN_TIMEOUT: Duration = Duration::from_secs(2); /// met (see `early_threshold_met`). pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 400; +// The window must fit within one interval: `early_aggregation_slot` subtracts +// it from the interval-2 offset, and the interval-1 tick schedules the check +// at `MILLISECONDS_PER_INTERVAL - EARLY_AGGREGATION_WINDOW_MS`. Keep this +// invariant self-enforcing so a future bump to the window can't silently +// underflow either subtraction. +const _: () = assert!( + EARLY_AGGREGATION_WINDOW_MS <= MILLISECONDS_PER_INTERVAL, + "EARLY_AGGREGATION_WINDOW_MS must not exceed one interval" +); + /// Wall-clock millisecond timestamp of `slot`'s interval-2 boundary (the /// normal aggregation start). pub(crate) fn interval2_boundary_ms(genesis_time_ms: u64, slot: u64) -> u64 { @@ -156,8 +166,12 @@ impl Message for AggregationDeadline { /// Self-message scheduled when a session starts early; fires at the /// interval-2 boundary and publishes any aggregates held back by the /// publish-alignment rule (aggregates must not reach gossip before -/// interval 2). -pub(crate) struct FlushAggregatePublishes; +/// interval 2). Carries the session slot so a timer delayed past the next +/// session start is fenced (like [`AggregationDeadline`]) and cannot flush +/// the newer session's buffer before its own boundary. +pub(crate) struct FlushAggregatePublishes { + pub(crate) slot: u64, +} impl Message for FlushAggregatePublishes { type Result = (); } diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index edda6c2c..4de5bace 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -475,7 +475,7 @@ impl BlockChainServer { lead_ms = lead.as_millis() as u64, "Starting aggregation session early" ); - send_after(lead, ctx.clone(), FlushAggregatePublishes); + send_after(lead, ctx.clone(), FlushAggregatePublishes { slot }); } // Independent token per session. Shutdown propagates via our @@ -1219,14 +1219,26 @@ impl Handler for BlockChainServer { } impl Handler for BlockChainServer { - async fn handle(&mut self, _msg: FlushAggregatePublishes, _ctx: &Context) { + async fn handle(&mut self, msg: FlushAggregatePublishes, _ctx: &Context) { + // Fence like `AggregationDeadline`: only the current session's flush + // drains the buffer. A flush timer delayed past the next session start + // would otherwise publish that newer session's buffered aggregates + // before its own interval-2 boundary, breaking publish alignment. The + // superseded session's own aggregates were already dropped by the + // stale-drain in `start_aggregation_session`. + let is_current = self + .current_aggregation + .as_ref() + .is_some_and(|session| session.session_id == msg.slot); + if !is_current { + return; + } let pending = std::mem::take(&mut self.pending_aggregate_publishes); if pending.is_empty() { return; } - let session_id = self.current_aggregation.as_ref().map(|s| s.session_id); info!( - session_id, + slot = msg.slot, count = pending.len(), "Publishing aggregates held back until the interval-2 boundary" ); diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index 4308e130..add89fcc 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -191,16 +191,24 @@ pub struct BuiltSwarm { /// /// Evaluated once at startup — runtime aggregator toggles do not resubscribe /// (hot-standby model); see the invariant note on [`SwarmConfig`]. +/// +/// A zero `attestation_committee_count` yields no validator subnets (rather +/// than dividing by zero); callers enforce `>= 1`, so this is a defensive +/// guard on the public API, not a reachable path in the binary. pub fn compute_subscription_subnets( validator_ids: &[u64], attestation_committee_count: u64, is_aggregator: bool, aggregate_subnet_ids: Option<&[u64]>, ) -> HashSet { - let mut subnets: HashSet = validator_ids - .iter() - .map(|vid| vid % attestation_committee_count) - .collect(); + let mut subnets: HashSet = if attestation_committee_count == 0 { + HashSet::new() + } else { + validator_ids + .iter() + .map(|vid| vid % attestation_committee_count) + .collect() + }; if is_aggregator { if let Some(explicit_ids) = aggregate_subnet_ids { subnets.extend(explicit_ids); From 98fa2ea2a5f1b0fe4fccf782ca77e4706ca2dcaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:34:17 -0300 Subject: [PATCH 17/33] feat(blockchain): narrow early-aggregation window to 200ms Halve EARLY_AGGREGATION_WINDOW_MS (400 -> 200). Rescale the lean_aggregation_early_start_lead_seconds histogram buckets to 0.025-0.2s so they keep full resolution over the new window, and update the spec/plan (also fixing an inverted deadline-bound phrasing). --- crates/blockchain/src/aggregation.rs | 2 +- crates/blockchain/src/metrics.rs | 2 +- .../plans/2026-07-01-early-aggregation-start.md | 12 ++++++------ .../2026-07-01-early-aggregation-start-design.md | 14 +++++++------- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index 7f77e43a..4124d2c7 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -43,7 +43,7 @@ pub(crate) const PRIOR_WORKER_JOIN_TIMEOUT: Duration = Duration::from_secs(2); /// Width of the early-aggregation window: a session may start at most this /// long before the interval-2 boundary, provided the signature threshold is /// met (see `early_threshold_met`). -pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 400; +pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 200; // The window must fit within one interval: `early_aggregation_slot` subtracts // it from the interval-2 offset, and the interval-1 tick schedules the check diff --git a/crates/blockchain/src/metrics.rs b/crates/blockchain/src/metrics.rs index 11b63f2c..76289723 100644 --- a/crates/blockchain/src/metrics.rs +++ b/crates/blockchain/src/metrics.rs @@ -388,7 +388,7 @@ static LEAN_AGGREGATION_EARLY_START_LEAD_SECONDS: std::sync::LazyLock register_histogram!( "lean_aggregation_early_start_lead_seconds", "How far before the interval-2 boundary an early aggregation session started", - vec![0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4] + vec![0.025, 0.05, 0.075, 0.1, 0.125, 0.15, 0.175, 0.2] ) .unwrap() }); diff --git a/docs/superpowers/plans/2026-07-01-early-aggregation-start.md b/docs/superpowers/plans/2026-07-01-early-aggregation-start.md index f20dafd5..575e2587 100644 --- a/docs/superpowers/plans/2026-07-01-early-aggregation-start.md +++ b/docs/superpowers/plans/2026-07-01-early-aggregation-start.md @@ -2,9 +2,9 @@ > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. -**Goal:** Start the committee-signature aggregation session up to 400 ms before the interval-2 boundary when a single attestation-data group already holds 2/3 of the signatures expected from this node's aggregation subnets, while holding back any early-produced aggregates from gossip until the boundary. +**Goal:** Start the committee-signature aggregation session up to 200 ms before the interval-2 boundary when a single attestation-data group already holds 2/3 of the signatures expected from this node's aggregation subnets, while holding back any early-produced aggregates from gossip until the boundary. -**Architecture:** The blockchain actor (`BlockChainServer`, GenServer pattern) gains an early-trigger check invoked from two sites: after every stored gossip signature (`NewAttestation` handler) and once via a timer fired at the window opening (`T2 − 400 ms`). The trigger calls the existing `start_aggregation_session`, which becomes the single session-start path for both early and normal (interval-2 tick) starts; the tick skips if the slot's session already exists. Aggregates produced before T2 are buffered on the actor and flushed by a `send_after` timer at T2. +**Architecture:** The blockchain actor (`BlockChainServer`, GenServer pattern) gains an early-trigger check invoked from two sites: after every stored gossip signature (`NewAttestation` handler) and once via a timer fired at the window opening (`T2 − 200 ms`). The trigger calls the existing `start_aggregation_session`, which becomes the single session-start path for both early and normal (interval-2 tick) starts; the tick skips if the slot's session already exists. Aggregates produced before T2 are buffered on the actor and flushed by a `send_after` timer at T2. **Tech Stack:** Rust (edition 2024), spawned-concurrency actors (`send_after` self-messages), prometheus metrics via `ethlambda_metrics`. @@ -26,8 +26,8 @@ slot start T1=+800ms T2=+1600ms T3=+2400ms T4=+3200ms |------------|------------|------------|------------| ^ attest ^ aggregate ^ safe tgt ^ accept/build - [T2-400, T2) = early window - ^ timer check (scheduled at T1 + 400ms) + [T2-200, T2) = early window + ^ timer check (scheduled at T1 + 600ms) ^..^..^ per-insert checks as gossip sigs arrive ``` @@ -179,7 +179,7 @@ In `crates/blockchain/src/aggregation.rs`, after `PRIOR_WORKER_JOIN_TIMEOUT` (~l /// Width of the early-aggregation window: a session may start at most this /// long before the interval-2 boundary, provided the signature threshold is /// met (see `early_threshold_met`). -pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 400; +pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 200; /// Wall-clock millisecond timestamp of `slot`'s interval-2 boundary (the /// normal aggregation start). @@ -814,7 +814,7 @@ Run a local 4-node devnet on this branch and verify the early-start behavior end Check the aggregator node's logs and metrics: -1. `"Early-aggregation threshold met"` and `"Starting aggregation session early"` appear for most slots (local gossip is fast; signatures land well inside the window). `lead_ms` values must be in `(0, 400]`. +1. `"Early-aggregation threshold met"` and `"Starting aggregation session early"` appear for most slots (local gossip is fast; signatures land well inside the window). `lead_ms` values must be in `(0, 200]`. 2. `"Publishing aggregates held back until the interval-2 boundary"` appears with `count >= 1`. 3. `lean_aggregation_early_starts_total` grows (metrics port, default `:5054`, path `/metrics`). 4. `lean_aggregation_early_start_lead_seconds` observations sit in `(0, 0.4]`. diff --git a/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md b/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md index e9ca5ddb..77b7fc2b 100644 --- a/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md +++ b/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md @@ -8,7 +8,7 @@ Committee-signature aggregation (leanVM XMSS proofs) is the dominant cost in the attestation pipeline. Today the aggregation session starts exactly at the interval-2 tick, even when all expected signatures arrived earlier. Starting the -session up to 400 ms early, when enough signatures are already present, gives the +session up to 200 ms early, when enough signatures are already present, gives the expensive proofs more wall-clock runway before block building consumes them at interval 4. @@ -36,7 +36,7 @@ interval 4. Aggregator-only, fires at most once per slot. -- **Window:** wall clock in `[T2 - 400 ms, T2)`, where +- **Window:** wall clock in `[T2 - 200 ms, T2)`, where `T2 = genesis_ms + slot * MILLISECONDS_PER_SLOT + 2 * MILLISECONDS_PER_INTERVAL` is the current slot's interval-2 boundary. - **Condition:** some current-slot attestation-data group has gossip signatures @@ -56,7 +56,7 @@ Aggregator-only, fires at most once per slot. 2. A one-shot `EarlyAggregationCheck` self-message (no payload — the slot is recomputed from the wall clock at fire time, so a late-fired timer can never act on a stale slot) scheduled at the interval-1 tick via - `send_after(400 ms)` (interval-1 tick + 400 ms = T2 - 400 ms), covering + `send_after(200 ms)` (interval-1 tick + 600 ms = T2 - 200 ms), covering the case where the threshold was already met before the window opened. If the interval-1 tick is skipped (overrun), the per-insert checks still apply. @@ -109,10 +109,10 @@ pattern: build early, publish aligned to the boundary). `AGGREGATION_DEADLINE`: 750 ms → 800 ms, still measured from session start. -- Early session started at `T2 - x` (x ≤ 400): deadline at `T2 - x + 800`, - i.e. at most `T2 + 400`. +- Early session started at `T2 - x` (0 < x ≤ 200): deadline at `T2 - x + 800`, + i.e. no earlier than `T2 + 600` (200 ms before the interval-3 boundary). - Normal session started at `T2`: deadline lands exactly on the interval-3 - boundary. This removes the previous 50 ms publish/propagation margin — + boundary. This removes the previous 50 ms publish/propagation margin; accepted trade-off (the deadline only stops *new* jobs from starting; a job mid-proof finishes and publishes slightly after). @@ -156,7 +156,7 @@ model (runtime toggles do not resubscribe subnets). - Counter `lean_aggregation_early_starts_total`: early sessions started. - Histogram for early-start lead time (`T2 - start`, seconds, buckets covering - 0–400 ms). + 0–200 ms). - Existing session logs gain an `early: bool` field so devnet log analysis can attribute publish-time shifts. From 50adb83e06ba463c533bae48a33801c3140f4fc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 16:03:56 -0300 Subject: [PATCH 18/33] chore: drop internal design/plan docs from the PR --- .../2026-07-01-early-aggregation-start.md | 832 ------------------ ...26-07-01-early-aggregation-start-design.md | 181 ---- 2 files changed, 1013 deletions(-) delete mode 100644 docs/superpowers/plans/2026-07-01-early-aggregation-start.md delete mode 100644 docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md diff --git a/docs/superpowers/plans/2026-07-01-early-aggregation-start.md b/docs/superpowers/plans/2026-07-01-early-aggregation-start.md deleted file mode 100644 index 575e2587..00000000 --- a/docs/superpowers/plans/2026-07-01-early-aggregation-start.md +++ /dev/null @@ -1,832 +0,0 @@ -# Early Aggregation Start Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Start the committee-signature aggregation session up to 200 ms before the interval-2 boundary when a single attestation-data group already holds 2/3 of the signatures expected from this node's aggregation subnets, while holding back any early-produced aggregates from gossip until the boundary. - -**Architecture:** The blockchain actor (`BlockChainServer`, GenServer pattern) gains an early-trigger check invoked from two sites: after every stored gossip signature (`NewAttestation` handler) and once via a timer fired at the window opening (`T2 − 200 ms`). The trigger calls the existing `start_aggregation_session`, which becomes the single session-start path for both early and normal (interval-2 tick) starts; the tick skips if the slot's session already exists. Aggregates produced before T2 are buffered on the actor and flushed by a `send_after` timer at T2. - -**Tech Stack:** Rust (edition 2024), spawned-concurrency actors (`send_after` self-messages), prometheus metrics via `ethlambda_metrics`. - -**Verification policy (per user):** No new unit tests. Each task verifies by compiling (plus existing test suites where they already exist); final validation is a local 4-node devnet run (Task 9). - -**Spec:** `docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md` - -**Key existing code:** -- `crates/blockchain/src/lib.rs` — actor, tick loop (`on_tick`), `start_aggregation_session` (~line 366), handlers (`NewAttestation` ~line 1034, `AggregateProduced` ~line 1050, `AggregationDone` ~line 1079, `AggregationDeadline` ~line 1099) -- `crates/blockchain/src/aggregation.rs` — session types, snapshot builders, worker, `AGGREGATION_DEADLINE` -- `crates/blockchain/src/metrics.rs` — prometheus registration patterns -- `crates/storage/src/store.rs` — `GossipSignatureBuffer` (~line 348), `Store` methods (~line 1451) -- `crates/net/p2p/src/lib.rs` — `build_swarm` subnet subscription (~lines 304–337) -- `bin/ethlambda/src/main.rs` — `BlockChain::spawn` (~line 213), `build_swarm(SwarmConfig {...})` (~line 230) - -**Timing model (4 s slots, 5 × 800 ms intervals):** - -``` -slot start T1=+800ms T2=+1600ms T3=+2400ms T4=+3200ms - |------------|------------|------------|------------| - ^ attest ^ aggregate ^ safe tgt ^ accept/build - [T2-200, T2) = early window - ^ timer check (scheduled at T1 + 600ms) - ^..^..^ per-insert checks as gossip sigs arrive -``` - ---- - -### Task 1: Store query — max gossip group count for a slot - -The early threshold needs "largest signature count among current-slot data groups" without cloning signatures (the existing `iter_gossip_signatures` snapshot clones every signature; signatures are ~3 KB each). - -**Files:** -- Modify: `crates/storage/src/store.rs` (buffer method after `snapshot()` ~line 465; `Store` method next to `gossip_signatures_count()` ~line 1451) - -- [ ] **Step 1: Implement the buffer method and Store wrapper** - -In `impl GossipSignatureBuffer`, after `snapshot()` (~line 465): - -```rust -/// Largest signature count among data groups whose attestation slot is `slot`. -fn max_group_count_for_slot(&self, slot: u64) -> usize { - self.data - .values() - .filter(|entry| entry.data.slot == slot) - .map(|entry| entry.signatures.len()) - .max() - .unwrap_or(0) -} -``` - -In `impl Store`, next to `gossip_signatures_count()` (~line 1451): - -```rust -/// Largest per-group signature count among gossip groups voting for `slot`. -/// -/// One lock, no signature clones — cheap enough to call per gossip insert. -/// Drives the early-aggregation threshold check. -pub fn max_gossip_group_count_for_slot(&self, slot: u64) -> usize { - let gossip = self.gossip_signatures.lock().unwrap(); - gossip.max_group_count_for_slot(slot) -} -``` - -- [ ] **Step 2: Build** - -Run: `cargo build -p ethlambda-storage` -Expected: clean build, no warnings (the `pub` Store method keeps the private buffer method alive). - -- [ ] **Step 3: Commit** - -```bash -git add crates/storage/src/store.rs -git commit -m "feat(storage): add max gossip group count query for early aggregation" -``` - ---- - -### Task 2: p2p — extract `compute_subscription_subnets` helper - -Single source of truth for the subnet set, callable from `main.rs` (Task 4) without touching `SwarmConfig`. - -**Files:** -- Modify: `crates/net/p2p/src/lib.rs` (subnet logic in `build_swarm` ~lines 304–337) - -- [ ] **Step 1: Implement the helper and use it in `build_swarm`** - -Add near `build_swarm` (above it, ~line 185): - -```rust -/// Compute the set of attestation subnets this node subscribes to (and, when -/// aggregating, aggregates over), per leanSpec (`src/lean_spec/__main__.py`): -/// every validator subscribes to its own subnet (`vid % committee_count`) for -/// mesh health; aggregators additionally subscribe to explicit -/// `aggregate_subnet_ids` and fall back to subnet 0 when the set would -/// otherwise be empty. -/// -/// Evaluated once at startup — runtime aggregator toggles do not resubscribe -/// (hot-standby model); see the invariant note on [`SwarmConfig`]. -pub fn compute_subscription_subnets( - validator_ids: &[u64], - attestation_committee_count: u64, - is_aggregator: bool, - aggregate_subnet_ids: Option<&[u64]>, -) -> HashSet { - let mut subnets: HashSet = validator_ids - .iter() - .map(|vid| vid % attestation_committee_count) - .collect(); - if is_aggregator { - if let Some(explicit_ids) = aggregate_subnet_ids { - subnets.extend(explicit_ids); - } - if subnets.is_empty() { - subnets.insert(0); - } - } - subnets -} -``` - -Then replace the inline logic in `build_swarm` (~lines 304–329). The metric must keep reflecting validator-only subnets, so that part stays: - -```rust - // Subscribe to attestation subnets — see `compute_subscription_subnets`. - // The committee metric should reflect validator membership only, not - // aggregator-only subscriptions. - let metric_subnet = config - .validator_ids - .iter() - .map(|vid| vid % config.attestation_committee_count) - .min() - .unwrap_or(0); - metrics::set_attestation_committee_subnet(metric_subnet); - - let subscription_subnets = compute_subscription_subnets( - &config.validator_ids, - config.attestation_committee_count, - config.is_aggregator, - config.aggregate_subnet_ids.as_deref(), - ); -``` - -(The `let mut attestation_topics ...` loop over `&subscription_subnets` at ~line 331 stays unchanged.) - -- [ ] **Step 2: Build and run existing p2p tests** - -Run: `cargo build -p ethlambda-p2p && cargo test -p ethlambda-p2p --lib` -Expected: clean build, existing tests pass (behavior of `build_swarm` is unchanged — same set, same subscriptions). - -- [ ] **Step 3: Commit** - -```bash -git add crates/net/p2p/src/lib.rs -git commit -m "refactor(p2p): extract subscription subnet computation into pure helper" -``` - ---- - -### Task 3: Blockchain — pure early-aggregation helpers - -Window math and threshold predicate. - -**Files:** -- Modify: `crates/blockchain/src/aggregation.rs` (new consts + functions after `PRIOR_WORKER_JOIN_TIMEOUT` ~line 37) - -- [ ] **Step 1: Implement consts and functions** - -In `crates/blockchain/src/aggregation.rs`, after `PRIOR_WORKER_JOIN_TIMEOUT` (~line 37). Also add `use crate::{MILLISECONDS_PER_INTERVAL, MILLISECONDS_PER_SLOT};` to the file's imports (both constants are `pub` in `lib.rs`). - -```rust -/// Width of the early-aggregation window: a session may start at most this -/// long before the interval-2 boundary, provided the signature threshold is -/// met (see `early_threshold_met`). -pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 200; - -/// Wall-clock millisecond timestamp of `slot`'s interval-2 boundary (the -/// normal aggregation start). -pub(crate) fn interval2_boundary_ms(genesis_time_ms: u64, slot: u64) -> u64 { - genesis_time_ms + slot * MILLISECONDS_PER_SLOT + 2 * MILLISECONDS_PER_INTERVAL -} - -/// If `now_ms` falls inside some slot's early-aggregation window -/// (`[T2 - EARLY_AGGREGATION_WINDOW_MS, T2)` with `T2` that slot's interval-2 -/// boundary), return that slot. -pub(crate) fn early_aggregation_slot(now_ms: u64, genesis_time_ms: u64) -> Option { - let since_genesis = now_ms.checked_sub(genesis_time_ms)?; - let ms_into_slot = since_genesis % MILLISECONDS_PER_SLOT; - let t2_offset = 2 * MILLISECONDS_PER_INTERVAL; - let in_window = - ms_into_slot >= t2_offset - EARLY_AGGREGATION_WINDOW_MS && ms_into_slot < t2_offset; - in_window.then_some(since_genesis / MILLISECONDS_PER_SLOT) -} - -/// Early-start threshold: a single attestation-data group holds at least 2/3 -/// of the signatures expected from this node's aggregation subnets. At most -/// one group per slot can satisfy this (each validator signs once per slot). -pub(crate) fn early_threshold_met(max_group_count: usize, expected: usize) -> bool { - expected > 0 && max_group_count * 3 >= expected * 2 -} -``` - -- [ ] **Step 2: Build** - -Run: `cargo build -p ethlambda-blockchain` -Expected: clean build (dead-code warnings possible until Tasks 6–7 wire the callers). - -- [ ] **Step 3: Commit** - -```bash -git add crates/blockchain/src/aggregation.rs -git commit -m "feat(blockchain): add early-aggregation window and threshold helpers" -``` - ---- - -### Task 4: Plumbing — subnet set into the actor, expected-count field - -Compile-level wiring, no behavior change yet. - -**Files:** -- Modify: `crates/blockchain/src/lib.rs` (`BlockChain::spawn` ~line 80, `BlockChainServer` struct ~line 138) -- Modify: `bin/ethlambda/src/main.rs` (~lines 206–223) - -- [ ] **Step 1: Add spawn parameter and fields** - -In `crates/blockchain/src/lib.rs`, change `BlockChain::spawn` signature (~line 80) — new `aggregation_subnets` parameter after `attestation_committee_count`: - -```rust - pub fn spawn( - store: Store, - validator_keys: HashMap, - aggregator: AggregatorController, - attestation_committee_count: u64, - aggregation_subnets: HashSet, - gate_duties: bool, - proposer_config: ProposerConfig, - ) -> BlockChain { -``` - -Inside `spawn`, before the `let handle = BlockChainServer {` block (~line 101), compute the expected count (guard the modulo against a zero committee count): - -```rust - // Denominator of the early-aggregation 2/3 threshold: how many - // validators attest on subnets this node aggregates. Computed once — - // the validator registry is static and `head_state()` clones the full - // state, so this must not run per gossip insert. - let validator_count = store.head_state().validators.len() as u64; - let early_aggregation_expected_sigs = if attestation_committee_count == 0 { - 0 - } else { - (0..validator_count) - .filter(|vid| { - aggregation_subnets.contains(&(vid % attestation_committee_count)) - }) - .count() - }; -``` - -Add to the `BlockChainServer` struct literal in `spawn` (`genesis_time` is already computed at ~line 90): - -```rust - genesis_time_ms: genesis_time * 1000, - early_aggregation_expected_sigs, - pending_aggregate_publishes: Vec::new(), -``` - -Add the fields to the `BlockChainServer` struct definition (after `attestation_committee_count`, ~line 170): - -```rust - /// Genesis time in milliseconds, cached at spawn. `store.config()` is an - /// uncached backend read, too heavy for the per-gossip-insert early - /// checks that need this. - genesis_time_ms: u64, - - /// Number of validators whose subnet is one this node aggregates — the - /// denominator of the early-aggregation 2/3 threshold. Computed once at - /// spawn (the validator registry is static). - early_aggregation_expected_sigs: usize, - - /// Aggregates produced before the interval-2 boundary, held back so they - /// are not gossiped early. Flushed by `FlushAggregatePublishes` at the - /// boundary; only ever holds the current slot's aggregates. - pending_aggregate_publishes: Vec, -``` - -(`HashSet` and `SignedAggregatedAttestation` are already imported in `lib.rs`. Existing `config().genesis_time * 1000` call sites in `on_tick`/`handle_tick` stay as they are — only the new code paths use the cached field.) - -- [ ] **Step 2: Update `main.rs`** - -In `bin/ethlambda/src/main.rs`, add `compute_subscription_subnets` to the `ethlambda_p2p` import (line 40): - -```rust -use ethlambda_p2p::{ - Bootnode, P2P, PeerId, SwarmConfig, build_swarm, compute_subscription_subnets, parse_enrs, -}; -``` - -Before `BlockChain::spawn` (~line 213), compute the set with the same startup inputs `build_swarm` uses, and pass it (borrow `aggregate_subnet_ids` — it is moved into `SwarmConfig` further down): - -```rust - // Same startup inputs build_swarm uses — single source of truth is the - // shared helper. Frozen at startup like the gossip subscriptions - // themselves (hot-standby model). - let aggregation_subnets = compute_subscription_subnets( - &validator_ids, - attestation_committee_count, - options.is_aggregator, - options.aggregate_subnet_ids.as_deref(), - ); - - let blockchain = BlockChain::spawn( - store.clone(), - validator_keys, - aggregator.clone(), - attestation_committee_count, - aggregation_subnets, - !options.disable_duty_sync_gate, - ProposerConfig { - enable_proposer_aggregation: options.enable_proposer_aggregation, - max_attestations_per_block: options.max_attestations_per_block, - }, - ); -``` - -- [ ] **Step 3: Build the workspace** - -Run: `cargo build --workspace` -Expected: clean build. Dead-code warnings on the two new fields are acceptable here (plain `cargo build` does not deny warnings); they become live in Tasks 6–7, before `make lint` runs in Task 8. Do not annotate with `#[allow(dead_code)]`. - -- [ ] **Step 4: Commit** - -```bash -git add crates/blockchain/src/lib.rs bin/ethlambda/src/main.rs -git commit -m "feat(blockchain): plumb aggregation subnet set and expected-signature count into actor" -``` - ---- - -### Task 5: Deadline change 750 → 800 ms - -**Files:** -- Modify: `crates/blockchain/src/aggregation.rs:28-33` -- Modify: `crates/blockchain/src/lib.rs` (stale `+750 ms` doc-comment reference on `start_aggregation_session`, ~line 365) - -- [ ] **Step 1: Change the constant and its comment** - -Replace: - -```rust -/// Soft deadline for committee-signature aggregation measured from the -/// interval-2 tick. After this much wall time elapses, the actor signals the -/// worker to stop via its cancellation token. The 50 ms budget before the next -/// interval (interval 3 at +800 ms) is reserved for publishing any late-arriving -/// aggregates and for gossip propagation margin. -pub(crate) const AGGREGATION_DEADLINE: Duration = Duration::from_millis(750); -``` - -with: - -```rust -/// Soft deadline for committee-signature aggregation measured from session -/// start. After this much wall time elapses, the actor signals the worker to -/// stop via its cancellation token. A session started exactly at interval 2 -/// gets the full interval (interval 3 is one interval later); a session -/// started early (see `early_aggregation_slot`) ends correspondingly earlier. -/// The deadline only stops new jobs from starting — a job mid-proof finishes -/// and publishes right after. -pub(crate) const AGGREGATION_DEADLINE: Duration = Duration::from_millis(800); -``` - -- [ ] **Step 2: Build and check references** - -Run: `cargo build -p ethlambda-blockchain && grep -rn "750" crates/blockchain/src/lib.rs` -Expected: clean build; the grep finds no stale `+750 ms` comment references (the doc comment on `start_aggregation_session` ~line 365 says "Schedule the `AggregationDeadline` self-message at +750 ms" — update it to "+`AGGREGATION_DEADLINE`"). - -- [ ] **Step 3: Commit** - -```bash -git add crates/blockchain/src/aggregation.rs crates/blockchain/src/lib.rs -git commit -m "feat(blockchain): extend aggregation deadline to a full interval" -``` - ---- - -### Task 6: Publish alignment — hold early aggregates until the interval-2 boundary - -**Files:** -- Modify: `crates/blockchain/src/aggregation.rs` (`AggregationSession` struct ~line 75, new message) -- Modify: `crates/blockchain/src/lib.rs` (imports ~line 16, `start_aggregation_session` ~line 366, `AggregateProduced` handler ~line 1050, `AggregationDone` handler ~line 1079, interval-2 tick arm ~line 312, new handler + helper) -- Modify: `crates/blockchain/src/metrics.rs` (new counter + histogram) - -- [ ] **Step 1: Add the `early` flag and the flush message in `aggregation.rs`** - -Add field to `AggregationSession` (~line 75): - -```rust -pub(crate) struct AggregationSession { - /// Slot at which this session was started; used as a fencing id so we can - /// drop late-arriving messages from a prior session. - pub(crate) session_id: u64, - /// Whether the session started before the slot's interval-2 boundary via - /// the early-aggregation trigger. - pub(crate) early: bool, - /// Child of the actor cancellation token; fires either at the deadline or - /// when the actor itself is stopping. - pub(crate) cancel: CancellationToken, - /// Handle to the `spawn_blocking` worker. Held so `stopped()` / new-session - /// start can await completion. - pub(crate) worker: tokio::task::JoinHandle<()>, -} -``` - -Add next to `AggregationDeadline` (~line 112): - -```rust -/// Self-message scheduled when a session starts early; fires at the -/// interval-2 boundary and publishes any aggregates held back by the -/// publish-alignment rule (aggregates must not reach gossip before -/// interval 2). -pub(crate) struct FlushAggregatePublishes; -impl Message for FlushAggregatePublishes { - type Result = (); -} -``` - -- [ ] **Step 2: Rework `start_aggregation_session` in `lib.rs`** - -Extend the `use crate::aggregation::{...}` import (~line 16) with `FlushAggregatePublishes`. - -In `start_aggregation_session` (~line 366): move the coverage emission here from the tick arm, and compute `early` + schedule the flush. After the prior-session join block and before the snapshot: - -```rust - coverage::emit_agg_start_new_coverage(&self.store, self.attestation_committee_count); -``` - -Replace the tail of the function (from `let session_id = slot;` through the final `self.current_aggregation = Some(...)`) with: - -```rust - let session_id = slot; - // Any leftovers from a prior slot mean its flush never fired (a - // backwards wall-clock step, or a flush timer delayed past the next - // session). Drop them — late aggregates are dropped, same policy as - // signatures that miss the snapshot. - let stale = std::mem::take(&mut self.pending_aggregate_publishes); - if !stale.is_empty() { - warn!( - count = stale.len(), - "Dropping stale pending aggregate publishes" - ); - } - let t2_ms = aggregation::interval2_boundary_ms(self.genesis_time_ms, slot); - let now_ms = unix_now_ms(); - let early = now_ms < t2_ms; - if early { - // Publish alignment: aggregates must not reach gossip before the - // interval-2 boundary. Aggregates produced before T2 are buffered - // in `pending_aggregate_publishes`; this timer flushes them at T2. - let lead = Duration::from_millis(t2_ms - now_ms); - metrics::inc_aggregation_early_starts(); - metrics::observe_aggregation_early_start_lead(lead); - info!( - %slot, - lead_ms = lead.as_millis() as u64, - "Starting aggregation session early" - ); - send_after(lead, ctx.clone(), FlushAggregatePublishes); - } - - // Independent token per session. Shutdown propagates via our - // #[stopped] hook which cancels any current session; the deadline - // timer cancels this specific session at +AGGREGATION_DEADLINE. - let cancel = CancellationToken::new(); - let actor_ref = ctx.actor_ref(); - - let worker_cancel = cancel.clone(); - let worker_actor = actor_ref.clone(); - let worker = tokio::task::spawn_blocking(move || { - run_aggregation_worker(snapshot, worker_actor, worker_cancel, session_id); - }); - - let _deadline_timer = send_after( - AGGREGATION_DEADLINE, - ctx.clone(), - AggregationDeadline { session_id }, - ); - - self.current_aggregation = Some(AggregationSession { - session_id, - early, - cancel, - worker, - }); -``` - -(`metrics::inc_aggregation_early_starts` / `observe_aggregation_early_start_lead` are added in Step 4 of this task.) - -Remove the emission from the interval-2 tick arm (~line 312), which becomes: - -```rust - // ==== interval 2 ==== - 2 => { - if is_aggregator { - self.start_aggregation_session(slot, ctx).await; - } else { - metrics::inc_aggregator_skipped_not_aggregator(); - } - } -``` - -(The `coverage::emit_agg_start_new_coverage` call and its old surrounding braces go away; the early-session skip guard lands in Task 7.) - -- [ ] **Step 3: Split publish out of `AggregateProduced` and add the flush handler** - -Add a helper method on `BlockChainServer` (near `on_gossip_attestation`, ~line 929): - -```rust - /// Publish an aggregated attestation to the aggregation gossip topic. - fn publish_aggregate(&self, aggregate: SignedAggregatedAttestation) { - if let Some(ref p2p) = self.p2p { - let _ = p2p - .publish_aggregated_attestation(aggregate) - .inspect_err(|err| error!(%err, "Failed to publish aggregated attestation")); - } - } -``` - -Rewrite the body of `impl Handler` (~line 1050) — the session-id fencing stays, then: - -```rust - aggregation::apply_aggregated_group(&mut self.store, &msg.output); - - let aggregate = SignedAggregatedAttestation { - data: msg.output.hashed.data().clone(), - proof: msg.output.proof, - }; - - // Publish alignment: hold back aggregates produced before this slot's - // interval-2 boundary; `FlushAggregatePublishes` publishes them at T2. - // (`session_id` is the session's slot.) - let t2_ms = aggregation::interval2_boundary_ms(self.genesis_time_ms, msg.session_id); - if unix_now_ms() < t2_ms { - self.pending_aggregate_publishes.push(aggregate); - return; - } - self.publish_aggregate(aggregate); -``` - -Add the flush handler next to the other aggregation handlers (~line 1099): - -```rust -impl Handler for BlockChainServer { - async fn handle(&mut self, _msg: FlushAggregatePublishes, _ctx: &Context) { - let pending = std::mem::take(&mut self.pending_aggregate_publishes); - if pending.is_empty() { - return; - } - info!( - count = pending.len(), - "Publishing aggregates held back until the interval-2 boundary" - ); - for aggregate in pending { - self.publish_aggregate(aggregate); - } - } -} -``` - -Add the `early` field to the `AggregationDone` completion log (~line 1085): before the `info!`, compute - -```rust - let early = self - .current_aggregation - .as_ref() - .is_some_and(|s| s.session_id == msg.session_id && s.early); -``` - -and add `early,` to the `info!(...)` field list (after `cancelled = msg.cancelled,`). - -- [ ] **Step 4: Add the two metrics** - -In `crates/blockchain/src/metrics.rs` — counter in the `// --- Counters ---` section, histogram in `// --- Histograms ---`, public fns near their statics following file style: - -```rust -static LEAN_AGGREGATION_EARLY_STARTS_TOTAL: std::sync::LazyLock = - std::sync::LazyLock::new(|| { - register_int_counter!( - "lean_aggregation_early_starts_total", - "Aggregation sessions started before the interval-2 boundary" - ) - .unwrap() - }); - -pub fn inc_aggregation_early_starts() { - LEAN_AGGREGATION_EARLY_STARTS_TOTAL.inc(); -} -``` - -```rust -static LEAN_AGGREGATION_EARLY_START_LEAD_SECONDS: std::sync::LazyLock = - std::sync::LazyLock::new(|| { - register_histogram!( - "lean_aggregation_early_start_lead_seconds", - "How far before the interval-2 boundary an early aggregation session started", - vec![0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4] - ) - .unwrap() - }); - -pub fn observe_aggregation_early_start_lead(lead: Duration) { - LEAN_AGGREGATION_EARLY_START_LEAD_SECONDS.observe(lead.as_secs_f64()); -} -``` - -- [ ] **Step 5: Build and run existing tests** - -Run: `cargo build -p ethlambda-blockchain && cargo test -p ethlambda-blockchain --lib` -Expected: clean build, existing unit tests pass. Behavior so far is identical for normal sessions (`early` is always false when started at interval 2, so no buffering, no flush timer, no metric increments). - -- [ ] **Step 6: Commit** - -```bash -git add crates/blockchain/src/aggregation.rs crates/blockchain/src/lib.rs crates/blockchain/src/metrics.rs -git commit -m "feat(blockchain): hold early-produced aggregates until the interval-2 boundary" -``` - ---- - -### Task 7: Early trigger — threshold checks, timer, and interval-2 skip - -**Files:** -- Modify: `crates/blockchain/src/aggregation.rs` (new message next to `FlushAggregatePublishes`) -- Modify: `crates/blockchain/src/lib.rs` (imports ~line 16, interval-1 arm ~line 290, interval-2 arm ~line 312, `NewAttestation` handler ~line 1034, new method + handler) - -- [ ] **Step 1: Add the check message in `aggregation.rs`** - -```rust -/// One-shot self-message scheduled at the interval-1 tick; fires when the -/// early-aggregation window opens (T2 - EARLY_AGGREGATION_WINDOW_MS) to run -/// the threshold check for signatures that all arrived before the window. -/// Arrivals inside the window are checked per insert instead. -pub(crate) struct EarlyAggregationCheck; -impl Message for EarlyAggregationCheck { - type Result = (); -} -``` - -- [ ] **Step 2: Add the trigger method on `BlockChainServer`** - -Extend the `use crate::aggregation::{...}` import with `EarlyAggregationCheck` and `EARLY_AGGREGATION_WINDOW_MS`. Add the method after `start_aggregation_session` (~line 418): - -```rust - /// Early-aggregation trigger: start the slot's session ahead of the - /// interval-2 tick when, inside the window `[T2 - EARLY_AGGREGATION_WINDOW_MS, T2)`, - /// a single attestation-data group already holds 2/3 of the signatures - /// expected from this node's aggregation subnets. Called after every - /// stored gossip signature and once at the window opening via - /// [`EarlyAggregationCheck`]. Fires at most once per slot: the started - /// session stays in `current_aggregation` (running or finished) until the - /// next session replaces it. - async fn maybe_start_early_aggregation(&mut self, ctx: &Context) { - if !self.aggregator.is_enabled() { - return; - } - let Some(slot) = aggregation::early_aggregation_slot(unix_now_ms(), self.genesis_time_ms) - else { - return; - }; - if self - .current_aggregation - .as_ref() - .is_some_and(|session| session.session_id == slot) - { - return; - } - let max_group = self.store.max_gossip_group_count_for_slot(slot); - if !aggregation::early_threshold_met(max_group, self.early_aggregation_expected_sigs) { - return; - } - info!( - %slot, - max_group, - expected = self.early_aggregation_expected_sigs, - "Early-aggregation threshold met" - ); - self.start_aggregation_session(slot, ctx).await; - } -``` - -- [ ] **Step 3: Wire the three trigger sites** - -**(a) Interval-1 arm** (~line 290) — at the end of the `1 => { ... }` block, after the attestation-production `if/else`: - -```rust - // Schedule the early-aggregation window check. This tick is - // one interval before T2, so the timer fires right as the - // window opens at T2 - EARLY_AGGREGATION_WINDOW_MS. - if is_aggregator { - send_after( - Duration::from_millis( - MILLISECONDS_PER_INTERVAL - EARLY_AGGREGATION_WINDOW_MS, - ), - ctx.clone(), - EarlyAggregationCheck, - ); - } -``` - -**(b) Timer handler** — next to the other aggregation handlers: - -```rust -impl Handler for BlockChainServer { - async fn handle(&mut self, _msg: EarlyAggregationCheck, ctx: &Context) { - self.maybe_start_early_aggregation(ctx).await; - } -} -``` - -**(c) Per-insert check** — `impl Handler` (~line 1034) becomes: - -```rust -impl Handler for BlockChainServer { - async fn handle(&mut self, msg: NewAttestation, ctx: &Context) { - self.on_gossip_attestation(&msg.attestation); - self.maybe_start_early_aggregation(ctx).await; - } -} -``` - -(The method's internal guards make this a few comparisons outside the window; `max_gossip_group_count_for_slot` is one lock with no clones inside it.) - -- [ ] **Step 4: Skip the interval-2 start when the early session exists** - -The `2 => { ... }` arm becomes: - -```rust - // ==== interval 2 ==== - 2 => { - if is_aggregator { - // The early trigger may have already started this slot's - // session (running or finished) — it IS the slot's session, - // so don't start a second one. - let already_started = self - .current_aggregation - .as_ref() - .is_some_and(|session| session.session_id == slot); - if !already_started { - self.start_aggregation_session(slot, ctx).await; - } - } else { - metrics::inc_aggregator_skipped_not_aggregator(); - } - } -``` - -- [ ] **Step 5: Build and run existing blockchain tests** - -Run: `cargo build --workspace && cargo test -p ethlambda-blockchain --lib` -Expected: clean build, existing tests pass. - -- [ ] **Step 6: Commit** - -```bash -git add crates/blockchain/src/aggregation.rs crates/blockchain/src/lib.rs -git commit -m "feat(blockchain): start aggregation early when 2/3 of subnet signatures arrived" -``` - ---- - -### Task 8: Lint and existing test suites - -**Files:** none (verification only) - -- [ ] **Step 1: Format and lint** - -Run: `make fmt && make lint` -Expected: no diffs from fmt; clippy clean with `-D warnings` (this is where any dead-code stragglers from Tasks 1–4 would surface — by now every added item has a caller). - -- [ ] **Step 2: Run the existing test suites** - -Run: `make test` -Expected: all workspace tests + forkchoice spec tests pass (no new tests were added; this guards against regressions in the touched paths). If fixtures are missing: `rm -rf leanSpec && make leanSpec/fixtures`. - -- [ ] **Step 3: Commit any fmt fallout** - -```bash -git add -u && git commit -m "chore: fmt" || true -``` - -(Skip the commit if the tree is clean.) - ---- - -### Task 9: Local 4-node devnet validation - -Run a local 4-node devnet on this branch and verify the early-start behavior end to end. Use the `devnet-runner` skill (`.claude/skills/devnet-runner/`) for the exact workflow; constraints that matter here: - -- **Release build only** — debug binaries stack-overflow in leanVM `rec_aggregation`. -- **Exactly one node with `--is-aggregator`** — multiple co-located aggregators stall finality via leanVM CPU contention. -- If using Docker Desktop, the VM needs 16+ GiB (leanVM peaks ~4–5 GiB per node); binary mode avoids this. - -- [ ] **Step 1: Start the devnet** - -4 nodes, fresh genesis, one aggregator. Let it run at least ~50 slots (~4 minutes). - -- [ ] **Step 2: Verify early-start behavior on the aggregator node** - -Check the aggregator node's logs and metrics: - -1. `"Early-aggregation threshold met"` and `"Starting aggregation session early"` appear for most slots (local gossip is fast; signatures land well inside the window). `lead_ms` values must be in `(0, 200]`. -2. `"Publishing aggregates held back until the interval-2 boundary"` appears with `count >= 1`. -3. `lean_aggregation_early_starts_total` grows (metrics port, default `:5054`, path `/metrics`). -4. `lean_aggregation_early_start_lead_seconds` observations sit in `(0, 0.4]`. -5. `"Committee signatures aggregated"` logs show `early=true` sessions. `"Prior aggregation worker still running"` warnings must be rare (the early start shrinks the worst-case gap to the previous slot's worker from 3250 ms to 2800 ms, so occasional joins under slow proofs are expected — every slot would be a bug). -6. No aggregate publish happens before its slot's interval-2 boundary: spot-check a few `"Starting aggregation session early"` slots and confirm the corresponding aggregated-attestation publish log lands at or after T2 (per-slot offset = `(log_epoch - GENESIS_TIME) mod 4 >= 1.6`). - -- [ ] **Step 3: Verify chain health** - -1. `lean_latest_finalized_slot` advances steadily on all 4 nodes (fresh local devnet should finalize nearly every justifiable slot). -2. Blocks carry attestations (`attestation_count > 0` in block logs). -3. No stalls, no panic/error-level logs attributable to aggregation. - -- [ ] **Step 4: Stop the devnet and report** - -Stop the nodes; summarize observed early-start rate (early sessions / slots), lead-time distribution, and finalization progress. diff --git a/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md b/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md deleted file mode 100644 index 77b7fc2b..00000000 --- a/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md +++ /dev/null @@ -1,181 +0,0 @@ -# Early Aggregation Start - -**Date:** 2026-07-01 -**Status:** Approved - -## Motivation - -Committee-signature aggregation (leanVM XMSS proofs) is the dominant cost in the -attestation pipeline. Today the aggregation session starts exactly at the -interval-2 tick, even when all expected signatures arrived earlier. Starting the -session up to 200 ms early, when enough signatures are already present, gives the -expensive proofs more wall-clock runway before block building consumes them at -interval 4. - -## Current behavior - -- The blockchain actor ticks at 800 ms interval boundaries. At interval 2 (if - `is_aggregator`), `start_aggregation_session` snapshots the current slot's - gossip signatures (`snapshot_current_slot_aggregation_inputs`) and spawns an - off-thread worker. A deadline timer cancels the session's token after - `AGGREGATION_DEADLINE` (750 ms). -- Gossip signatures accumulate in the store per attestation-data group as - `NewAttestation` messages arrive (plus self-delivery of own validators' - attestations at interval 1). Only aggregators store them. -- Each produced aggregate is applied to the store and published to gossip - immediately in the `AggregateProduced` handler. -- Subnet assignment is `vid % attestation_committee_count`. The set of subnets a - node subscribes to (own-validator subnets, plus `--aggregate-subnet-ids` for - aggregators, with a fallback to subnet 0 for validator-less aggregators) is - computed inside `build_swarm` in the p2p crate; the blockchain actor does not - know it. - -## New behavior - -### Early trigger - -Aggregator-only, fires at most once per slot. - -- **Window:** wall clock in `[T2 - 200 ms, T2)`, where - `T2 = genesis_ms + slot * MILLISECONDS_PER_SLOT + 2 * MILLISECONDS_PER_INTERVAL` - is the current slot's interval-2 boundary. -- **Condition:** some current-slot attestation-data group has gossip signatures - from at least 2/3 of the expected validators: - `group_count * 3 >= expected * 2`, with `expected > 0`. `group_count` is the - number of stored gossip signatures in that group (one per validator). - `expected` = number of validators whose - `vid % attestation_committee_count` is in this node's aggregation subnet set, - computed once at actor spawn (the validator registry is static, and - `Store::head_state()` clones the full state so per-check reads would be - needlessly expensive). - Note that at most one group per slot can reach 2/3 (each validator signs once - per slot), so the per-group condition can hold for at most one group. -- **Check sites:** - 1. After each gossip-signature insert in the `NewAttestation` handler, while - the wall clock is inside the window. - 2. A one-shot `EarlyAggregationCheck` self-message (no payload — the slot is - recomputed from the wall clock at fire time, so a late-fired timer can - never act on a stale slot) scheduled at the interval-1 tick via - `send_after(200 ms)` (interval-1 tick + 600 ms = T2 - 200 ms), covering - the case where the threshold was already met before the window opened. If - the interval-1 tick is skipped (overrun), the per-insert checks still - apply. -- **Once-per-slot guard:** a session with `session_id == slot` already exists in - `current_aggregation` (the field persists after the worker finishes; it is - only replaced at the next session start). - -### Action on trigger - -Call the existing `start_aggregation_session(slot, ctx)` — the full current-slot -snapshot, not just the group that hit the threshold. This is the slot's one and -only session; it merely starts early. The existing prior-session cancel+join in -`start_aggregation_session` handles a still-running previous-slot worker -identically in both paths. - -### Interval-2 tick - -If `current_aggregation` already holds this slot's session (running or -finished), skip starting a new one. Otherwise start normally — the unchanged -fallback for slots where the threshold is never met. - -`emit_agg_start_new_coverage` moves from the interval-2 tick into -`start_aggregation_session`, so the coverage report reflects the store state the -session actually snapshotted in both paths. - -### Publish alignment - -Aggregates must not reach gossip before interval 2 (mirrors the block prebuild -pattern: build early, publish aligned to the boundary). - -- `AggregateProduced` handler: apply the output to the store immediately - (nothing local consumes `new_aggregated_payloads` before interval 4, so early - application is unobservable locally). If the wall clock is `>= T2` for the - session's slot, publish immediately (unchanged). If `< T2`, push the - `SignedAggregatedAttestation` into a pending-publish buffer on the actor. -- When starting an *early* session, schedule a `FlushAggregatePublishes` - self-message via `send_after(T2 - now)`. Its handler publishes and drains the - buffer. This is the primary flush mechanism (not the interval-2 tick, which - can be skipped on overrun); the handler is idempotent (draining an empty - buffer is a no-op). -- The buffer is bounded in practice by the number of data groups in one slot. - Entries are never carried across slots: the flush timer fires at T2 of the - same slot that buffered them, and as a backstop `start_aggregation_session` - drains (drops, with a warning) any leftovers before installing a new session - — covering a backwards wall-clock step or a flush timer delayed past the - next session. Late aggregates are dropped, the same policy as signatures - that miss the snapshot. - -### Deadline change - -`AGGREGATION_DEADLINE`: 750 ms → 800 ms, still measured from session start. - -- Early session started at `T2 - x` (0 < x ≤ 200): deadline at `T2 - x + 800`, - i.e. no earlier than `T2 + 600` (200 ms before the interval-3 boundary). -- Normal session started at `T2`: deadline lands exactly on the interval-3 - boundary. This removes the previous 50 ms publish/propagation margin; - accepted trade-off (the deadline only stops *new* jobs from starting; a job - mid-proof finishes and publishes slightly after). - -### Subnet-set plumbing - -The subscription-subnet computation is hoisted out of `build_swarm` into a -shared pure helper in the p2p crate: -`compute_subscription_subnets(validator_ids, committee_count, is_aggregator, explicit_ids) -> HashSet`. -Both call sites use it with the same startup inputs, keeping a single source of -truth without churning `SwarmConfig`: - -- `build_swarm` (subscribes to exactly this set, behavior unchanged), and -- `main.rs`, which passes the resulting set to `BlockChain::spawn` for the - `expected` count. - -The set is frozen at startup, matching the existing hot-standby aggregator -model (runtime toggles do not resubscribe subnets). - -## Failure modes and edge cases - -- **Snapshot returns `None` after trigger**: unreachable in practice — the - trigger requires stored signatures, and `on_gossip_attestation` only stores - signatures whose pubkeys resolved during verification, so the snapshot - always yields at least one job for them. If it did happen, no session is - created and `current_aggregation` is left `None` (the prior session was - already taken), so the once-per-slot guard would not hold and per-insert - checks would retry; interval 2 remains the fallback. No signatures are - lost. -- **Threshold never met:** interval 2 starts the session exactly as today. -- **Signatures arriving after the early snapshot:** never aggregated that slot. - Accepted trade-off (explicitly chosen over a top-up session). -- **Wall-clock vs monotonic drift:** checks use `unix_now_ms()` like the rest of - the tick logic; the idempotency-guard pattern already tolerates drift. The - worst case of drift is a slightly mistimed early start or flush, never a - correctness issue. -- **Non-aggregators:** never store gossip signatures, and all check sites are - additionally guarded by `AggregatorController::is_enabled()` read at check - time. - -## Observability - -- Counter `lean_aggregation_early_starts_total`: early sessions started. -- Histogram for early-start lead time (`T2 - start`, seconds, buckets covering - 0–200 ms). -- Existing session logs gain an `early: bool` field so devnet log analysis can - attribute publish-time shifts. - -## Testing - -Per user decision, no new unit tests. Validation is: - -- Existing suites pass unchanged (normal interval-2 path is the fallback and - keeps its semantics; only the deadline constant changes). -- Local 4-node devnet run (1 aggregator, binary mode): early-start rate and - lead-time distribution from logs, held-back publish flushes landing at or - after the interval-2 boundary (per-slot offset), lockstep finalization, no - aggregation warnings/errors. - -## Non-goals - -- No gossip resubscription or topic changes. -- No change to the fork-choice attestation pipeline (`new_attestations` / - `known_attestations`). -- No top-up/second session per slot. -- No persistence of the pending-publish buffer (in-memory actor state, same as - the rest of the aggregation pipeline). From 8c4dc38b5f3e3589a4cd9afc71f8ba04262e95a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 16:17:52 -0300 Subject: [PATCH 19/33] feat(blockchain): set early-aggregation window to 600ms Window opens at T2-600 = 200ms into interval 1, so the trigger can fire as soon as attestations start arriving. Rescale the lead-time histogram buckets to 0.075-0.6s to match. --- crates/blockchain/src/aggregation.rs | 2 +- crates/blockchain/src/metrics.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index 4124d2c7..8721888b 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -43,7 +43,7 @@ pub(crate) const PRIOR_WORKER_JOIN_TIMEOUT: Duration = Duration::from_secs(2); /// Width of the early-aggregation window: a session may start at most this /// long before the interval-2 boundary, provided the signature threshold is /// met (see `early_threshold_met`). -pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 200; +pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 600; // The window must fit within one interval: `early_aggregation_slot` subtracts // it from the interval-2 offset, and the interval-1 tick schedules the check diff --git a/crates/blockchain/src/metrics.rs b/crates/blockchain/src/metrics.rs index 76289723..5ae6e9e0 100644 --- a/crates/blockchain/src/metrics.rs +++ b/crates/blockchain/src/metrics.rs @@ -388,7 +388,7 @@ static LEAN_AGGREGATION_EARLY_START_LEAD_SECONDS: std::sync::LazyLock register_histogram!( "lean_aggregation_early_start_lead_seconds", "How far before the interval-2 boundary an early aggregation session started", - vec![0.025, 0.05, 0.075, 0.1, 0.125, 0.15, 0.175, 0.2] + vec![0.075, 0.15, 0.225, 0.3, 0.375, 0.45, 0.525, 0.6] ) .unwrap() }); From e0907a60039722eb284fcc8ba9dae2664f170bfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 17:20:54 -0300 Subject: [PATCH 20/33] refactor(blockchain): compute early-aggregation threshold in closed form Replace the per-validator subnet scan in BlockChain::spawn with the closed form 2*N/(3*C) (two-thirds of one committee's expected votes), folding the 2/3 fraction into the precomputed threshold so early_threshold_met is now a direct >= comparison. Drop the now-unused aggregation_subnets parameter and its main.rs plumbing; compute_subscription_subnets is only used inside the p2p crate now, so downgrade it to pub(crate). --- bin/ethlambda/src/main.rs | 15 +------------- crates/blockchain/src/aggregation.rs | 12 ++++++----- crates/blockchain/src/lib.rs | 31 ++++++++++++++-------------- crates/net/p2p/src/lib.rs | 15 +++++++------- 4 files changed, 30 insertions(+), 43 deletions(-) diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index 19db7eb5..914c6c86 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -37,9 +37,7 @@ use ethlambda_blockchain::MILLISECONDS_PER_SLOT; use ethlambda_blockchain::block_builder::ProposerConfig; use ethlambda_blockchain::key_manager::ValidatorKeyPair; use ethlambda_network_api::{InitBlockChain, InitP2P, ToBlockChainToP2PRef, ToP2PToBlockChainRef}; -use ethlambda_p2p::{ - Bootnode, P2P, PeerId, SwarmConfig, build_swarm, compute_subscription_subnets, parse_enrs, -}; +use ethlambda_p2p::{Bootnode, P2P, PeerId, SwarmConfig, build_swarm, parse_enrs}; use ethlambda_types::primitives::{H256, HashTreeRoot as _}; use ethlambda_types::{ aggregator::AggregatorController, @@ -212,22 +210,11 @@ async fn main() -> eyre::Result<()> { // and the API server (which exposes GET/POST admin endpoints). let aggregator = AggregatorController::new(options.is_aggregator); - // Same startup inputs build_swarm uses — single source of truth is the - // shared helper. Frozen at startup like the gossip subscriptions - // themselves (hot-standby model). - let aggregation_subnets = compute_subscription_subnets( - &validator_ids, - attestation_committee_count, - options.is_aggregator, - options.aggregate_subnet_ids.as_deref(), - ); - let blockchain = BlockChain::spawn( store.clone(), validator_keys, aggregator.clone(), attestation_committee_count, - aggregation_subnets, !options.disable_duty_sync_gate, ProposerConfig { enable_proposer_aggregation: options.enable_proposer_aggregation, diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index 8721888b..80b2b259 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -73,11 +73,13 @@ pub(crate) fn early_aggregation_slot(now_ms: u64, genesis_time_ms: u64) -> Optio in_window.then_some(since_genesis / MILLISECONDS_PER_SLOT) } -/// Early-start threshold: a single attestation-data group holds at least 2/3 -/// of the signatures expected from this node's aggregation subnets. At most -/// one group per slot can satisfy this (each validator signs once per slot). -pub(crate) fn early_threshold_met(max_group_count: usize, expected: usize) -> bool { - expected > 0 && max_group_count * 3 >= expected * 2 +/// Early-start threshold: the largest single attestation-data group already +/// holds at least `min_group_sigs` signatures. `min_group_sigs` is the +/// precomputed two-thirds-of-a-committee count (see `BlockChain::spawn`), so +/// the 2/3 fraction is not re-applied here. At most one group per slot can +/// satisfy this (each validator signs once per slot). +pub(crate) fn early_threshold_met(max_group_count: usize, min_group_sigs: usize) -> bool { + min_group_sigs > 0 && max_group_count >= min_group_sigs } /// A single pre-prepared aggregation group. diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 4de5bace..5f8e5e3e 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -83,7 +83,6 @@ impl BlockChain { validator_keys: HashMap, aggregator: AggregatorController, attestation_committee_count: u64, - aggregation_subnets: HashSet, gate_duties: bool, proposer_config: ProposerConfig, ) -> BlockChain { @@ -100,17 +99,16 @@ impl BlockChain { (now_ms.saturating_sub(genesis_time * 1000) / MILLISECONDS_PER_SLOT) as u32; key_manager.advance_keys_to(current_slot); - // Denominator of the early-aggregation 2/3 threshold: how many - // validators attest on subnets this node aggregates. Computed once — - // the validator registry is static and `head_state()` clones the full - // state, so this must not run per gossip insert. + // Minimum single-group signature count that triggers early + // aggregation: two-thirds of one committee's expected votes, + // `2 * N / (3 * C)`. Closed form (no per-validator subnet scan), + // computed once; assumes an even committee split and one committee's + // worth of votes per group. let validator_count = store.head_state().validators.len() as u64; - let early_aggregation_expected_sigs = if attestation_committee_count == 0 { + let early_aggregation_min_group_sigs = if attestation_committee_count == 0 { 0 } else { - (0..validator_count) - .filter(|vid| aggregation_subnets.contains(&(vid % attestation_committee_count))) - .count() + (2 * validator_count / (3 * attestation_committee_count)) as usize }; let handle = BlockChainServer { @@ -124,7 +122,7 @@ impl BlockChain { last_tick_instant: None, attestation_committee_count, genesis_time_ms: genesis_time * 1000, - early_aggregation_expected_sigs, + early_aggregation_min_group_sigs, pending_aggregate_publishes: Vec::new(), proposer_config, pre_merge_coverage: None, @@ -196,10 +194,11 @@ pub struct BlockChainServer { /// code. genesis_time_ms: u64, - /// Number of validators whose subnet is one this node aggregates — the - /// denominator of the early-aggregation 2/3 threshold. Computed once at - /// spawn (the validator registry is static). - early_aggregation_expected_sigs: usize, + /// Minimum signatures in one attestation-data group that trigger early + /// aggregation: two-thirds of a committee's expected votes, + /// `2 * validator_count / (3 * committee_count)`. Computed once at spawn + /// (the validator registry is static). + early_aggregation_min_group_sigs: usize, /// Aggregates produced before the interval-2 boundary, held back so they /// are not gossiped early. Flushed by `FlushAggregatePublishes` at the @@ -531,13 +530,13 @@ impl BlockChainServer { return; } let max_group = self.store.max_gossip_group_count_for_slot(slot); - if !aggregation::early_threshold_met(max_group, self.early_aggregation_expected_sigs) { + if !aggregation::early_threshold_met(max_group, self.early_aggregation_min_group_sigs) { return; } info!( %slot, max_group, - expected = self.early_aggregation_expected_sigs, + min_group_sigs = self.early_aggregation_min_group_sigs, "Early-aggregation threshold met" ); self.start_aggregation_session(slot, ctx).await; diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index add89fcc..d6a3509c 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -182,20 +182,19 @@ pub struct BuiltSwarm { pub(crate) bootnode_addrs: HashMap, } -/// Compute the set of attestation subnets this node subscribes to (and, when -/// aggregating, aggregates over), per leanSpec (`src/lean_spec/__main__.py`): -/// every validator subscribes to its own subnet (`vid % committee_count`) for -/// mesh health; aggregators additionally subscribe to explicit -/// `aggregate_subnet_ids` and fall back to subnet 0 when the set would -/// otherwise be empty. +/// Compute the set of attestation subnets this node subscribes to, per +/// leanSpec (`src/lean_spec/__main__.py`): every validator subscribes to its +/// own subnet (`vid % committee_count`) for mesh health; aggregators +/// additionally subscribe to explicit `aggregate_subnet_ids` and fall back to +/// subnet 0 when the set would otherwise be empty. /// /// Evaluated once at startup — runtime aggregator toggles do not resubscribe /// (hot-standby model); see the invariant note on [`SwarmConfig`]. /// /// A zero `attestation_committee_count` yields no validator subnets (rather /// than dividing by zero); callers enforce `>= 1`, so this is a defensive -/// guard on the public API, not a reachable path in the binary. -pub fn compute_subscription_subnets( +/// guard, not a reachable path in the binary. +pub(crate) fn compute_subscription_subnets( validator_ids: &[u64], attestation_committee_count: u64, is_aggregator: bool, From ac628dacef25b015fe971aa6426b6d1bf1bf1623 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 17:26:34 -0300 Subject: [PATCH 21/33] refactor(blockchain): read genesis time and early-agg threshold at runtime Drop the cached genesis_time_ms and early_aggregation_min_group_sigs fields from BlockChainServer. Genesis time is read via store.config() at each use site (matching the tick paths); the early-aggregation threshold is computed on demand from head_state, reached only inside the early window and only until a session starts. --- crates/blockchain/src/lib.rs | 59 +++++++++++++++--------------------- 1 file changed, 25 insertions(+), 34 deletions(-) diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 5f8e5e3e..0df433a1 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -99,18 +99,6 @@ impl BlockChain { (now_ms.saturating_sub(genesis_time * 1000) / MILLISECONDS_PER_SLOT) as u32; key_manager.advance_keys_to(current_slot); - // Minimum single-group signature count that triggers early - // aggregation: two-thirds of one committee's expected votes, - // `2 * N / (3 * C)`. Closed form (no per-validator subnet scan), - // computed once; assumes an even committee split and one committee's - // worth of votes per group. - let validator_count = store.head_state().validators.len() as u64; - let early_aggregation_min_group_sigs = if attestation_committee_count == 0 { - 0 - } else { - (2 * validator_count / (3 * attestation_committee_count)) as usize - }; - let handle = BlockChainServer { store, p2p: None, @@ -121,8 +109,6 @@ impl BlockChain { current_aggregation: None, last_tick_instant: None, attestation_committee_count, - genesis_time_ms: genesis_time * 1000, - early_aggregation_min_group_sigs, pending_aggregate_publishes: Vec::new(), proposer_config, pre_merge_coverage: None, @@ -184,22 +170,10 @@ pub struct BlockChainServer { last_tick_instant: Option, /// Number of attestation committees (= subnet count). Used by the - /// attestation aggregate coverage emission. + /// attestation aggregate coverage emission and the early-aggregation + /// threshold. attestation_committee_count: u64, - /// Genesis time in milliseconds, cached at spawn. `store.config()` is an - /// uncached backend read, too heavy for the per-gossip-insert early - /// checks that need this. The tick paths still recompute it from - /// `store.config()` — deliberately left alone; prefer this field in new - /// code. - genesis_time_ms: u64, - - /// Minimum signatures in one attestation-data group that trigger early - /// aggregation: two-thirds of a committee's expected votes, - /// `2 * validator_count / (3 * committee_count)`. Computed once at spawn - /// (the validator registry is static). - early_aggregation_min_group_sigs: usize, - /// Aggregates produced before the interval-2 boundary, held back so they /// are not gossiped early. Flushed by `FlushAggregatePublishes` at the /// boundary; only ever holds the current slot's aggregates. @@ -459,7 +433,8 @@ impl BlockChainServer { "Dropping stale pending aggregate publishes" ); } - let t2_ms = aggregation::interval2_boundary_ms(self.genesis_time_ms, slot); + let genesis_time_ms = self.store.config().genesis_time * 1000; + let t2_ms = aggregation::interval2_boundary_ms(genesis_time_ms, slot); let now_ms = unix_now_ms(); let early = now_ms < t2_ms; if early { @@ -518,8 +493,8 @@ impl BlockChainServer { if !self.aggregator.is_enabled() { return; } - let Some(slot) = aggregation::early_aggregation_slot(unix_now_ms(), self.genesis_time_ms) - else { + let genesis_time_ms = self.store.config().genesis_time * 1000; + let Some(slot) = aggregation::early_aggregation_slot(unix_now_ms(), genesis_time_ms) else { return; }; if self @@ -530,18 +505,33 @@ impl BlockChainServer { return; } let max_group = self.store.max_gossip_group_count_for_slot(slot); - if !aggregation::early_threshold_met(max_group, self.early_aggregation_min_group_sigs) { + let min_group_sigs = self.early_aggregation_min_group_sigs(); + if !aggregation::early_threshold_met(max_group, min_group_sigs) { return; } info!( %slot, max_group, - min_group_sigs = self.early_aggregation_min_group_sigs, + min_group_sigs, "Early-aggregation threshold met" ); self.start_aggregation_session(slot, ctx).await; } + /// Minimum signatures in one attestation-data group that trigger early + /// aggregation: two-thirds of one committee's expected votes, + /// `2 * validator_count / (3 * committee_count)`. Computed on demand rather + /// than cached; only reached inside the early-aggregation window and only + /// until a session starts, so the `head_state` read (memoized on the + /// stable head root) runs a handful of times per slot at most. + fn early_aggregation_min_group_sigs(&self) -> usize { + if self.attestation_committee_count == 0 { + return 0; + } + let validator_count = self.store.head_state().validators.len() as u64; + (2 * validator_count / (3 * self.attestation_committee_count)) as usize + } + /// Returns the validator ID if any of our validators is the proposer for this slot. fn get_our_proposer(&self, slot: u64) -> Option { let head_state = self.store.head_state(); @@ -1208,7 +1198,8 @@ impl Handler for BlockChainServer { // Publish alignment: hold back aggregates produced before this slot's // interval-2 boundary; `FlushAggregatePublishes` publishes them at T2. // (`session_id` is the session's slot.) - let t2_ms = aggregation::interval2_boundary_ms(self.genesis_time_ms, msg.session_id); + let genesis_time_ms = self.store.config().genesis_time * 1000; + let t2_ms = aggregation::interval2_boundary_ms(genesis_time_ms, msg.session_id); if unix_now_ms() < t2_ms { self.pending_aggregate_publishes.push(aggregate); return; From ed61bdcd569bcf13fdcb38cde2c4fa00826c65e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 18:21:21 -0300 Subject: [PATCH 22/33] refactor(blockchain): delay early aggregates in the worker, drop FlushAggregatePublishes Move publish alignment from the actor into the aggregation worker. The worker now receives the interval-2 boundary timestamp and, for each aggregate it produces before that boundary, uses send_after to delay the AggregateProduced message until the boundary (sending immediately if already past it). This removes the FlushAggregatePublishes self-message, the pending_aggregate_publishes buffer, and the T2 check in the AggregateProduced handler. Delivery timers are tied to the actor's cancellation token via Context::from_ref, so they drop on shutdown just like the old buffer did. --- crates/blockchain/src/aggregation.rs | 49 +++++++++++-------- crates/blockchain/src/lib.rs | 72 ++++------------------------ 2 files changed, 37 insertions(+), 84 deletions(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index 80b2b259..2e113ece 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -21,7 +21,7 @@ use ethlambda_types::{ state::Validator, }; use spawned_concurrency::message::Message; -use spawned_concurrency::tasks::ActorRef; +use spawned_concurrency::tasks::{ActorRef, Context, send_after}; use tokio_util::sync::CancellationToken; use tracing::{info, warn}; @@ -165,19 +165,6 @@ impl Message for AggregationDeadline { type Result = (); } -/// Self-message scheduled when a session starts early; fires at the -/// interval-2 boundary and publishes any aggregates held back by the -/// publish-alignment rule (aggregates must not reach gossip before -/// interval 2). Carries the session slot so a timer delayed past the next -/// session start is fenced (like [`AggregationDeadline`]) and cannot flush -/// the newer session's buffer before its own boundary. -pub(crate) struct FlushAggregatePublishes { - pub(crate) slot: u64, -} -impl Message for FlushAggregatePublishes { - type Result = (); -} - /// One-shot self-message scheduled at the interval-1 tick; fires when the /// early-aggregation window opens (T2 - EARLY_AGGREGATION_WINDOW_MS) to run /// the threshold check for signatures that all arrived before the window. @@ -572,11 +559,19 @@ pub(crate) fn aggregation_bits_from_validator_indices(bits: &[u64]) -> Aggregati /// Pulls jobs from the snapshot, runs [`aggregate_job`] for each, and streams /// successful aggregates back to the actor as [`AggregateProduced`] messages. /// Emits [`AggregationDone`] when the loop exits (completion or cancellation). +/// +/// Publish alignment: aggregates must not reach the actor (and thus gossip) +/// before the interval-2 boundary. `publish_at_ms` is that boundary as a UNIX +/// millisecond timestamp; a produced aggregate whose wall clock is still before +/// it is delivered via [`send_after`] timed to land at the boundary, otherwise +/// it is sent immediately. A normal interval-2 session starts at the boundary, +/// so its aggregates are always past it and sent without delay. pub(crate) fn run_aggregation_worker( snapshot: AggregationSnapshot, actor: ActorRef, cancel: CancellationToken, session_id: u64, + publish_at_ms: u64, ) { let start = Instant::now(); let groups_considered = snapshot.groups_considered; @@ -624,12 +619,26 @@ pub(crate) fn run_aggregation_worker( total_raw_sigs += raw_sigs; total_children += children; - if actor - .send(AggregateProduced { session_id, output }) - .is_err() - { - // Actor is gone; no point producing more. - break; + // Hold the aggregate until the interval-2 boundary (early session), or + // send now if already past it. `send_after` is fire-and-forget: it + // spawns a timer that delivers the message and is cancelled only if the + // actor stops, so the produced aggregate is not lost when the worker's + // own loop ends. + let now_ms = crate::unix_now_ms(); + if now_ms >= publish_at_ms { + if actor + .send(AggregateProduced { session_id, output }) + .is_err() + { + // Actor is gone; no point producing more. + break; + } + } else { + send_after( + Duration::from_millis(publish_at_ms - now_ms), + Context::from_ref(&actor), + AggregateProduced { session_id, output }, + ); } } diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 0df433a1..448a115e 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -16,7 +16,7 @@ use ethlambda_types::{ use crate::aggregation::{ AGGREGATION_DEADLINE, AggregateProduced, AggregationDeadline, AggregationDone, AggregationSession, EARLY_AGGREGATION_WINDOW_MS, EarlyAggregationCheck, - FlushAggregatePublishes, PRIOR_WORKER_JOIN_TIMEOUT, run_aggregation_worker, + PRIOR_WORKER_JOIN_TIMEOUT, run_aggregation_worker, }; use crate::key_manager::ValidatorKeyPair; use crate::sync_status::SyncStatusTracker; @@ -109,7 +109,6 @@ impl BlockChain { current_aggregation: None, last_tick_instant: None, attestation_committee_count, - pending_aggregate_publishes: Vec::new(), proposer_config, pre_merge_coverage: None, sync_status: SyncStatusTracker::new(gate_duties), @@ -174,11 +173,6 @@ pub struct BlockChainServer { /// threshold. attestation_committee_count: u64, - /// Aggregates produced before the interval-2 boundary, held back so they - /// are not gossiped early. Flushed by `FlushAggregatePublishes` at the - /// boundary; only ever holds the current slot's aggregates. - pending_aggregate_publishes: Vec, - /// Proposer-side block-building policy proposer_config: ProposerConfig, @@ -421,26 +415,14 @@ impl BlockChainServer { }; let session_id = slot; - // Any leftovers from a prior slot mean its flush never fired (a - // backwards wall-clock step, or a flush timer delayed past the next - // session). Drop them — late aggregates are dropped, same policy as - // signatures that miss the snapshot. - let stale = std::mem::take(&mut self.pending_aggregate_publishes); - if !stale.is_empty() { - warn!( - %slot, - count = stale.len(), - "Dropping stale pending aggregate publishes" - ); - } let genesis_time_ms = self.store.config().genesis_time * 1000; let t2_ms = aggregation::interval2_boundary_ms(genesis_time_ms, slot); let now_ms = unix_now_ms(); let early = now_ms < t2_ms; if early { - // Publish alignment: aggregates must not reach gossip before the - // interval-2 boundary. Aggregates produced before T2 are buffered - // in `pending_aggregate_publishes`; this timer flushes them at T2. + // Publish alignment lives in the worker: it holds each produced + // aggregate until `t2_ms` (the interval-2 boundary) before sending + // it back, so nothing reaches gossip early. let lead = Duration::from_millis(t2_ms - now_ms); metrics::inc_aggregation_early_starts(); metrics::observe_aggregation_early_start_lead(lead); @@ -449,7 +431,6 @@ impl BlockChainServer { lead_ms = lead.as_millis() as u64, "Starting aggregation session early" ); - send_after(lead, ctx.clone(), FlushAggregatePublishes { slot }); } // Independent token per session. Shutdown propagates via our @@ -461,7 +442,7 @@ impl BlockChainServer { let worker_cancel = cancel.clone(); let worker_actor = actor_ref.clone(); let worker = tokio::task::spawn_blocking(move || { - run_aggregation_worker(snapshot, worker_actor, worker_cancel, session_id); + run_aggregation_worker(snapshot, worker_actor, worker_cancel, session_id, t2_ms); }); let _deadline_timer = send_after( @@ -1188,56 +1169,19 @@ impl Handler for BlockChainServer { return; } + // Publish alignment is enforced upstream: the worker delays delivery of + // this message until the interval-2 boundary, so by the time it lands + // the aggregate is safe to apply and gossip immediately. aggregation::apply_aggregated_group(&mut self.store, &msg.output); let aggregate = SignedAggregatedAttestation { data: msg.output.hashed.data().clone(), proof: msg.output.proof, }; - - // Publish alignment: hold back aggregates produced before this slot's - // interval-2 boundary; `FlushAggregatePublishes` publishes them at T2. - // (`session_id` is the session's slot.) - let genesis_time_ms = self.store.config().genesis_time * 1000; - let t2_ms = aggregation::interval2_boundary_ms(genesis_time_ms, msg.session_id); - if unix_now_ms() < t2_ms { - self.pending_aggregate_publishes.push(aggregate); - return; - } self.publish_aggregate(aggregate); } } -impl Handler for BlockChainServer { - async fn handle(&mut self, msg: FlushAggregatePublishes, _ctx: &Context) { - // Fence like `AggregationDeadline`: only the current session's flush - // drains the buffer. A flush timer delayed past the next session start - // would otherwise publish that newer session's buffered aggregates - // before its own interval-2 boundary, breaking publish alignment. The - // superseded session's own aggregates were already dropped by the - // stale-drain in `start_aggregation_session`. - let is_current = self - .current_aggregation - .as_ref() - .is_some_and(|session| session.session_id == msg.slot); - if !is_current { - return; - } - let pending = std::mem::take(&mut self.pending_aggregate_publishes); - if pending.is_empty() { - return; - } - info!( - slot = msg.slot, - count = pending.len(), - "Publishing aggregates held back until the interval-2 boundary" - ); - for aggregate in pending { - self.publish_aggregate(aggregate); - } - } -} - impl Handler for BlockChainServer { async fn handle(&mut self, _msg: EarlyAggregationCheck, ctx: &Context) { self.maybe_start_early_aggregation(ctx).await; From 4ae261b7263bd10cd4fbe2cfb8cf7ea405fa82a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 18:26:15 -0300 Subject: [PATCH 23/33] refactor(blockchain): pass worker publish time as SystemTime Type the aggregation worker's publish boundary as a SystemTime instead of a raw u64 millisecond timestamp. The worker derives each aggregate's delay via duration_since(SystemTime::now()), whose Err branch is exactly the send-immediately (past-boundary) case. SystemTime (not Instant) because the boundary is derived from genesis wall-clock time. --- crates/blockchain/src/aggregation.rs | 47 +++++++++++++++------------- crates/blockchain/src/lib.rs | 15 ++++++--- 2 files changed, 36 insertions(+), 26 deletions(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index 2e113ece..733ef002 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -9,7 +9,7 @@ //! results back as [`AggregateProduced`] / [`AggregationDone`] messages. use std::collections::HashSet; -use std::time::{Duration, Instant}; +use std::time::{Duration, Instant, SystemTime}; use ethlambda_crypto::aggregate_mixed; use ethlambda_storage::Store; @@ -561,17 +561,17 @@ pub(crate) fn aggregation_bits_from_validator_indices(bits: &[u64]) -> Aggregati /// Emits [`AggregationDone`] when the loop exits (completion or cancellation). /// /// Publish alignment: aggregates must not reach the actor (and thus gossip) -/// before the interval-2 boundary. `publish_at_ms` is that boundary as a UNIX -/// millisecond timestamp; a produced aggregate whose wall clock is still before -/// it is delivered via [`send_after`] timed to land at the boundary, otherwise -/// it is sent immediately. A normal interval-2 session starts at the boundary, -/// so its aggregates are always past it and sent without delay. +/// before the interval-2 boundary. `publish_at` is that boundary as a wall-clock +/// instant; a produced aggregate still ahead of it is delivered via +/// [`send_after`] timed to land at the boundary, otherwise it is sent +/// immediately. A normal interval-2 session starts at the boundary, so its +/// aggregates are always past it and sent without delay. pub(crate) fn run_aggregation_worker( snapshot: AggregationSnapshot, actor: ActorRef, cancel: CancellationToken, session_id: u64, - publish_at_ms: u64, + publish_at: SystemTime, ) { let start = Instant::now(); let groups_considered = snapshot.groups_considered; @@ -623,22 +623,25 @@ pub(crate) fn run_aggregation_worker( // send now if already past it. `send_after` is fire-and-forget: it // spawns a timer that delivers the message and is cancelled only if the // actor stops, so the produced aggregate is not lost when the worker's - // own loop ends. - let now_ms = crate::unix_now_ms(); - if now_ms >= publish_at_ms { - if actor - .send(AggregateProduced { session_id, output }) - .is_err() - { - // Actor is gone; no point producing more. - break; + // own loop ends. `duration_since` errs once the boundary has passed, + // which is exactly the send-immediately case. + match publish_at.duration_since(SystemTime::now()) { + Ok(delay) if !delay.is_zero() => { + send_after( + delay, + Context::from_ref(&actor), + AggregateProduced { session_id, output }, + ); + } + _ => { + if actor + .send(AggregateProduced { session_id, output }) + .is_err() + { + // Actor is gone; no point producing more. + break; + } } - } else { - send_after( - Duration::from_millis(publish_at_ms - now_ms), - Context::from_ref(&actor), - AggregateProduced { session_id, output }, - ); } } diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 448a115e..591c2565 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -417,12 +417,13 @@ impl BlockChainServer { let session_id = slot; let genesis_time_ms = self.store.config().genesis_time * 1000; let t2_ms = aggregation::interval2_boundary_ms(genesis_time_ms, slot); + // Interval-2 boundary as a wall-clock instant; the worker holds each + // produced aggregate until this before sending it back, so nothing + // reaches gossip early. + let publish_at = SystemTime::UNIX_EPOCH + Duration::from_millis(t2_ms); let now_ms = unix_now_ms(); let early = now_ms < t2_ms; if early { - // Publish alignment lives in the worker: it holds each produced - // aggregate until `t2_ms` (the interval-2 boundary) before sending - // it back, so nothing reaches gossip early. let lead = Duration::from_millis(t2_ms - now_ms); metrics::inc_aggregation_early_starts(); metrics::observe_aggregation_early_start_lead(lead); @@ -442,7 +443,13 @@ impl BlockChainServer { let worker_cancel = cancel.clone(); let worker_actor = actor_ref.clone(); let worker = tokio::task::spawn_blocking(move || { - run_aggregation_worker(snapshot, worker_actor, worker_cancel, session_id, t2_ms); + run_aggregation_worker( + snapshot, + worker_actor, + worker_cancel, + session_id, + publish_at, + ); }); let _deadline_timer = send_after( From df18f775dcfe9c905a23c101293cd8ede2154a04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 18:34:16 -0300 Subject: [PATCH 24/33] refactor: inline single-use early-aggregation helpers; use if-else in worker Inline five helpers that ended up with a single caller: compute_subscription_subnets (back into build_swarm), interval2_boundary_ms and early_threshold_met (into their call sites), the early_aggregation_min_group_sigs method (into the trigger), and publish_aggregate (into the AggregateProduced handler). Keep early_aggregation_slot (non-trivial window math, reads clearer named) and the Store/GossipSignatureBuffer delegation pair (the file's encapsulation pattern). Also replace the worker's match on duration_since with an if-else on the unwrapped delay. --- crates/blockchain/src/aggregation.rs | 53 ++++++++-------------- crates/blockchain/src/lib.rs | 53 ++++++++++------------ crates/net/p2p/src/lib.rs | 66 +++++++++------------------- 3 files changed, 62 insertions(+), 110 deletions(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index 733ef002..a3f242dd 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -42,7 +42,7 @@ pub(crate) const PRIOR_WORKER_JOIN_TIMEOUT: Duration = Duration::from_secs(2); /// Width of the early-aggregation window: a session may start at most this /// long before the interval-2 boundary, provided the signature threshold is -/// met (see `early_threshold_met`). +/// met (see the check in `maybe_start_early_aggregation`). pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 600; // The window must fit within one interval: `early_aggregation_slot` subtracts @@ -55,12 +55,6 @@ const _: () = assert!( "EARLY_AGGREGATION_WINDOW_MS must not exceed one interval" ); -/// Wall-clock millisecond timestamp of `slot`'s interval-2 boundary (the -/// normal aggregation start). -pub(crate) fn interval2_boundary_ms(genesis_time_ms: u64, slot: u64) -> u64 { - genesis_time_ms + slot * MILLISECONDS_PER_SLOT + 2 * MILLISECONDS_PER_INTERVAL -} - /// If `now_ms` falls inside some slot's early-aggregation window /// (`[T2 - EARLY_AGGREGATION_WINDOW_MS, T2)` with `T2` that slot's interval-2 /// boundary), return that slot. @@ -73,15 +67,6 @@ pub(crate) fn early_aggregation_slot(now_ms: u64, genesis_time_ms: u64) -> Optio in_window.then_some(since_genesis / MILLISECONDS_PER_SLOT) } -/// Early-start threshold: the largest single attestation-data group already -/// holds at least `min_group_sigs` signatures. `min_group_sigs` is the -/// precomputed two-thirds-of-a-committee count (see `BlockChain::spawn`), so -/// the 2/3 fraction is not re-applied here. At most one group per slot can -/// satisfy this (each validator signs once per slot). -pub(crate) fn early_threshold_met(max_group_count: usize, min_group_sigs: usize) -> bool { - min_group_sigs > 0 && max_group_count >= min_group_sigs -} - /// A single pre-prepared aggregation group. /// /// Built on the actor thread from a store snapshot; consumed by an off-thread @@ -620,28 +605,28 @@ pub(crate) fn run_aggregation_worker( total_children += children; // Hold the aggregate until the interval-2 boundary (early session), or - // send now if already past it. `send_after` is fire-and-forget: it + // send now if already at/past it. `send_after` is fire-and-forget: it // spawns a timer that delivers the message and is cancelled only if the // actor stops, so the produced aggregate is not lost when the worker's // own loop ends. `duration_since` errs once the boundary has passed, - // which is exactly the send-immediately case. - match publish_at.duration_since(SystemTime::now()) { - Ok(delay) if !delay.is_zero() => { - send_after( - delay, - Context::from_ref(&actor), - AggregateProduced { session_id, output }, - ); - } - _ => { - if actor - .send(AggregateProduced { session_id, output }) - .is_err() - { - // Actor is gone; no point producing more. - break; - } + // which collapses to a zero delay here. + let delay = publish_at + .duration_since(SystemTime::now()) + .unwrap_or(Duration::ZERO); + if delay.is_zero() { + if actor + .send(AggregateProduced { session_id, output }) + .is_err() + { + // Actor is gone; no point producing more. + break; } + } else { + send_after( + delay, + Context::from_ref(&actor), + AggregateProduced { session_id, output }, + ); } } diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 591c2565..a24d2758 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -416,7 +416,7 @@ impl BlockChainServer { let session_id = slot; let genesis_time_ms = self.store.config().genesis_time * 1000; - let t2_ms = aggregation::interval2_boundary_ms(genesis_time_ms, slot); + let t2_ms = genesis_time_ms + slot * MILLISECONDS_PER_SLOT + 2 * MILLISECONDS_PER_INTERVAL; // Interval-2 boundary as a wall-clock instant; the worker holds each // produced aggregate until this before sending it back, so nothing // reaches gossip early. @@ -493,8 +493,18 @@ impl BlockChainServer { return; } let max_group = self.store.max_gossip_group_count_for_slot(slot); - let min_group_sigs = self.early_aggregation_min_group_sigs(); - if !aggregation::early_threshold_met(max_group, min_group_sigs) { + // Trigger once the largest current-slot group holds two-thirds of one + // committee's expected votes, `2 * N / (3 * C)` (0 when there is no + // committee, which never triggers). The head-state read is memoized on + // the stable head root and only runs inside the window until a session + // starts, so it costs a handful of reads per slot at most. + let min_group_sigs = if self.attestation_committee_count == 0 { + 0 + } else { + let validator_count = self.store.head_state().validators.len() as u64; + (2 * validator_count / (3 * self.attestation_committee_count)) as usize + }; + if min_group_sigs == 0 || max_group < min_group_sigs { return; } info!( @@ -506,20 +516,6 @@ impl BlockChainServer { self.start_aggregation_session(slot, ctx).await; } - /// Minimum signatures in one attestation-data group that trigger early - /// aggregation: two-thirds of one committee's expected votes, - /// `2 * validator_count / (3 * committee_count)`. Computed on demand rather - /// than cached; only reached inside the early-aggregation window and only - /// until a session starts, so the `head_state` read (memoized on the - /// stable head root) runs a handful of times per slot at most. - fn early_aggregation_min_group_sigs(&self) -> usize { - if self.attestation_committee_count == 0 { - return 0; - } - let validator_count = self.store.head_state().validators.len() as u64; - (2 * validator_count / (3 * self.attestation_committee_count)) as usize - } - /// Returns the validator ID if any of our validators is the proposer for this slot. fn get_our_proposer(&self, slot: u64) -> Option { let head_state = self.store.head_state(); @@ -1030,15 +1026,6 @@ impl BlockChainServer { } } - /// Publish an aggregated attestation to the aggregation gossip topic. - fn publish_aggregate(&self, aggregate: SignedAggregatedAttestation) { - if let Some(ref p2p) = self.p2p { - let _ = p2p - .publish_aggregated_attestation(aggregate) - .inspect_err(|err| error!(%err, "Failed to publish aggregated attestation")); - } - } - fn on_gossip_attestation(&mut self, attestation: &SignedAttestation) { // Read fresh here too: a gossip event can arrive between ticks, and // if the admin API just toggled, the first gossip after the toggle @@ -1181,11 +1168,15 @@ impl Handler for BlockChainServer { // the aggregate is safe to apply and gossip immediately. aggregation::apply_aggregated_group(&mut self.store, &msg.output); - let aggregate = SignedAggregatedAttestation { - data: msg.output.hashed.data().clone(), - proof: msg.output.proof, - }; - self.publish_aggregate(aggregate); + if let Some(ref p2p) = self.p2p { + let aggregate = SignedAggregatedAttestation { + data: msg.output.hashed.data().clone(), + proof: msg.output.proof, + }; + let _ = p2p + .publish_aggregated_attestation(aggregate) + .inspect_err(|err| error!(%err, "Failed to publish aggregated attestation")); + } } } diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index d6a3509c..51ce011d 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -182,43 +182,6 @@ pub struct BuiltSwarm { pub(crate) bootnode_addrs: HashMap, } -/// Compute the set of attestation subnets this node subscribes to, per -/// leanSpec (`src/lean_spec/__main__.py`): every validator subscribes to its -/// own subnet (`vid % committee_count`) for mesh health; aggregators -/// additionally subscribe to explicit `aggregate_subnet_ids` and fall back to -/// subnet 0 when the set would otherwise be empty. -/// -/// Evaluated once at startup — runtime aggregator toggles do not resubscribe -/// (hot-standby model); see the invariant note on [`SwarmConfig`]. -/// -/// A zero `attestation_committee_count` yields no validator subnets (rather -/// than dividing by zero); callers enforce `>= 1`, so this is a defensive -/// guard, not a reachable path in the binary. -pub(crate) fn compute_subscription_subnets( - validator_ids: &[u64], - attestation_committee_count: u64, - is_aggregator: bool, - aggregate_subnet_ids: Option<&[u64]>, -) -> HashSet { - let mut subnets: HashSet = if attestation_committee_count == 0 { - HashSet::new() - } else { - validator_ids - .iter() - .map(|vid| vid % attestation_committee_count) - .collect() - }; - if is_aggregator { - if let Some(explicit_ids) = aggregate_subnet_ids { - subnets.extend(explicit_ids); - } - if subnets.is_empty() { - subnets.insert(0); - } - } - subnets -} - /// Build and configure the libp2p swarm, dial bootnodes, subscribe to topics. pub fn build_swarm( config: SwarmConfig, @@ -338,8 +301,14 @@ pub fn build_swarm( .subscribe(&aggregation_topic) .unwrap(); - // Subscribe to attestation subnets — see `compute_subscription_subnets`. - // The committee metric should reflect validator membership only, not + // Subscribe to attestation subnets per leanSpec (`src/lean_spec/__main__.py`): + // every validator subscribes to its own subnet (`vid % committee_count`) + // for mesh health; aggregators additionally subscribe to explicit + // `aggregate_subnet_ids` and fall back to subnet 0 when they have no + // validators of their own. Frozen at startup — runtime aggregator toggles + // do not resubscribe (hot-standby model); see the invariant on `SwarmConfig`. + // + // The committee metric reflects validator membership only, not // aggregator-only subscriptions. let metric_subnet = config .validator_ids @@ -349,12 +318,19 @@ pub fn build_swarm( .unwrap_or(0); metrics::set_attestation_committee_subnet(metric_subnet); - let subscription_subnets = compute_subscription_subnets( - &config.validator_ids, - config.attestation_committee_count, - config.is_aggregator, - config.aggregate_subnet_ids.as_deref(), - ); + let mut subscription_subnets: HashSet = config + .validator_ids + .iter() + .map(|vid| vid % config.attestation_committee_count) + .collect(); + if config.is_aggregator { + if let Some(ref explicit_ids) = config.aggregate_subnet_ids { + subscription_subnets.extend(explicit_ids); + } + if subscription_subnets.is_empty() { + subscription_subnets.insert(0); + } + } let mut attestation_topics: HashMap = HashMap::new(); for &subnet_id in &subscription_subnets { From 61d5490805ccd0b5ae804201e741598939f33ed1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 18:39:17 -0300 Subject: [PATCH 25/33] refactor(blockchain): inline early_aggregation_slot into the trigger Fold the window-detection math into maybe_start_early_aggregation (its only caller) and drop the helper. Update the two doc comments that referenced it. --- crates/blockchain/src/aggregation.rs | 28 ++++++++-------------------- crates/blockchain/src/lib.rs | 15 +++++++++++---- 2 files changed, 19 insertions(+), 24 deletions(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index a3f242dd..bc91a0e7 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -25,15 +25,15 @@ use spawned_concurrency::tasks::{ActorRef, Context, send_after}; use tokio_util::sync::CancellationToken; use tracing::{info, warn}; -use crate::{MILLISECONDS_PER_INTERVAL, MILLISECONDS_PER_SLOT, metrics}; +use crate::{MILLISECONDS_PER_INTERVAL, metrics}; /// Soft deadline for committee-signature aggregation measured from session /// start. After this much wall time elapses, the actor signals the worker to /// stop via its cancellation token. A session started exactly at interval 2 /// gets the full interval (interval 3 is one interval later); a session -/// started early (see `early_aggregation_slot`) ends correspondingly earlier. -/// The deadline only stops new jobs from starting — a job mid-proof finishes -/// and publishes right after. +/// started early (see `maybe_start_early_aggregation`) ends correspondingly +/// earlier. The deadline only stops new jobs from starting — a job mid-proof +/// finishes and publishes right after. pub(crate) const AGGREGATION_DEADLINE: Duration = Duration::from_millis(800); /// Upper bound we wait for a prior worker to exit if it is still running when /// the next session is about to start. Reached only in pathological cases @@ -45,28 +45,16 @@ pub(crate) const PRIOR_WORKER_JOIN_TIMEOUT: Duration = Duration::from_secs(2); /// met (see the check in `maybe_start_early_aggregation`). pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 600; -// The window must fit within one interval: `early_aggregation_slot` subtracts -// it from the interval-2 offset, and the interval-1 tick schedules the check -// at `MILLISECONDS_PER_INTERVAL - EARLY_AGGREGATION_WINDOW_MS`. Keep this -// invariant self-enforcing so a future bump to the window can't silently +// The window must fit within one interval: `maybe_start_early_aggregation` +// subtracts it from the interval-2 offset, and the interval-1 tick schedules +// the check at `MILLISECONDS_PER_INTERVAL - EARLY_AGGREGATION_WINDOW_MS`. Keep +// this invariant self-enforcing so a future bump to the window can't silently // underflow either subtraction. const _: () = assert!( EARLY_AGGREGATION_WINDOW_MS <= MILLISECONDS_PER_INTERVAL, "EARLY_AGGREGATION_WINDOW_MS must not exceed one interval" ); -/// If `now_ms` falls inside some slot's early-aggregation window -/// (`[T2 - EARLY_AGGREGATION_WINDOW_MS, T2)` with `T2` that slot's interval-2 -/// boundary), return that slot. -pub(crate) fn early_aggregation_slot(now_ms: u64, genesis_time_ms: u64) -> Option { - let since_genesis = now_ms.checked_sub(genesis_time_ms)?; - let ms_into_slot = since_genesis % MILLISECONDS_PER_SLOT; - let t2_offset = 2 * MILLISECONDS_PER_INTERVAL; - let in_window = - ms_into_slot >= t2_offset - EARLY_AGGREGATION_WINDOW_MS && ms_into_slot < t2_offset; - in_window.then_some(since_genesis / MILLISECONDS_PER_SLOT) -} - /// A single pre-prepared aggregation group. /// /// Built on the actor thread from a store snapshot; consumed by an off-thread diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index a24d2758..b884a62d 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -481,10 +481,19 @@ impl BlockChainServer { if !self.aggregator.is_enabled() { return; } + // Only fire inside the early-aggregation window + // `[T2 - EARLY_AGGREGATION_WINDOW_MS, T2)`, where T2 is the current + // slot's interval-2 boundary; the slot is derived from the wall clock. let genesis_time_ms = self.store.config().genesis_time * 1000; - let Some(slot) = aggregation::early_aggregation_slot(unix_now_ms(), genesis_time_ms) else { + let Some(ms_since_genesis) = unix_now_ms().checked_sub(genesis_time_ms) else { return; }; + let ms_into_slot = ms_since_genesis % MILLISECONDS_PER_SLOT; + let t2_offset = 2 * MILLISECONDS_PER_INTERVAL; + if ms_into_slot < t2_offset - EARLY_AGGREGATION_WINDOW_MS || ms_into_slot >= t2_offset { + return; + } + let slot = ms_since_genesis / MILLISECONDS_PER_SLOT; if self .current_aggregation .as_ref() @@ -495,9 +504,7 @@ impl BlockChainServer { let max_group = self.store.max_gossip_group_count_for_slot(slot); // Trigger once the largest current-slot group holds two-thirds of one // committee's expected votes, `2 * N / (3 * C)` (0 when there is no - // committee, which never triggers). The head-state read is memoized on - // the stable head root and only runs inside the window until a session - // starts, so it costs a handful of reads per slot at most. + // committee, which never triggers). let min_group_sigs = if self.attestation_committee_count == 0 { 0 } else { From 3c1a4395425763adacc3176ad2511f5948ac6778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 18:50:58 -0300 Subject: [PATCH 26/33] feat(blockchain): gate early-aggregation check on current-slot attestations A late- or future-slot attestation cannot advance the current slot's gossip group counts, so running the early-aggregation threshold check for it is wasted work. Gate the attestation-driven call on the attestation being for the store's current slot (store.time() / INTERVALS_PER_SLOT). The window-opening timer (EarlyAggregationCheck) still fires regardless. --- crates/blockchain/src/lib.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index b884a62d..9a701c6d 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -470,7 +470,7 @@ impl BlockChainServer { /// interval-2 tick when, inside the window `[T2 - EARLY_AGGREGATION_WINDOW_MS, T2)`, /// a single attestation-data group already holds 2/3 of the signatures /// expected from this node's aggregation subnets. Called after every - /// stored gossip signature and once at the window opening via + /// stored current-slot gossip signature and once at the window opening via /// [`EarlyAggregationCheck`]. Fires at most once per slot: the started /// session stays in `current_aggregation` (running or finished) until the /// next session replaces it. The latch has one hole: if the snapshot @@ -1141,7 +1141,13 @@ impl Handler for BlockChainServer { impl Handler for BlockChainServer { async fn handle(&mut self, msg: NewAttestation, ctx: &Context) { self.on_gossip_attestation(&msg.attestation); - self.maybe_start_early_aggregation(ctx).await; + // Early aggregation only advances the current slot's group counts, so a + // late- or future-slot attestation can never cross the threshold; skip + // the check unless this attestation is for the store's current slot. + let current_slot = self.store.time() / INTERVALS_PER_SLOT; + if msg.attestation.data.slot == current_slot { + self.maybe_start_early_aggregation(ctx).await; + } } } From 49bfc8896ce136734cd6aa39a7dd82a2ec3f3f21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 18:54:19 -0300 Subject: [PATCH 27/33] revert(p2p): drop no-op build_swarm refactor churn The subnet-computation reorganization in build_swarm was leftover churn from the earlier extract/re-inline of compute_subscription_subnets and produced no behavioral change (min over the deduped set equals min over the mapped iterator; the subscription set collects identically). Restore main's single-pass version to keep this PR focused on early aggregation. --- crates/net/p2p/src/lib.rs | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index 51ce011d..de34e469 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -302,31 +302,27 @@ pub fn build_swarm( .unwrap(); // Subscribe to attestation subnets per leanSpec (`src/lean_spec/__main__.py`): - // every validator subscribes to its own subnet (`vid % committee_count`) - // for mesh health; aggregators additionally subscribe to explicit - // `aggregate_subnet_ids` and fall back to subnet 0 when they have no - // validators of their own. Frozen at startup — runtime aggregator toggles - // do not resubscribe (hot-standby model); see the invariant on `SwarmConfig`. - // - // The committee metric reflects validator membership only, not - // aggregator-only subscriptions. - let metric_subnet = config + // every validator subscribes to its own subnet for mesh health; aggregators + // additionally subscribe to explicit `aggregate_subnet_ids` and fall back to + // subnet 0 when they have no validators of their own. + let validator_subnets: HashSet = config .validator_ids .iter() .map(|vid| vid % config.attestation_committee_count) - .min() - .unwrap_or(0); + .collect(); + + // The committee metric should reflect validator membership only, not + // aggregator-only subscriptions. + let metric_subnet = validator_subnets.iter().copied().min().unwrap_or(0); metrics::set_attestation_committee_subnet(metric_subnet); - let mut subscription_subnets: HashSet = config - .validator_ids - .iter() - .map(|vid| vid % config.attestation_committee_count) - .collect(); + let mut subscription_subnets = validator_subnets; if config.is_aggregator { if let Some(ref explicit_ids) = config.aggregate_subnet_ids { subscription_subnets.extend(explicit_ids); } + // Fall back to subnet 0 only when the aggregator has no validators + // and no explicit subnets — otherwise leave the set as configured. if subscription_subnets.is_empty() { subscription_subnets.insert(0); } From 72cbf213c6dfa4d73f0286af685dfadd4c248731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 3 Jul 2026 15:40:13 -0300 Subject: [PATCH 28/33] feat(blockchain): round the early-aggregation threshold up, not down The 2/3-of-expected-votes threshold used floor division, so for counts not divisible by 3 it triggered slightly below a true 2/3. Round up with div_ceil so the early start waits for a genuine 2/3 supermajority of one committee's expected votes (ceil(2*N / (3*C))). --- crates/blockchain/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 9a701c6d..cbca5f39 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -503,13 +503,13 @@ impl BlockChainServer { } let max_group = self.store.max_gossip_group_count_for_slot(slot); // Trigger once the largest current-slot group holds two-thirds of one - // committee's expected votes, `2 * N / (3 * C)` (0 when there is no - // committee, which never triggers). + // committee's expected votes, rounded up: `ceil(2 * N / (3 * C))` + // (0 only when there are no validators, which never triggers). let min_group_sigs = if self.attestation_committee_count == 0 { 0 } else { let validator_count = self.store.head_state().validators.len() as u64; - (2 * validator_count / (3 * self.attestation_committee_count)) as usize + (2 * validator_count).div_ceil(3 * self.attestation_committee_count) as usize }; if min_group_sigs == 0 || max_group < min_group_sigs { return; From 632979d188d897640b94aea2349a76a1be67a55f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 3 Jul 2026 16:13:28 -0300 Subject: [PATCH 29/33] refactor(blockchain): type the early-aggregation window as a Duration EARLY_AGGREGATION_WINDOW_MS was a bare u64 of milliseconds; make it a Duration (EARLY_AGGREGATION_WINDOW) so its unit is carried in the type. The timer site now subtracts Durations directly; the wall-clock window bounds check converts once via as_millis(), and the compile-time fits-within-one-interval assert uses the const as_millis(). --- crates/blockchain/src/aggregation.rs | 12 ++++++------ crates/blockchain/src/lib.rs | 17 ++++++++--------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index bc91a0e7..6155946e 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -2,7 +2,7 @@ //! pure functions it runs. //! //! The blockchain actor fires one aggregation session per slot — at interval 2, -//! or up to [`EARLY_AGGREGATION_WINDOW_MS`] early when the 2/3 signature +//! or up to [`EARLY_AGGREGATION_WINDOW`] early when the 2/3 signature //! threshold is met — via //! [`run_aggregation_worker`]. The actor stays on its message loop; the worker //! runs the expensive XMSS proofs on a `spawn_blocking` thread and streams @@ -43,16 +43,16 @@ pub(crate) const PRIOR_WORKER_JOIN_TIMEOUT: Duration = Duration::from_secs(2); /// Width of the early-aggregation window: a session may start at most this /// long before the interval-2 boundary, provided the signature threshold is /// met (see the check in `maybe_start_early_aggregation`). -pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 600; +pub(crate) const EARLY_AGGREGATION_WINDOW: Duration = Duration::from_millis(600); // The window must fit within one interval: `maybe_start_early_aggregation` // subtracts it from the interval-2 offset, and the interval-1 tick schedules -// the check at `MILLISECONDS_PER_INTERVAL - EARLY_AGGREGATION_WINDOW_MS`. Keep +// the check at `MILLISECONDS_PER_INTERVAL - EARLY_AGGREGATION_WINDOW`. Keep // this invariant self-enforcing so a future bump to the window can't silently // underflow either subtraction. const _: () = assert!( - EARLY_AGGREGATION_WINDOW_MS <= MILLISECONDS_PER_INTERVAL, - "EARLY_AGGREGATION_WINDOW_MS must not exceed one interval" + EARLY_AGGREGATION_WINDOW.as_millis() <= MILLISECONDS_PER_INTERVAL as u128, + "EARLY_AGGREGATION_WINDOW must not exceed one interval" ); /// A single pre-prepared aggregation group. @@ -139,7 +139,7 @@ impl Message for AggregationDeadline { } /// One-shot self-message scheduled at the interval-1 tick; fires when the -/// early-aggregation window opens (T2 - EARLY_AGGREGATION_WINDOW_MS) to run +/// early-aggregation window opens (T2 - EARLY_AGGREGATION_WINDOW) to run /// the threshold check for signatures that all arrived before the window. /// Arrivals inside the window are checked per insert instead. pub(crate) struct EarlyAggregationCheck; diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index cbca5f39..b3b85e44 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -15,8 +15,8 @@ use ethlambda_types::{ use crate::aggregation::{ AGGREGATION_DEADLINE, AggregateProduced, AggregationDeadline, AggregationDone, - AggregationSession, EARLY_AGGREGATION_WINDOW_MS, EarlyAggregationCheck, - PRIOR_WORKER_JOIN_TIMEOUT, run_aggregation_worker, + AggregationSession, EARLY_AGGREGATION_WINDOW, EarlyAggregationCheck, PRIOR_WORKER_JOIN_TIMEOUT, + run_aggregation_worker, }; use crate::key_manager::ValidatorKeyPair; use crate::sync_status::SyncStatusTracker; @@ -313,12 +313,10 @@ impl BlockChainServer { // Schedule the early-aggregation window check. This tick is // one interval before T2, so the timer fires right as the - // window opens at T2 - EARLY_AGGREGATION_WINDOW_MS. + // window opens at T2 - EARLY_AGGREGATION_WINDOW. if is_aggregator { send_after( - Duration::from_millis( - MILLISECONDS_PER_INTERVAL - EARLY_AGGREGATION_WINDOW_MS, - ), + Duration::from_millis(MILLISECONDS_PER_INTERVAL) - EARLY_AGGREGATION_WINDOW, ctx.clone(), EarlyAggregationCheck, ); @@ -467,7 +465,7 @@ impl BlockChainServer { } /// Early-aggregation trigger: start the slot's session ahead of the - /// interval-2 tick when, inside the window `[T2 - EARLY_AGGREGATION_WINDOW_MS, T2)`, + /// interval-2 tick when, inside the window `[T2 - EARLY_AGGREGATION_WINDOW, T2)`, /// a single attestation-data group already holds 2/3 of the signatures /// expected from this node's aggregation subnets. Called after every /// stored current-slot gossip signature and once at the window opening via @@ -482,7 +480,7 @@ impl BlockChainServer { return; } // Only fire inside the early-aggregation window - // `[T2 - EARLY_AGGREGATION_WINDOW_MS, T2)`, where T2 is the current + // `[T2 - EARLY_AGGREGATION_WINDOW, T2)`, where T2 is the current // slot's interval-2 boundary; the slot is derived from the wall clock. let genesis_time_ms = self.store.config().genesis_time * 1000; let Some(ms_since_genesis) = unix_now_ms().checked_sub(genesis_time_ms) else { @@ -490,7 +488,8 @@ impl BlockChainServer { }; let ms_into_slot = ms_since_genesis % MILLISECONDS_PER_SLOT; let t2_offset = 2 * MILLISECONDS_PER_INTERVAL; - if ms_into_slot < t2_offset - EARLY_AGGREGATION_WINDOW_MS || ms_into_slot >= t2_offset { + let window_ms = EARLY_AGGREGATION_WINDOW.as_millis() as u64; + if ms_into_slot < t2_offset - window_ms || ms_into_slot >= t2_offset { return; } let slot = ms_since_genesis / MILLISECONDS_PER_SLOT; From 10aba858f3926c82f5d30d45119155c62ecb0021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Tue, 7 Jul 2026 19:35:08 -0300 Subject: [PATCH 30/33] fix(blockchain): size early-aggregation threshold by all subscribed subnets The early-aggregation trigger fires when the largest current-slot gossip group holds 2/3 of the votes it is expected to collect. Groups are keyed by attestation data, not by subnet, so a single group gathers signatures from every subnet the node subscribes to. The threshold assumed one committee's worth (ceil(2*N/(3*C))), so a node subscribed to multiple subnets tripped it after only ~2/3 of a single subnet arrived, starting aggregation far too early. Compute the subscribed-subnet set once via a shared `attestation_subscription_subnets` helper, hand the same set to both the P2P swarm (to open subscriptions) and the blockchain actor, and size the threshold from the number of network validators whose committee subnet is one of ours. --- bin/ethlambda/src/main.rs | 26 +++++++---- crates/blockchain/src/lib.rs | 35 ++++++++++++-- crates/net/p2p/src/lib.rs | 91 +++++++++++++++++++++++------------- 3 files changed, 107 insertions(+), 45 deletions(-) diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index b933b928..aeabbbbe 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -37,7 +37,9 @@ use ethlambda_blockchain::MILLISECONDS_PER_SLOT; use ethlambda_blockchain::block_builder::ProposerConfig; use ethlambda_blockchain::key_manager::ValidatorKeyPair; use ethlambda_network_api::{InitBlockChain, InitP2P, ToBlockChainToP2PRef, ToP2PToBlockChainRef}; -use ethlambda_p2p::{Bootnode, P2P, PeerId, SwarmConfig, build_swarm, parse_enrs}; +use ethlambda_p2p::{ + Bootnode, P2P, PeerId, SwarmConfig, attestation_subscription_subnets, build_swarm, parse_enrs, +}; use ethlambda_types::primitives::{H256, HashTreeRoot as _}; use ethlambda_types::{ aggregator::AggregatorController, @@ -213,11 +215,25 @@ async fn main() -> eyre::Result<()> { // and the API server (which exposes GET/POST admin endpoints). let aggregator = AggregatorController::new(options.is_aggregator); + // Attestation subnets this node subscribes to, computed once and shared by + // the P2P swarm (to open gossip subscriptions) and the blockchain actor + // (to size the early-aggregation threshold), so both agree on which subnets + // feed this node's gossip groups. Subscriptions are fixed at startup and + // are not re-evaluated when the aggregator role is toggled at runtime; see + // the hot-standby note on SwarmConfig. + let subscribed_subnets = attestation_subscription_subnets( + &validator_ids, + attestation_committee_count, + options.is_aggregator, + options.aggregate_subnet_ids.as_deref(), + ); + let blockchain = BlockChain::spawn( store.clone(), validator_keys, aggregator.clone(), attestation_committee_count, + subscribed_subnets.clone(), !options.disable_duty_sync_gate, ProposerConfig { enable_proposer_aggregation: options.enable_proposer_aggregation, @@ -225,19 +241,13 @@ async fn main() -> eyre::Result<()> { }, ); - // Note: SwarmConfig.is_aggregator is intentionally a plain bool, not the - // AggregatorController — subnet subscriptions are decided once here and - // are not re-evaluated at runtime. Toggling via the admin API affects - // aggregation logic but not the gossip mesh. See crates/net/p2p/src/lib.rs - // for the invariant. let built = build_swarm(SwarmConfig { node_key: node_p2p_key, bootnodes, listening_socket: p2p_socket, validator_ids, attestation_committee_count, - is_aggregator: options.is_aggregator, - aggregate_subnet_ids: options.aggregate_subnet_ids, + subscription_subnets: subscribed_subnets, }) .wrap_err("failed to build swarm")?; diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 83ae1941..bd1de0fd 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -109,6 +109,7 @@ impl BlockChain { validator_keys: HashMap, aggregator: AggregatorController, attestation_committee_count: u64, + subscribed_subnets: HashSet, gate_duties: bool, proposer_config: ProposerConfig, ) -> BlockChain { @@ -135,6 +136,7 @@ impl BlockChain { current_aggregation: None, last_tick_instant: None, attestation_committee_count, + subscribed_subnets, proposer_config, pre_merge_coverage: None, sync_status: SyncStatusTracker::new(gate_duties), @@ -199,6 +201,15 @@ pub struct BlockChainServer { /// threshold. attestation_committee_count: u64, + /// Attestation subnets this node subscribes to (its validators' own + /// subnets plus any aggregator-only subnets), computed once at startup and + /// shared with the P2P swarm via [`ethlambda_p2p::attestation_subscription_subnets`]. + /// Sizes the early-aggregation threshold: gossip groups are keyed by + /// attestation data rather than subnet, so a single group gathers + /// signatures from every subscribed subnet at once, and the threshold must + /// scale with how many subnets feed it rather than assuming one. + subscribed_subnets: HashSet, + /// Proposer-side block-building policy proposer_config: ProposerConfig, @@ -525,14 +536,30 @@ impl BlockChainServer { return; } let max_group = self.store.max_gossip_group_count_for_slot(slot); - // Trigger once the largest current-slot group holds two-thirds of one - // committee's expected votes, rounded up: `ceil(2 * N / (3 * C))` - // (0 only when there are no validators, which never triggers). + // Trigger once the largest current-slot group holds two-thirds of the + // votes we expect it to collect, rounded up. Groups are keyed by + // attestation data (not by subnet), so one group gathers signatures + // from every subnet we subscribe to; the expected count is therefore + // the number of network validators whose committee subnet is one of + // ours, not a single committee's worth. With `N` validators across `C` + // committees, subnet `s` holds `N / C` validators, plus one more when + // `s < N % C`. (0 only when there are no such validators, which never + // triggers.) let min_group_sigs = if self.attestation_committee_count == 0 { 0 } else { let validator_count = self.store.head_state().validators.len() as u64; - (2 * validator_count).div_ceil(3 * self.attestation_committee_count) as usize + let committee_count = self.attestation_committee_count; + let expected_votes: u64 = self + .subscribed_subnets + .iter() + .filter(|&&subnet| subnet < committee_count) + .map(|&subnet| { + validator_count / committee_count + + u64::from(subnet < validator_count % committee_count) + }) + .sum(); + (2 * expected_votes).div_ceil(3) as usize }; if min_group_sigs == 0 || max_group < min_group_sigs { return; diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index de34e469..dba01b11 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -154,22 +154,59 @@ pub(crate) struct Behaviour { /// Configuration for building the libp2p swarm. /// -/// INVARIANT: `is_aggregator` is consumed once during [`build_swarm`] to decide -/// subnet subscriptions and is NOT stored on [`P2PServer`]. Runtime toggles -/// of the aggregator role via the admin API (see -/// [`ethlambda_types::aggregator::AggregatorController`]) intentionally do -/// not resubscribe gossip subnets — this is the leanSpec PR #636 scope -/// limitation ("hot-standby model"). If a runtime reader is ever added on -/// the P2P side, it must consult the shared `AggregatorController` instead -/// of a bool captured here, or the runtime toggle will silently diverge. +/// INVARIANT: `subscription_subnets` is the fixed set of attestation subnets +/// this node subscribes to. It is computed once by the caller via +/// [`attestation_subscription_subnets`] and shared with the blockchain actor, +/// so both agree on exactly which subnets feed this node's gossip groups. The +/// set is consumed during [`build_swarm`] and NOT stored on [`P2PServer`]: +/// runtime toggles of the aggregator role via the admin API (see +/// [`ethlambda_types::aggregator::AggregatorController`]) intentionally do not +/// resubscribe gossip subnets; this is the leanSpec PR #636 "hot-standby model" +/// scope limitation. A node that may aggregate at runtime must include those +/// subnets here at startup. pub struct SwarmConfig { pub node_key: Vec, pub bootnodes: Vec, pub listening_socket: SocketAddr, pub validator_ids: Vec, pub attestation_committee_count: u64, - pub is_aggregator: bool, - pub aggregate_subnet_ids: Option>, + /// Attestation subnets to subscribe to, precomputed via + /// [`attestation_subscription_subnets`]. + pub subscription_subnets: HashSet, +} + +/// The attestation subnets a node subscribes to, per leanSpec +/// (`src/lean_spec/__main__.py`): every validator subscribes to its own +/// committee subnet (`validator_id % attestation_committee_count`) for mesh +/// health, and an aggregator additionally subscribes to any explicit +/// `aggregate_subnet_ids`, falling back to subnet 0 when it would otherwise +/// subscribe to none. +/// +/// Computed once at startup and shared by [`build_swarm`] (to open the gossip +/// subscriptions) and the blockchain actor (to size the early-aggregation +/// threshold), so both agree on exactly which subnets feed this node's gossip +/// groups. +pub fn attestation_subscription_subnets( + validator_ids: &[u64], + attestation_committee_count: u64, + is_aggregator: bool, + aggregate_subnet_ids: Option<&[u64]>, +) -> HashSet { + let mut subnets: HashSet = validator_ids + .iter() + .map(|vid| vid % attestation_committee_count) + .collect(); + if is_aggregator { + if let Some(ids) = aggregate_subnet_ids { + subnets.extend(ids.iter().copied()); + } + // Fall back to subnet 0 only when the aggregator has no validators and + // no explicit subnets; otherwise leave the set as configured. + if subnets.is_empty() { + subnets.insert(0); + } + } + subnets } /// Result of building the swarm — contains all pieces needed to start the P2P actor. @@ -301,35 +338,23 @@ pub fn build_swarm( .subscribe(&aggregation_topic) .unwrap(); - // Subscribe to attestation subnets per leanSpec (`src/lean_spec/__main__.py`): - // every validator subscribes to its own subnet for mesh health; aggregators - // additionally subscribe to explicit `aggregate_subnet_ids` and fall back to - // subnet 0 when they have no validators of their own. - let validator_subnets: HashSet = config - .validator_ids - .iter() - .map(|vid| vid % config.attestation_committee_count) - .collect(); + // Subscribe to attestation subnets per leanSpec (`src/lean_spec/__main__.py`). + // `config.subscription_subnets` was precomputed by the caller via + // `attestation_subscription_subnets`; the blockchain actor is handed the + // same set so both agree on which subnets feed this node's gossip groups. // The committee metric should reflect validator membership only, not // aggregator-only subscriptions. - let metric_subnet = validator_subnets.iter().copied().min().unwrap_or(0); + let metric_subnet = config + .validator_ids + .iter() + .map(|vid| vid % config.attestation_committee_count) + .min() + .unwrap_or(0); metrics::set_attestation_committee_subnet(metric_subnet); - let mut subscription_subnets = validator_subnets; - if config.is_aggregator { - if let Some(ref explicit_ids) = config.aggregate_subnet_ids { - subscription_subnets.extend(explicit_ids); - } - // Fall back to subnet 0 only when the aggregator has no validators - // and no explicit subnets — otherwise leave the set as configured. - if subscription_subnets.is_empty() { - subscription_subnets.insert(0); - } - } - let mut attestation_topics: HashMap = HashMap::new(); - for &subnet_id in &subscription_subnets { + for &subnet_id in &config.subscription_subnets { let topic = attestation_subnet_topic(subnet_id); swarm.behaviour_mut().gossipsub.subscribe(&topic)?; info!(subnet_id, "Subscribed to attestation subnet"); From ce9cd14b8ca54b26c475c423fbe186abaf21097e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Tue, 7 Jul 2026 19:41:57 -0300 Subject: [PATCH 31/33] docs: simplify comments --- crates/blockchain/src/lib.rs | 5 +---- crates/net/p2p/src/lib.rs | 17 +++-------------- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 2a3bdc7d..36abcbfb 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -207,10 +207,7 @@ pub struct BlockChainServer { /// Attestation subnets this node subscribes to (its validators' own /// subnets plus any aggregator-only subnets), computed once at startup and /// shared with the P2P swarm via [`ethlambda_p2p::attestation_subscription_subnets`]. - /// Sizes the early-aggregation threshold: gossip groups are keyed by - /// attestation data rather than subnet, so a single group gathers - /// signatures from every subscribed subnet at once, and the threshold must - /// scale with how many subnets feed it rather than assuming one. + /// Used to scale the early-aggregation threshold. subscribed_subnets: HashSet, /// Proposer-side block-building policy diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index 68e1e920..4726ce25 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -175,17 +175,11 @@ pub struct SwarmConfig { pub subscription_subnets: HashSet, } -/// The attestation subnets a node subscribes to, per leanSpec -/// (`src/lean_spec/__main__.py`): every validator subscribes to its own -/// committee subnet (`validator_id % attestation_committee_count`) for mesh -/// health, and an aggregator additionally subscribes to any explicit +/// The attestation subnets a node subscribes to: every validator subscribes +/// to its own committee subnet (`validator_id % attestation_committee_count`) +/// for mesh health, and an aggregator additionally subscribes to any explicit /// `aggregate_subnet_ids`, falling back to subnet 0 when it would otherwise /// subscribe to none. -/// -/// Computed once at startup and shared by [`build_swarm`] (to open the gossip -/// subscriptions) and the blockchain actor (to size the early-aggregation -/// threshold), so both agree on exactly which subnets feed this node's gossip -/// groups. pub fn attestation_subscription_subnets( validator_ids: &[u64], attestation_committee_count: u64, @@ -341,11 +335,6 @@ pub fn build_swarm( .subscribe(&aggregation_topic) .unwrap(); - // Subscribe to attestation subnets per leanSpec (`src/lean_spec/__main__.py`). - // `config.subscription_subnets` was precomputed by the caller via - // `attestation_subscription_subnets`; the blockchain actor is handed the - // same set so both agree on which subnets feed this node's gossip groups. - // The committee metric should reflect validator membership only, not // aggregator-only subscriptions. let metric_subnet = config From f96c240d067cb5ffcae64c2973b7f3aef6cb15bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Tue, 7 Jul 2026 19:48:26 -0300 Subject: [PATCH 32/33] refactor(blockchain): bundle spawn dependencies into BlockChainConfig --- bin/ethlambda/src/main.rs | 20 +++++++++++--------- crates/blockchain/src/lib.rs | 35 +++++++++++++++++++++++++++++------ 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index f8d39462..57588163 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -52,7 +52,7 @@ use serde::Deserialize; use tracing::{error, info, warn}; use tracing_subscriber::{EnvFilter, Layer, Registry, layer::SubscriberExt}; -use ethlambda_blockchain::{BlockChain, SyncStatusController}; +use ethlambda_blockchain::{BlockChain, BlockChainConfig, SyncStatusController}; use ethlambda_rpc::RpcConfig; use ethlambda_storage::{ MAX_RESUMABLE_DB_STATE_AGE, StorageBackend, Store, backend::RocksDBBackend, @@ -238,14 +238,16 @@ async fn main() -> eyre::Result<()> { let blockchain = BlockChain::spawn( store.clone(), validator_keys, - aggregator.clone(), - sync_status.clone(), - attestation_committee_count, - subscribed_subnets.clone(), - !options.disable_duty_sync_gate, - ProposerConfig { - enable_proposer_aggregation: options.enable_proposer_aggregation, - max_attestations_per_block: options.max_attestations_per_block, + BlockChainConfig { + aggregator: aggregator.clone(), + sync_status_controller: sync_status.clone(), + attestation_committee_count, + gate_duties: !options.disable_duty_sync_gate, + subscribed_subnets: subscribed_subnets.clone(), + proposer_config: ProposerConfig { + enable_proposer_aggregation: options.enable_proposer_aggregation, + max_attestations_per_block: options.max_attestations_per_block, + }, }, ); diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 36abcbfb..675933f1 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -44,6 +44,25 @@ pub struct BlockChain { handle: ActorRef, } +/// Startup configuration for the [`BlockChain`] actor: the distinct +/// dependencies wired in once at spawn, grouped to keep the constructor's +/// signature small. +pub struct BlockChainConfig { + /// Committee-aggregator role, toggleable at runtime via the admin API. + pub aggregator: AggregatorController, + /// Runtime-readable sync status: written by the actor each tick and read + /// by the RPC `/lean/v0/node/syncing` endpoint. + pub sync_status_controller: SyncStatusController, + /// Number of attestation committees (= subnet count). + pub attestation_committee_count: u64, + /// Whether the sync-gate suppresses validator duties (vs observe-only). + pub gate_duties: bool, + /// Attestation subnets this node subscribes to. + pub subscribed_subnets: HashSet, + /// Proposer-side block-building policy. + pub proposer_config: ProposerConfig, +} + /// Milliseconds per interval (800ms ticks). pub const MILLISECONDS_PER_INTERVAL: u64 = 800; /// Number of intervals per slot (5 intervals of 800ms = 4 seconds). @@ -108,13 +127,17 @@ impl BlockChain { pub fn spawn( store: Store, validator_keys: HashMap, - aggregator: AggregatorController, - sync_status_controller: SyncStatusController, - attestation_committee_count: u64, - subscribed_subnets: HashSet, - gate_duties: bool, - proposer_config: ProposerConfig, + config: BlockChainConfig, ) -> BlockChain { + let BlockChainConfig { + aggregator, + sync_status_controller, + attestation_committee_count, + gate_duties, + subscribed_subnets, + proposer_config, + } = config; + metrics::set_is_aggregator(aggregator.is_enabled()); metrics::set_node_sync_status(metrics::SyncStatus::Idle); let genesis_time = store.config().genesis_time; From 1d76134592287870b405d71087a00366abc84a3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Tue, 7 Jul 2026 19:53:19 -0300 Subject: [PATCH 33/33] refactor(main): extract BlockChainConfig into a variable --- bin/ethlambda/src/main.rs | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index 57588163..0ca6d077 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -235,21 +235,19 @@ async fn main() -> eyre::Result<()> { // metric's startup value. let sync_status = SyncStatusController::default(); - let blockchain = BlockChain::spawn( - store.clone(), - validator_keys, - BlockChainConfig { - aggregator: aggregator.clone(), - sync_status_controller: sync_status.clone(), - attestation_committee_count, - gate_duties: !options.disable_duty_sync_gate, - subscribed_subnets: subscribed_subnets.clone(), - proposer_config: ProposerConfig { - enable_proposer_aggregation: options.enable_proposer_aggregation, - max_attestations_per_block: options.max_attestations_per_block, - }, + let blockchain_config = BlockChainConfig { + aggregator: aggregator.clone(), + sync_status_controller: sync_status.clone(), + attestation_committee_count, + gate_duties: !options.disable_duty_sync_gate, + subscribed_subnets: subscribed_subnets.clone(), + proposer_config: ProposerConfig { + enable_proposer_aggregation: options.enable_proposer_aggregation, + max_attestations_per_block: options.max_attestations_per_block, }, - ); + }; + + let blockchain = BlockChain::spawn(store.clone(), validator_keys, blockchain_config); let built = build_swarm(SwarmConfig { node_key: node_p2p_key,