Add execution proof observability

beacon_node/beacon_chain/src/execution_proof_verification/gossip_verified_execution_proof.rs

@@ -4,6 +4,7 @@ use crate::canonical_head::CanonicalHead;
 use crate::execution_proof_verification::observed_execution_proofs::{
     ObservedExecutionProofs, ProofObservation,
 };
+use crate::metrics;
 use crate::pending_payload_cache::PendingPayloadCache;
 use crate::shuffling_cache::{ShufflingCache, with_cached_shuffling};
 use crate::validator_pubkey_cache::ValidatorPubkeyCache;
@@ -260,19 +261,32 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         self: &Arc<Self>,
         proof: Arc<SignedExecutionProofEnvelope>,
     ) -> Result<GossipVerifiedExecutionProof, Error> {
+        let proof_type = proof.proof_type();
         let chain = self.clone();
-        self.task_executor
-            .clone()
-            .spawn_blocking_handle(
-                move || {
-                    let ctx = chain.execution_proof_gossip_verification_context();
-                    GossipVerifiedExecutionProof::new(proof, &ctx)
-                },
-                "gossip_execution_proof_verification_handle",
-            )
-            .ok_or(BeaconChainError::RuntimeShutdown)?
-            .await
-            .map_err(BeaconChainError::TokioJoin)?
+        let result: Result<_, Error> = async {
+            self.task_executor
+                .clone()
+                .spawn_blocking_handle(
+                    move || {
+                        let ctx = chain.execution_proof_gossip_verification_context();
+                        GossipVerifiedExecutionProof::new(proof, &ctx)
+                    },
+                    "gossip_execution_proof_verification_handle",
+                )
+                .ok_or(BeaconChainError::RuntimeShutdown)?
+                .await
+                .map_err(BeaconChainError::TokioJoin)?
+        }
+        .await;
+        let (outcome, reason) = match &result {
+            Ok(_) => ("accepted", "valid"),
+            Err(error) => (error.outcome(), error.as_str()),
+        };
+        metrics::inc_counter_vec(
+            &metrics::EXECUTION_PROOF_VERIFICATION_TOTAL,
+            &[proof_type.into(), outcome, reason],
+        );
+        result
     }
 }
 

beacon_node/beacon_chain/src/execution_proof_verification/mod.rs

@@ -56,6 +56,46 @@ pub enum Error {
     BeaconChainError(Box<BeaconChainError>),
 }
 
+impl Error {
+    /// A stable classification of this gossip verification error.
+    pub const fn outcome(&self) -> &'static str {
+        match self {
+            Self::ProofAlreadySeen
+            | Self::ValidProofAlreadyKnown
+            | Self::DuplicateFromValidator { .. }
+            | Self::UnknownBlockRoot { .. }
+            | Self::PastFinalizedSlot { .. }
+            | Self::PayloadUnavailable { .. } => "ignored",
+            Self::EmptyProofData
+            | Self::UnknownValidatorIndex(_)
+            | Self::ValidatorNotActive { .. }
+            | Self::InvalidSignature
+            | Self::InvalidProof => "rejected",
+            Self::ProofEngineMissing | Self::ProofEngine(_) | Self::BeaconChainError(_) => "error",
+        }
+    }
+
+    /// A stable, bounded description of this gossip verification error.
+    pub const fn as_str(&self) -> &'static str {
+        match self {
+            Self::ProofAlreadySeen => "proof_already_seen",
+            Self::ValidProofAlreadyKnown => "valid_proof_already_known",
+            Self::DuplicateFromValidator { .. } => "duplicate_from_validator",
+            Self::UnknownBlockRoot { .. } => "unknown_block_root",
+            Self::PastFinalizedSlot { .. } => "past_finalized_slot",
+            Self::PayloadUnavailable { .. } => "payload_unavailable",
+            Self::EmptyProofData => "empty_proof_data",
+            Self::UnknownValidatorIndex(_) => "unknown_validator_index",
+            Self::ValidatorNotActive { .. } => "validator_not_active",
+            Self::InvalidSignature => "invalid_signature",
+            Self::InvalidProof => "invalid_proof",
+            Self::ProofEngineMissing => "proof_engine_missing",
+            Self::ProofEngine(_) => "proof_engine",
+            Self::BeaconChainError(_) => "beacon_chain",
+        }
+    }
+}
+
 impl From<BeaconChainError> for Error {
     fn from(e: BeaconChainError) -> Self {
         Error::BeaconChainError(Box::new(e))
@@ -75,3 +115,37 @@ impl From<ObservationError> for Error {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn error_metric_values_are_stable() {
+        assert_eq!(Error::ProofAlreadySeen.outcome(), "ignored");
+        assert_eq!(Error::InvalidSignature.outcome(), "rejected");
+        assert_eq!(Error::ProofEngineMissing.outcome(), "error");
+
+        assert_eq!(Error::ProofAlreadySeen.as_str(), "proof_already_seen");
+        assert_eq!(
+            Error::PayloadUnavailable {
+                beacon_block_root: Hash256::default(),
+            }
+            .as_str(),
+            "payload_unavailable"
+        );
+        assert_eq!(Error::InvalidSignature.as_str(), "invalid_signature");
+        assert_eq!(
+            Error::ProofEngine(ProofEngineError::ProofVerifierError {
+                message: "failed".to_string(),
+                error_type: "internal",
+            })
+            .as_str(),
+            "proof_engine"
+        );
+        assert_eq!(
+            Error::BeaconChainError(Box::new(BeaconChainError::RuntimeShutdown)).as_str(),
+            "beacon_chain"
+        );
+    }
+}

beacon_node/beacon_chain/src/metrics.rs

@@ -2110,6 +2110,55 @@ pub static PENDING_PAYLOAD_CACHE_SIZE: LazyLock<Result<IntGauge>> = LazyLock::ne
         "Number of entries in the pending payload availability cache.",
     )
 });
+pub static PENDING_PAYLOAD_CACHE_EXECUTION_PROOFS: LazyLock<Result<IntGauge>> =
+    LazyLock::new(|| {
+        try_create_int_gauge(
+            "pending_payload_cache_execution_proofs",
+            "Number of distinct execution proofs stored in the pending payload cache.",
+        )
+    });
+pub static PENDING_PAYLOAD_CACHE_PAYLOADS_AWAITING_EXECUTION_PROOFS: LazyLock<Result<IntGauge>> =
+    LazyLock::new(|| {
+        try_create_int_gauge(
+            "pending_payload_cache_payloads_awaiting_execution_proofs",
+            "Number of executed payload envelopes waiting for more distinct execution proofs.",
+        )
+    });
+pub static PENDING_PAYLOAD_CACHE_REQUIRED_EXECUTION_PROOFS: LazyLock<Result<IntGauge>> =
+    LazyLock::new(|| {
+        try_create_int_gauge(
+            "pending_payload_cache_required_execution_proofs",
+            "Configured number of distinct execution proofs required before payload import.",
+        )
+    });
+
+/*
+ * Execution proof metrics
+ */
+pub static EXECUTION_PROOF_RECEIVED_TOTAL: LazyLock<Result<IntCounterVec>> = LazyLock::new(|| {
+    try_create_int_counter_vec(
+        "beacon_execution_proof_received_total",
+        "Count of execution proof envelopes received by source and proof type.",
+        &["source", "proof_type"],
+    )
+});
+pub static EXECUTION_PROOF_VERIFICATION_TOTAL: LazyLock<Result<IntCounterVec>> =
+    LazyLock::new(|| {
+        try_create_int_counter_vec(
+            "beacon_execution_proof_verification_total",
+            "Count of execution proof verification outcomes by proof type, outcome, and reason.",
+            &["proof_type", "outcome", "reason"],
+        )
+    });
+pub static EXECUTION_PROOF_GOSSIP_PROCESSING_SECONDS: LazyLock<Result<HistogramVec>> =
+    LazyLock::new(|| {
+        try_create_histogram_vec_with_buckets(
+            "beacon_execution_proof_gossip_processing_seconds",
+            "Time spent processing execution proofs received over gossip by proof type.",
+            decimal_buckets(-3, 1),
+            &["proof_type"],
+        )
+    });
 pub static DATA_AVAILABILITY_RECONSTRUCTION_TIME: LazyLock<Result<Histogram>> =
     LazyLock::new(|| {
         try_create_histogram(
@@ -2221,6 +2270,24 @@ pub fn scrape_for_metrics<T: BeaconChainTypes>(beacon_chain: &BeaconChain<T>) {
         &PENDING_PAYLOAD_CACHE_SIZE,
         beacon_chain.pending_payload_cache.cache_size(),
     );
+    set_gauge_by_usize(
+        &PENDING_PAYLOAD_CACHE_EXECUTION_PROOFS,
+        beacon_chain
+            .pending_payload_cache
+            .cached_execution_proof_count(),
+    );
+    set_gauge_by_usize(
+        &PENDING_PAYLOAD_CACHE_PAYLOADS_AWAITING_EXECUTION_PROOFS,
+        beacon_chain
+            .pending_payload_cache
+            .payloads_awaiting_execution_proofs(),
+    );
+    set_gauge_by_usize(
+        &PENDING_PAYLOAD_CACHE_REQUIRED_EXECUTION_PROOFS,
+        beacon_chain
+            .pending_payload_cache
+            .required_execution_proofs(),
+    );
 
     if let Some((size, num_lookups)) = beacon_chain.pre_finalization_block_cache.metrics() {
         set_gauge_by_usize(&PRE_FINALIZATION_BLOCK_CACHE_SIZE, size);

beacon_node/beacon_chain/src/pending_payload_cache/mod.rs

@@ -579,6 +579,31 @@ impl<T: BeaconChainTypes> PendingPayloadCache<T> {
         self.availability_cache.read().len()
     }
 
+    /// Number of distinct execution proofs held across all pending payloads.
+    pub fn cached_execution_proof_count(&self) -> usize {
+        self.availability_cache
+            .read()
+            .iter()
+            .map(|(_, components)| components.execution_proofs.len())
+            .sum()
+    }
+
+    /// Number of executed payload envelopes that remain gated on execution proofs.
+    pub fn payloads_awaiting_execution_proofs(&self) -> usize {
+        if self.required_execution_proofs == 0 {
+            return 0;
+        }
+
+        self.availability_cache
+            .read()
+            .iter()
+            .filter(|(_, components)| {
+                components.envelope.is_some()
+                    && components.execution_proofs.len() < self.required_execution_proofs
+            })
+            .count()
+    }
+
     // ── Internal helpers ──
 
     fn check_availability(
@@ -1038,7 +1063,10 @@ mod data_availability_checker_tests {
     #[tokio::test]
     async fn execution_proof_gates_availability() {
         let s = setup_gated(NodeCustodyType::Fullnode);
+        assert_eq!(s.cache.cached_execution_proof_count(), 0);
+        assert_eq!(s.cache.payloads_awaiting_execution_proofs(), 0);
         s.put_envelope();
+        assert_eq!(s.cache.payloads_awaiting_execution_proofs(), 1);
         assert_missing(s.put_columns(s.custody.clone()));
 
         let assigned = ProofType::all();
@@ -1047,6 +1075,8 @@ mod data_availability_checker_tests {
         for _ in 0..=REQUIRED_EXECUTION_PROOFS {
             assert_missing(s.put_proof(assigned[0]));
         }
+        assert_eq!(s.cache.cached_execution_proof_count(), 1);
+        assert_eq!(s.cache.payloads_awaiting_execution_proofs(), 1);
 
         // Distinct provers up to the requirement flip it to available.
         let mut availability = None;
@@ -1055,6 +1085,11 @@ mod data_availability_checker_tests {
         }
         let envelope = assert_available(availability.expect("gate needs two provers or more"));
         assert_eq!(envelope.block_root, s.block_root);
+        assert_eq!(
+            s.cache.cached_execution_proof_count(),
+            REQUIRED_EXECUTION_PROOFS
+        );
+        assert_eq!(s.cache.payloads_awaiting_execution_proofs(), 0);
 
         assert_missing(s.put_proof(assigned[REQUIRED_EXECUTION_PROOFS]));
     }

beacon_node/http_api/src/beacon/execution_proofs.rs

@@ -16,6 +16,7 @@ use crate::utils::{
 };
 use crate::version::add_ssz_content_type_header;
 use beacon_chain::execution_proof_verification::Error as ProofError;
+use beacon_chain::metrics as beacon_chain_metrics;
 use beacon_chain::{AvailabilityProcessingStatus, BeaconChain, BeaconChainTypes};
 use bytes::Bytes;
 use eth2::types::{self as api_types, Failure};
@@ -224,6 +225,19 @@ async fn publish_execution_proof<T: BeaconChainTypes>(
     let beacon_block_root = proof.beacon_block_root();
     let proof_type = proof.proof_type();
     let validator_index = proof.validator_index;
+    let proof_bytes = proof.message.proof_data.len();
+
+    beacon_chain_metrics::inc_counter_vec(
+        &beacon_chain_metrics::EXECUTION_PROOF_RECEIVED_TOTAL,
+        &["http_api", proof_type.into()],
+    );
+    debug!(
+        %beacon_block_root,
+        %proof_type,
+        validator_index,
+        proof_bytes,
+        "Received execution proof from HTTP API"
+    );
 
     let verified = match chain.verify_execution_proof_for_gossip(proof.clone()).await {
         Ok(verified) => verified,

beacon_node/network/src/network_beacon_processor/gossip_methods.rs

@@ -12,6 +12,7 @@ use beacon_chain::data_column_verification::{
 };
 use beacon_chain::execution_proof_verification::Error as ExecutionProofError;
 use beacon_chain::fetch_blobs::PartialHeaderOrBid;
+use beacon_chain::metrics as beacon_chain_metrics;
 use beacon_chain::partial_data_column_assembler::UpdatedPartials;
 use beacon_chain::payload_bid_verification::PayloadBidError;
 use beacon_chain::payload_envelope_verification::{
@@ -4167,6 +4168,25 @@ impl<T: BeaconChainTypes> NetworkBeaconProcessor<T> {
     ) {
         let beacon_block_root = execution_proof.beacon_block_root();
         let proof_type = execution_proof.proof_type();
+        let _timer = beacon_chain_metrics::start_timer_vec(
+            &beacon_chain_metrics::EXECUTION_PROOF_GOSSIP_PROCESSING_SECONDS,
+            &[proof_type.into()],
+        );
+        let validator_index = execution_proof.validator_index;
+        let proof_bytes = execution_proof.message.proof_data.len();
+
+        beacon_chain_metrics::inc_counter_vec(
+            &beacon_chain_metrics::EXECUTION_PROOF_RECEIVED_TOTAL,
+            &["gossip", proof_type.into()],
+        );
+        debug!(
+            %peer_id,
+            %beacon_block_root,
+            %proof_type,
+            validator_index,
+            proof_bytes,
+            "Received execution proof from gossip"
+        );
 
         match self
             .chain

beacon_node/proof_engine/Cargo.toml

@@ -13,6 +13,7 @@ test-utils = []
 ethereum_serde_utils = { workspace = true }
 ethereum_ssz = { workspace = true }
 hex = { workspace = true }
+metrics = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 types = { workspace = true }

beacon_node/proof_engine/src/config.rs

@@ -123,9 +123,10 @@ impl ProofEngineConfig {
 
         #[cfg(not(feature = "ere-verifier"))]
         {
-            Err(ProofEngineError::ProofVerifierError(
-                "Lighthouse was built without `ere-verifier`".to_string(),
-            ))
+            Err(ProofEngineError::ProofVerifierError {
+                message: "Lighthouse was built without `ere-verifier`".to_string(),
+                error_type: "proof_verifier_error",
+            })
         }
     }
 }