From 6291d6c7066e6ecd1f48ed61089fb3657619d7c3 Mon Sep 17 00:00:00 2001 From: Coredevjay Date: Tue, 20 Oct 2026 04:38:47 +0000 Subject: [PATCH] fix(api): harden newsletter tokens, SendGrid ECDSA webhook, persist chain events and markets Closes #1728 Closes #1729 Closes #1733 Closes #1734 --- docs/deployment.md | 2 +- docs/webhook-security.md | 4 +- services/api/Cargo.toml | 1 + ...23_hash_newsletter_confirmation_tokens.sql | 5 ++ .../024_chain_events_and_market_ingestion.sql | 17 +++++ ...chain_events_and_market_ingestion_down.sql | 4 ++ services/api/src/blockchain.rs | 23 ++++++- services/api/src/db.rs | 63 ++++++++++++++++++- services/api/src/handlers.rs | 2 +- services/api/src/security.rs | 55 +++++++++++++++- 10 files changed, 166 insertions(+), 10 deletions(-) create mode 100644 services/api/database/migrations/023_hash_newsletter_confirmation_tokens.sql create mode 100644 services/api/database/migrations/024_chain_events_and_market_ingestion.sql create mode 100644 services/api/database/migrations/rollbacks/024_chain_events_and_market_ingestion_down.sql diff --git a/docs/deployment.md b/docs/deployment.md index 79e06904..88bfb287 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -203,7 +203,7 @@ The API service is configured entirely via environment variables. Variables are | `NEWSLETTER_TOKEN_TTL_SECS` | `86400` | Confirmation token expiry | | `OTLP_ENDPOINT` | *(none)* | OpenTelemetry collector endpoint | | `TRACE_SAMPLE_RATE` | `0.1` | Fraction of requests traced (0–1) | -| `SENDGRID_WEBHOOK_SECRET` | *(none)* | SendGrid webhook signature secret | +| `SENDGRID_WEBHOOK_SECRET` | *(none)* | SendGrid webhook ECDSA public key (base64 DER) | | `ADMIN_WHITELIST_IPS` | *(none — admin routes unrestricted)* | Comma-separated CIDR allowlist | ### ECS task secrets and environment mapping diff --git a/docs/webhook-security.md b/docs/webhook-security.md index c301be1f..7d985957 100644 --- a/docs/webhook-security.md +++ b/docs/webhook-security.md @@ -16,7 +16,7 @@ The `/webhooks/sendgrid` endpoint receives and processes email delivery status e SendGrid signs every webhook request with an HMAC-SHA256 signature. The signature is verified by the `sendgrid_webhook_middleware` before the request reaches the handler. **Configuration:** -- `SENDGRID_WEBHOOK_SECRET`: The shared secret used to verify signatures. Must be set in production. +- `SENDGRID_WEBHOOK_SECRET`: SendGrid's Signed Event Webhook **public key** (base64 DER, ECDSA P-256). Must be set in production. HMAC-SHA256 with a shared secret is accepted only outside production, for local testing. - Signature is in the `X-Twilio-Email-Event-Webhook-Signature` header - Verification happens in `security::sendgrid_webhook_middleware` before body parsing @@ -89,7 +89,7 @@ Provides fallback deduplication if: | Variable | Default | Purpose | |---|---|---| | `WEBHOOK_REPLAY_WINDOW_SECS` | `300` | Duration (seconds) that Redis nonces are retained | -| `SENDGRID_WEBHOOK_SECRET` | (required) | Shared HMAC key for signature verification | +| `SENDGRID_WEBHOOK_SECRET` | (required) | SendGrid ECDSA public key (base64 DER) for signature verification | **Production Checklist:** - [ ] `SENDGRID_WEBHOOK_SECRET` is set and matches the SendGrid dashboard diff --git a/services/api/Cargo.toml b/services/api/Cargo.toml index 73764721..58a590fa 100644 --- a/services/api/Cargo.toml +++ b/services/api/Cargo.toml @@ -59,6 +59,7 @@ hmac = "0.12" rand = { version = "0.8", features = ["getrandom"] } hex = "0.4" base64 = "0.22" +p256 = { version = "0.13", features = ["ecdsa", "pem"] } subtle = "2.5" secrecy = { version = "0.8", features = ["serde"] } ipnet = "2" diff --git a/services/api/database/migrations/023_hash_newsletter_confirmation_tokens.sql b/services/api/database/migrations/023_hash_newsletter_confirmation_tokens.sql new file mode 100644 index 00000000..59a383d0 --- /dev/null +++ b/services/api/database/migrations/023_hash_newsletter_confirmation_tokens.sql @@ -0,0 +1,5 @@ +-- Store newsletter confirmation tokens as SHA-256 hex digests (matches unsubscribe tokens). +UPDATE newsletter_subscribers +SET confirmation_token = encode(sha256(convert_to(confirmation_token, 'UTF8')), 'hex') +WHERE confirmation_token IS NOT NULL + AND confirmation_token !~ '^[0-9a-f]{64}$'; diff --git a/services/api/database/migrations/024_chain_events_and_market_ingestion.sql b/services/api/database/migrations/024_chain_events_and_market_ingestion.sql new file mode 100644 index 00000000..13949cc5 --- /dev/null +++ b/services/api/database/migrations/024_chain_events_and_market_ingestion.sql @@ -0,0 +1,17 @@ +-- Durable store for chain events (Redis is only a cache). +CREATE TABLE IF NOT EXISTS chain_events ( + event_id TEXT PRIMARY KEY, + ledger BIGINT NOT NULL, + topic TEXT NOT NULL, + tx_hash TEXT, + payload JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); +CREATE INDEX IF NOT EXISTS chain_events_ledger_idx ON chain_events (ledger); + +-- Markets: on-chain identity + resolved-outcome consistency. +ALTER TABLE markets ADD COLUMN IF NOT EXISTS on_chain_id BIGINT; +CREATE UNIQUE INDEX IF NOT EXISTS markets_on_chain_id_key ON markets (on_chain_id); +ALTER TABLE markets DROP CONSTRAINT IF EXISTS markets_resolved_requires_outcome; +ALTER TABLE markets ADD CONSTRAINT markets_resolved_requires_outcome + CHECK (status <> 'resolved' OR outcome_index IS NOT NULL) NOT VALID; diff --git a/services/api/database/migrations/rollbacks/024_chain_events_and_market_ingestion_down.sql b/services/api/database/migrations/rollbacks/024_chain_events_and_market_ingestion_down.sql new file mode 100644 index 00000000..97a5132f --- /dev/null +++ b/services/api/database/migrations/rollbacks/024_chain_events_and_market_ingestion_down.sql @@ -0,0 +1,4 @@ +ALTER TABLE markets DROP CONSTRAINT IF EXISTS markets_resolved_requires_outcome; +DROP INDEX IF EXISTS markets_on_chain_id_key; +ALTER TABLE markets DROP COLUMN IF EXISTS on_chain_id; +DROP TABLE IF EXISTS chain_events; diff --git a/services/api/src/blockchain.rs b/services/api/src/blockchain.rs index e9efd881..c091b586 100644 --- a/services/api/src/blockchain.rs +++ b/services/api/src/blockchain.rs @@ -1053,6 +1053,9 @@ impl BlockchainClient { let events = self.fetch_events_since(cursor_ledger + 1).await?; for event in events { + self.db + .chain_event_insert(&event.id, event.ledger, &event.topic, event.tx_hash.as_deref(), &event.value) + .await?; let event_key = format!("{}:event:{}", keys::CHAIN_PREFIX, event.id); self.cache .set_json(&event_key, &event, Duration::from_secs(30 * 60)) @@ -1074,7 +1077,21 @@ impl BlockchainClient { } for market_id in &self.sync_market_ids { - let _ = self.market_data_cached(*market_id).await; + if let Ok(m) = self.market_data_cached(*market_id).await { + if let Err(e) = self + .db + .market_upsert_from_chain( + m.market_id, + m.title.as_deref(), + m.status.as_deref(), + m.resolved_outcome.map(|o| o as i32), + &m.onchain_volume, + ) + .await + { + tracing::warn!(market_id, error = %e, "market upsert failed"); + } + } let _ = self.oracle_result_cached(*market_id).await; } @@ -1428,6 +1445,10 @@ impl BlockchainClient { if event.ledger > confirmed_tip { continue; } + self.db + .chain_event_insert(&event.id, event.ledger, &event.topic, event.tx_hash.as_deref(), &event.value) + .await + .map_err(|e| BlockchainError::Internal(e.to_string()))?; let event_key = format!("{}:event:{}", keys::CHAIN_PREFIX, event.id); self.cache .set_json(&event_key, &event, Duration::from_secs(30 * 60)) diff --git a/services/api/src/db.rs b/services/api/src/db.rs index a8e9569c..21f61683 100644 --- a/services/api/src/db.rs +++ b/services/api/src/db.rs @@ -336,6 +336,65 @@ impl Database { Ok(value) } + /// Durably persist a chain event; idempotent on `event_id`. + pub async fn chain_event_insert( + &self, + event_id: &str, + ledger: u32, + topic: &str, + tx_hash: Option<&str>, + payload: &serde_json::Value, + ) -> anyhow::Result<()> { + self.with_timeout("chain_event_insert", sqlx::query( + "INSERT INTO chain_events (event_id, ledger, topic, tx_hash, payload) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (event_id) DO NOTHING", + ) + .bind(event_id) + .bind(ledger as i64) + .bind(topic) + .bind(tx_hash) + .bind(payload) + .execute(&self.pool)).await.map_err(anyhow::Error::from)?; + Ok(()) + } + + /// Upsert a market read from the chain, keyed by `on_chain_id`. + /// A `resolved` status without an outcome is stored as `active` to satisfy + /// the table's consistency constraint. + pub async fn market_upsert_from_chain( + &self, + on_chain_id: i64, + title: Option<&str>, + status: Option<&str>, + outcome_index: Option, + total_volume: &str, + ) -> anyhow::Result<()> { + let (status, outcome_index) = match (status, outcome_index) { + (Some("resolved"), Some(o)) => ("resolved", Some(o)), + (Some("cancelled"), o) => ("cancelled", o), + _ => ("active", None), + }; + let title = title.map(str::to_owned).unwrap_or_else(|| format!("Market {on_chain_id}")); + self.with_timeout("market_upsert_from_chain", sqlx::query( + "INSERT INTO markets (on_chain_id, title, status, outcome_index, total_volume, ends_at, resolved_at) + VALUES ($1, $2, $3, $4, $5::NUMERIC, NOW(), CASE WHEN $3 = 'resolved' THEN NOW() END) + ON CONFLICT (on_chain_id) DO UPDATE SET + title = EXCLUDED.title, + status = EXCLUDED.status, + outcome_index = EXCLUDED.outcome_index, + total_volume = EXCLUDED.total_volume, + resolved_at = COALESCE(markets.resolved_at, EXCLUDED.resolved_at)", + ) + .bind(on_chain_id) + .bind(title) + .bind(status) + .bind(outcome_index) + .bind(total_volume) + .execute(&self.pool)).await.map_err(anyhow::Error::from)?; + Ok(()) + } + pub async fn newsletter_get_by_email( &self, normalized_email: &str, @@ -383,7 +442,7 @@ impl Database { ) .bind(normalized_email) .bind(source) - .bind(confirmation_token) + .bind(crate::newsletter::hash_unsubscribe_token(confirmation_token)) .execute(&self.pool)).await.map_err(anyhow::Error::from)?; Ok(()) @@ -400,7 +459,7 @@ impl Database { WHERE confirmation_token = $1 AND created_at > NOW() - ($2 || ' seconds')::INTERVAL", ) - .bind(token) + .bind(crate::newsletter::hash_unsubscribe_token(token)) .bind(token_ttl_secs as i64) .execute(&self.pool)).await.map_err(anyhow::Error::from)?; diff --git a/services/api/src/handlers.rs b/services/api/src/handlers.rs index 314bc8a3..eedb1429 100644 --- a/services/api/src/handlers.rs +++ b/services/api/src/handlers.rs @@ -546,7 +546,7 @@ pub async fn newsletter_subscribe( .unwrap_or(false); if !already_active { - let token = Uuid::new_v4().to_string(); + let token = crate::newsletter::generate_opaque_unsubscribe_token().0; state .db .newsletter_upsert_pending(&email, &source, &token) diff --git a/services/api/src/security.rs b/services/api/src/security.rs index 8880f7f1..1b2685a1 100644 --- a/services/api/src/security.rs +++ b/services/api/src/security.rs @@ -495,8 +495,10 @@ pub async fn metrics_auth_middleware( /// SendGrid webhook signature verification middleware. /// -/// Verifies the `X-Twilio-Email-Event-Webhook-Signature` header using HMAC-SHA256 -/// against the raw request body. The `SENDGRID_WEBHOOK_SECRET` must be configured +/// Verifies the `X-Twilio-Email-Event-Webhook-Signature` header as an ECDSA (P-256) +/// signature over `timestamp + body`. `SENDGRID_WEBHOOK_SECRET` holds SendGrid's +/// base64 DER public key (HMAC-SHA256 is accepted only outside production, for +/// local testing). It must be configured /// for webhook security, except in development environment where it passes through. /// /// Replay protection: rejects requests whose `X-Twilio-Email-Event-Webhook-Timestamp` @@ -561,7 +563,7 @@ pub async fn sendgrid_webhook_middleware( // Signature covers timestamp + payload per SendGrid spec let signed_payload = format!("{}{}", ts_str, String::from_utf8_lossy(&bytes)); - if !signing::verify_signature(signed_payload.as_bytes(), sig, secret) { + if !signing::verify_sendgrid_signature(signed_payload.as_bytes(), sig, secret, is_dev) { return Err(StatusCode::UNAUTHORIZED); } @@ -580,6 +582,35 @@ pub mod signing { type HmacSha256 = Hmac; + /// Verify a SendGrid Signed Event Webhook signature (base64 DER ECDSA P-256 + /// over `timestamp + payload`) against the base64 DER public key `key`. + pub fn verify_ecdsa_signature(payload: &[u8], signature: &str, key: &str) -> bool { + use p256::ecdsa::{signature::Verifier, Signature, VerifyingKey}; + use p256::pkcs8::DecodePublicKey; + + let Ok(key_der) = BASE64.decode(key.trim()) else { return false }; + let Ok(vk) = VerifyingKey::from_public_key_der(&key_der) else { return false }; + let Ok(sig_der) = BASE64.decode(signature.trim()) else { return false }; + let Ok(sig) = Signature::from_der(&sig_der) else { return false }; + vk.verify(payload, &sig).is_ok() + } + + /// ECDSA verification; when `allow_hmac` (non-production only) and `key` + /// is not a valid public key, fall back to the legacy HMAC mode. + pub fn verify_sendgrid_signature(payload: &[u8], signature: &str, key: &str, allow_hmac: bool) -> bool { + use p256::pkcs8::DecodePublicKey; + let is_public_key = BASE64 + .decode(key.trim()) + .ok() + .and_then(|d| p256::ecdsa::VerifyingKey::from_public_key_der(&d).ok()) + .is_some(); + if is_public_key { + verify_ecdsa_signature(payload, signature, key) + } else { + allow_hmac && verify_signature(payload, signature, key) + } + } + pub fn verify_signature(payload: &[u8], signature: &str, secret: &str) -> bool { let mut mac = match HmacSha256::new_from_slice(secret.as_bytes()) { Ok(m) => m, @@ -681,6 +712,24 @@ impl IntoResponse for SecurityError { #[cfg(test)] mod tests { + #[test] + fn sendgrid_ecdsa_signature_roundtrip() { + use base64::{engine::general_purpose::STANDARD as B64, Engine}; + use p256::ecdsa::{signature::Signer, Signature, SigningKey}; + use p256::pkcs8::EncodePublicKey; + + let sk = SigningKey::from_slice(&[7u8; 32]).unwrap(); + let pk = B64.encode(sk.verifying_key().to_public_key_der().unwrap().as_bytes()); + let payload = b"1700000000[{\"event\":\"delivered\"}]"; + let sig: Signature = sk.sign(payload); + let sig_b64 = B64.encode(sig.to_der().as_bytes()); + + assert!(super::signing::verify_sendgrid_signature(payload, &sig_b64, &pk, false)); + assert!(!super::signing::verify_sendgrid_signature(b"tampered", &sig_b64, &pk, false)); + // HMAC fallback is only honoured when explicitly allowed and key isn't a public key. + assert!(!super::signing::verify_sendgrid_signature(payload, &sig_b64, "not-a-key", false)); + } + use super::*; use axum::http::HeaderMap; use ipnet::IpNet;