From 9e8f94a6aa8ba076f74a64b1d9c8d3f730e70bc1 Mon Sep 17 00:00:00 2001 From: Deepak Jain Date: Tue, 22 Sep 2026 17:42:30 +0000 Subject: [PATCH 1/2] fix(server): redact upstream error content from request and SSE logs Signed-off-by: Deepak Jain --- crates/switchyard-server/src/lib.rs | 36 ++++++++++++++- crates/switchyard-server/src/sse.rs | 72 ++++++++++++++++++++++++++++- 2 files changed, 106 insertions(+), 2 deletions(-) diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 58ebc0d97..c5f32d4fb 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -1341,7 +1341,13 @@ fn upstream_error(status: StatusCode, body: &str) -> Response { .as_str() .filter(|code| !code.is_empty()) .unwrap_or("upstream_error"); - error_response(status, message, "upstream_error", code) + let mut response = error_response(status, message, "upstream_error", code); + // The provider's message can quote request content, so the request log + // records only the stable error class, not the message. + response.extensions_mut().insert(RequestLogError(format!( + "upstream_error {code} (HTTP {status})" + ))); + response } // Keep error details until the endpoint chooses its response format. @@ -1987,4 +1993,32 @@ mod tests { Some("invalid request") ); } + + // The provider's message can quote request content, so the request log + // records only the error class while the client still sees the message. + #[test] + fn upstream_error_redacts_request_log_error() { + const LEAKED: &str = "SECRET-quoted-request-content"; + let error = LlmClientError::UpstreamHttp { + status: StatusCode::BAD_GATEWAY, + body: format!( + r#"{{"error":{{"message":"validation failed: {LEAKED}","code":"invalid_request_error"}}}}"# + ), + }; + let response = client_error(&error); + + let logged = response + .extensions() + .get::() + .map(|error| error.0.as_str()) + .unwrap_or(""); + assert!(!logged.contains(LEAKED), "{logged}"); + assert!(logged.contains("upstream_error"), "{logged}"); + + let api_error = response + .extensions() + .get::() + .expect("ApiError extension"); + assert!(api_error.message.contains(LEAKED), "{}", api_error.message); + } } diff --git a/crates/switchyard-server/src/sse.rs b/crates/switchyard-server/src/sse.rs index e685ba5e5..1eac493c0 100644 --- a/crates/switchyard-server/src/sse.rs +++ b/crates/switchyard-server/src/sse.rs @@ -9,6 +9,7 @@ use std::sync::Arc; use axum::response::sse::{Event, Sse}; use futures_util::Stream; use serde_json::{Value, json}; +use switchyard_runner::stream_error_summary; use switchyard_translation::{LlmStreamError, RawEventStream, WireFormat}; use crate::redaction::Redactor; @@ -45,7 +46,14 @@ pub(crate) fn frame_stream( }) } Err(LlmStreamError::Client(error)) => { - tracing::warn!(error = %error, "stream iteration failed"); + // The error text can quote request content, so the log + // records only the stable error class. + let summary = stream_error_summary(&error, None); + tracing::warn!( + error.kind = summary.kind.as_str(), + error.upstream_status = summary.upstream_status, + "stream iteration failed" + ); failed = true; error_event(target_format, error.to_string(), &redactor) } @@ -122,6 +130,7 @@ mod tests { use axum::{body::to_bytes, response::IntoResponse}; use futures_util::stream; use switchyard_protocol::LlmClientError; + use tracing_subscriber::layer::SubscriberExt; use super::*; @@ -181,4 +190,65 @@ mod tests { assert!(!body.contains("[DONE]")); Ok(()) } + + // Collects rendered warn events so the test can assert against the final + // log sink rather than a field mid-pipeline. + #[derive(Clone, Default)] + struct WarnCapture(Arc>>); + + impl tracing_subscriber::Layer for WarnCapture { + fn on_event( + &self, + event: &tracing::Event<'_>, + _ctx: tracing_subscriber::layer::Context<'_, tracing_subscriber::Registry>, + ) { + if *event.metadata().level() != tracing::Level::WARN { + return; + } + let mut fields = String::new(); + event.record( + &mut |field: &tracing::field::Field, value: &dyn std::fmt::Debug| { + fields.push_str(&format!("{}={value:?} ", field.name())); + }, + ); + self.0.lock().unwrap().push(fields); + } + } + + // The client error text can quote request content, so the stream-failure + // warn log records only the stable error class. + #[test] + fn stream_client_error_warn_redacts_upstream_body() -> TestResult { + const LEAKED: &str = "SECRET-quoted-request-content"; + let capture = WarnCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + + let body = tracing::subscriber::with_default(subscriber, || { + runtime.block_on(chat_body(vec![Err(LlmStreamError::Client( + LlmClientError::UpstreamHttp { + status: axum::http::StatusCode::BAD_GATEWAY, + body: format!("upstream failed: {LEAKED}"), + }, + ))])) + })?; + + // The client still sees the error text in-band. + assert!(body.contains(LEAKED), "{body}"); + + let events = capture.0.lock().unwrap().clone(); + assert!( + events + .iter() + .any(|event| event.contains("stream iteration failed")), + "{events:?}" + ); + assert!( + !events.iter().any(|event| event.contains(LEAKED)), + "{events:?}" + ); + Ok(()) + } } From 3c3d855945ae05c289fcae945449f47454e8b08c Mon Sep 17 00:00:00 2001 From: Deepak Jain Date: Fri, 25 Sep 2026 09:12:28 -0700 Subject: [PATCH 2/2] refactor(server): move owned error response extensions Signed-off-by: Deepak Jain --- crates/switchyard-server/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 7386adccd..fe5bdef83 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -1400,11 +1400,11 @@ impl ApiError { } } -fn render_error_response(response: Response, wire_format: WireFormat) -> Response { - let Some(error) = response.extensions().get::().cloned() else { +fn render_error_response(mut response: Response, wire_format: WireFormat) -> Response { + let Some(error) = response.extensions_mut().remove::() else { return response; }; - let log_error = response.extensions().get::().cloned(); + let log_error = response.extensions_mut().remove::(); let mut rendered = error.into_response(wire_format); if let Some(log_error) = log_error { rendered.extensions_mut().insert(log_error);