Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions crates/switchyard-server/tests/server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3392,6 +3392,49 @@ async fn json_extractor_statuses_keep_api_specific_error_envelopes() -> TestResu
Ok(())
}

#[tokio::test]
async fn chat_multiple_choice_requests_are_rejected_before_upstream_dispatch() -> TestResult {
let (upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;

for stream in [false, true] {
let response = send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": ROUTE_MODEL,
"messages": [{"role": "user", "content": "hello"}],
"n": 2,
"stream": stream,
})),
)
.await?;
assert_eq!(response.status, StatusCode::BAD_REQUEST, "stream={stream}");
assert_eq!(
response
.headers
.get("content-type")
.and_then(|value| value.to_str().ok()),
Some("application/json"),
"stream={stream}"
);
assert_eq!(
response.json()?,
json!({
"error": {
"message": "invalid value at $.n: multiple OpenAI Chat choices are not supported; set `n` to 1",
"type": "invalid_request_error",
"code": "invalid_body"
}
}),
"stream={stream}"
);
}

assert!(upstream.calls.lock().await.is_empty());
Ok(())
}

#[tokio::test]
async fn models_endpoint_reports_declared_route_capabilities_and_null_when_undeclared() -> TestResult
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,12 @@ impl FormatCodec for OpenAiChatCodec {

fn decode_request(&self, body: &Value, policy: &TranslationPolicy) -> Result<DecodedRequest> {
let body = object(body, "$")?;
if body.get("n").and_then(Value::as_u64).is_some_and(|n| n > 1) {
return Err(TranslationError::InvalidValue {
path: "$.n".to_string(),
message: "multiple OpenAI Chat choices are not supported; set `n` to 1".to_string(),
});
}
let mut diagnostics = Vec::new();
let mut request = LlmRequest {
model: body
Expand Down Expand Up @@ -269,6 +275,22 @@ impl FormatCodec for OpenAiChatCodec {
_policy: &TranslationPolicy,
) -> Result<DecodedResponse> {
let object = object(body, "$")?;
let choices = object.get("choices").and_then(Value::as_array);
let has_unsupported_choices = choices.is_some_and(|choices| {
choices.len() > 1
|| choices.iter().any(|choice| {
choice
.get("index")
.and_then(Value::as_number)
.is_some_and(|index| index.as_f64() != Some(0.0))
})
Comment thread
coderabbitai[bot] marked this conversation as resolved.
});
if has_unsupported_choices {
return Err(TranslationError::InvalidValue {
path: "$.choices".to_string(),
message: "multiple OpenAI Chat choices are not supported".to_string(),
});
}
let mut response = AggLlmResponse {
id: object
.get("id")
Expand All @@ -289,9 +311,7 @@ impl FormatCodec for OpenAiChatCodec {
_policy,
),
};
if let Some(choice) = object
.get("choices")
.and_then(Value::as_array)
if let Some(choice) = choices
.and_then(|choices| choices.first())
.and_then(Value::as_object)
{
Expand Down
23 changes: 17 additions & 6 deletions crates/switchyard-translation/src/codecs/openai_chat/stream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,22 @@ fn decode_openai_chat_stream(
}];
}

let choices = object.get("choices").and_then(Value::as_array);
let has_unsupported_choices = choices.is_some_and(|choices| {
choices.len() > 1
|| choices.iter().any(|choice| {
choice
.get("index")
.and_then(Value::as_number)
.is_some_and(|index| index.as_f64() != Some(0.0))
})
});
if has_unsupported_choices {
return vec![LlmResponseChunk::DecodeError {
message: "multiple OpenAI Chat choices are not supported".to_string(),
}];
}

let mut out = Vec::new();
let mut identity_changed = false;
if state.model.is_none() {
Expand All @@ -91,12 +107,7 @@ fn decode_openai_chat_stream(
out.push(LlmResponseChunk::Usage(usage));
}

for choice in object
.get("choices")
.and_then(Value::as_array)
.into_iter()
.flatten()
{
for choice in choices.into_iter().flatten() {
let Some(choice) = choice.as_object() else {
continue;
};
Expand Down
6 changes: 1 addition & 5 deletions crates/switchyard-translation/tests/lossless_roundtrip.rs
Original file line number Diff line number Diff line change
Expand Up @@ -549,6 +549,7 @@ fn response_fixture(format: WireFormat) -> Value {
"object": "chat.completion",
"created": 1780000000,
"model": "gpt-5.2",
// OpenAI Chat response translation supports one choice.
"choices": [
{
"index": 0,
Expand Down Expand Up @@ -576,11 +577,6 @@ fn response_fixture(format: WireFormat) -> Value {
},
"finish_reason": "tool_calls",
"logprobs": {"content": []}
},
{
"index": 1,
"message": {"role": "assistant", "content": "alternate"},
"finish_reason": "stop"
}
],
"usage": {
Expand Down
48 changes: 47 additions & 1 deletion crates/switchyard-translation/tests/request_translation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -314,7 +314,8 @@ fn openai_target_prompt_preserves_native_request_fields() -> TestResult {
"modalities": ["text", "audio"],
"audio": {"voice": "alloy", "format": "wav"},
"web_search_options": {"search_context_size": "high"},
"n": 2,
// OpenAI Chat supports a single choice in this preservation fixture.
"n": 1,
"logit_bias": {"42": -1},
"frequency_penalty": 0.2,
"presence_penalty": 0.3,
Expand Down Expand Up @@ -3098,6 +3099,51 @@ fn malformed_request_fields_are_rejected() {
}
}

#[test]
fn openai_chat_request_rejects_multiple_choices_for_buffered_and_streaming() {
Comment thread
coderabbitai[bot] marked this conversation as resolved.
let engine = TranslationEngine::default();
for stream in [false, true] {
let body = json!({
"model": "gpt-4o",
"messages": [{"role": "user", "content": "hello"}],
"n": 2,
"stream": stream,
});
let error = match engine.decode_request(
WireFormat::OpenAiChat,
&body,
&TranslationPolicy::default(),
) {
Ok(_) => panic!("stream={stream} request with n=2 must be rejected"),
Err(error) => error,
};
assert_eq!(error.kind(), "InvalidValue");
assert_eq!(
error.to_string(),
"invalid value at $.n: multiple OpenAI Chat choices are not supported; set `n` to 1"
);
}
}

#[test]
fn openai_chat_request_accepts_single_choice_defaults() {
let engine = TranslationEngine::default();
for n in [None, Some(1_u64)] {
let mut body = json!({
"model": "gpt-4o",
"messages": [{"role": "user", "content": "hello"}],
});
if let Some(n) = n {
body["n"] = json!(n);
}
if let Err(error) =
engine.decode_request(WireFormat::OpenAiChat, &body, &TranslationPolicy::default())
{
panic!("n={n:?} must remain valid: {error}");
}
}
}

// --- Invalid-role rejection ----------------------------------
// A transparent router must reject the same payloads the upstream provider
// would, rather than silently coercing an unknown role (e.g. "api") to `user`
Expand Down
42 changes: 42 additions & 0 deletions crates/switchyard-translation/tests/response_translation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,48 @@ fn openai_chat_response_translates_to_anthropic_message() -> TestResult {
Ok(())
}

#[test]
fn openai_chat_buffered_response_rejects_unsupported_choice_sets() {
let engine = TranslationEngine::default();
let cases = [
json!({
"id": "chatcmpl-two",
"choices": [
{"index": 0, "message": {"role": "assistant", "content": "first"}, "finish_reason": "stop"},
{"index": 1, "message": {"role": "assistant", "content": "second"}, "finish_reason": "stop"}
]
}),
json!({
"id": "chatcmpl-index-one",
"choices": [
{"index": 1, "message": {"role": "assistant", "content": "second"}, "finish_reason": "stop"}
]
}),
json!({
"id": "chatcmpl-negative-index",
"choices": [
{"index": -1, "message": {"role": "assistant", "content": "invalid"}, "finish_reason": "stop"}
]
}),
];

for body in cases {
let error = match engine.decode_response(
WireFormat::OpenAiChat,
&body,
&TranslationPolicy::default(),
) {
Ok(_) => panic!("unsupported choice set must be rejected: {body}"),
Err(error) => error,
};
assert_eq!(error.kind(), "InvalidValue");
assert_eq!(
error.to_string(),
"invalid value at $.choices: multiple OpenAI Chat choices are not supported"
);
}
}

// Verifies Anthropic message responses map to OpenAI Chat completions.
#[test]
fn anthropic_message_response_translates_to_openai_chat_completion() -> TestResult {
Expand Down
104 changes: 104 additions & 0 deletions crates/switchyard-translation/tests/stream_translation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1857,6 +1857,110 @@ fn openai_chat_error_frame_decodes_to_stream_error() -> TestResult {
Ok(())
}

#[test]
fn openai_chat_provider_error_precedes_unsupported_choices() {
let mut state = StreamTranslationState::new(WireFormat::OpenAiChat, WireFormat::OpenAiChat);
let state_before = state.clone();
let event = json!({
"error": {"message": "upstream exploded", "type": "server_error"},
"choices": [
{"index": 0, "delta": {"content": "first"}},
{"index": 1, "delta": {"content": "second"}}
]
});

let chunks = decode_stream_event(&mut state, WireFormat::OpenAiChat, &event);

assert_eq!(
chunks,
vec![LlmResponseChunk::StreamError {
message: "upstream exploded".to_string(),
}]
);
assert_eq!(state, state_before);
}

#[test]
fn openai_chat_stream_rejects_unsupported_choice_sets_before_state_changes() {
let cases = [
json!({
"id": "chatcmpl-two",
"model": "gpt-4o",
"choices": [
{
"index": 0,
"delta": {"tool_calls": [{
"index": 0,
"id": "call_1",
"function": {"name": "lookup", "arguments": "{}"}
}]},
"finish_reason": "tool_calls"
},
{"index": 1, "delta": {"content": "second"}}
],
"usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}
}),
json!({
"id": "chatcmpl-index-one",
"model": "gpt-4o",
"choices": [{"index": 1, "delta": {"content": "second"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}
}),
json!({
"id": "chatcmpl-negative-index",
"model": "gpt-4o",
"choices": [{"index": -1, "delta": {"content": "invalid"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}
}),
];

for event in cases {
let mut state =
StreamTranslationState::new(WireFormat::OpenAiChat, WireFormat::OpenAiResponses);
let state_before = state.clone();
let chunks = decode_stream_event(&mut state, WireFormat::OpenAiChat, &event);
assert_eq!(
chunks,
vec![LlmResponseChunk::DecodeError {
message: "multiple OpenAI Chat choices are not supported".to_string(),
}]
);
assert_eq!(state, state_before);
}
}

#[test]
fn openai_chat_same_format_multiple_choices_emit_one_terminal_error_without_raw_replay()
-> TestResult {
let engine = TranslationEngine::default();
let format = WireFormat::OpenAiChat;
let event = json!({
"id": "chatcmpl-two",
"model": "gpt-4o",
"provider_extension": "must not be replayed",
"choices": [
{"index": 0, "delta": {"content": "first"}},
{"index": 1, "delta": {"content": "second"}, "finish_reason": "stop"}
],
"usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}
});
let mut decode_state = StreamTranslationState::new(format, format);
let decode_state_before = decode_state.clone();
let preserved = engine.decode_stream_event(&mut decode_state, format, event)?;
assert_eq!(decode_state, decode_state_before);

let mut encode_state = StreamTranslationState::new(format, format);
let emitted = engine.encode_stream_event(&mut encode_state, format, preserved)?;
assert_eq!(
emitted,
vec![json!({
"error": {"message": "multiple OpenAI Chat choices are not supported"}
})]
);
assert!(engine.finish_stream(&mut encode_state, format)?.is_empty());
Ok(())
}

// Verifies the streaming encoder matches the buffered one: both Responses usage detail objects
// are present even when the upstream reports no cache or reasoning breakdown.
#[test]
Expand Down
Loading