diff --git a/CHANGELOG.md b/CHANGELOG.md index 040d34c8..dc66df45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * `ChatOllama()` can now be created without a `model`, so `ChatOllama().list_models()` works. Sending a message without a model still raises an error that lists the available models. (#166) * `ChatGoogle()` now reports `finish_reason == "content_filter"` for Gemini's `IMAGE_SAFETY`, `IMAGE_PROHIBITED_CONTENT`, `IMAGE_RECITATION` and `LANGUAGE` finish reasons, instead of passing the raw Gemini string through, and reports `"tool_use"` (rather than `"success"`) when a turn ends with a tool request, as `ChatOpenAI()` does. (#473) * The error raised for a tool request with invalid JSON arguments from `ChatOpenAI()` and `ChatOpenAICompletions()` (and the providers built on them) now shows the offending arguments, instead of the literal text `{args}`. +* When streaming, `ChatAnthropic()` (and `ChatBedrockAnthropic()`) now takes the input and cache token counts from the final `message_delta` event, which reports cumulative usage, as the non-streaming response does. With server tools such as web search, those counts grow during the response, so streamed turns under-reported their tokens and cost (e.g. missing the cache writes made while searching). (#466) * `content_image_url()` with a `data:` URL no longer silently drops the `detail` argument: the setting is now carried by inline image content and forwarded to OpenAI's Responses and Chat Completions APIs (regular image URLs already forwarded it). Other providers continue to ignore `detail`. (#467) * `ChatBedrock()` on a Converse model no longer drops redacted reasoning blocks when streaming. They were discarded as they arrived, so a streamed turn lost them, while a non-streamed one kept them. (#468) * `ChatBedrockAnthropic()` now uses native structured outputs for `data_model` on Claude 4.5 and later, as `ChatAnthropic()` does. Its Bedrock model IDs (`anthropic.claude-...`, `us.anthropic.claude-...`) weren't recognized, so it fell back to forcing a tool call, which Claude Opus 5.5 and Sonnet 5.5 reject with a 400. diff --git a/chatlas/_provider_anthropic.py b/chatlas/_provider_anthropic.py index 3a0b4e0e..a8a6adfd 100644 --- a/chatlas/_provider_anthropic.py +++ b/chatlas/_provider_anthropic.py @@ -704,6 +704,18 @@ def stream_merge_chunks(self, completion, chunk): completion.stop_reason = chunk.delta.stop_reason completion.stop_sequence = chunk.delta.stop_sequence completion.usage.output_tokens = chunk.usage.output_tokens + # Usage on message_delta is cumulative and, with server tools, can + # exceed message_start's input and cache counts. Fields are omitted + # when not applicable, in which case message_start's value stands. + for field in ( + "input_tokens", + "cache_creation_input_tokens", + "cache_read_input_tokens", + "server_tool_use", + ): + value = getattr(chunk.usage, field, None) + if value is not None: + setattr(completion.usage, field, value) return completion diff --git a/tests/test_provider_anthropic.py b/tests/test_provider_anthropic.py index d904029f..4bace956 100644 --- a/tests/test_provider_anthropic.py +++ b/tests/test_provider_anthropic.py @@ -9,10 +9,13 @@ CitationsWebSearchResultLocation, DocumentBlock, Message, + MessageDeltaUsage, RawContentBlockDeltaEvent, RawContentBlockStartEvent, RawContentBlockStopEvent, + RawMessageDeltaEvent, RawMessageStartEvent, + ServerToolUsage, TextBlock, TextDelta, Usage, @@ -302,6 +305,93 @@ def test_anthropic_web_fetch_citation_uses_document_index_for_source(): assert citation.source.title == "Second document" +def test_anthropic_stream_usage_takes_cumulative_message_delta_counts(): + # With server tools, message_delta carries the cumulative input and cache + # counts, which differ from message_start (values from the + # test_anthropic_web_search_streaming recording). + provider = AnthropicProvider(model="claude-haiku-4-5", api_key="dummy", kwargs=None) + start_usage = Usage( + input_tokens=2247, + cache_creation_input_tokens=0, + cache_read_input_tokens=0, + output_tokens=26, + ) + start = RawMessageStartEvent( + type="message_start", + message=Message( + id="msg", + type="message", + role="assistant", + model="claude-haiku-4-5", + content=[], + stop_reason=None, + stop_sequence=None, + usage=start_usage, + ), + ) + delta = RawMessageDeltaEvent.model_validate( + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": MessageDeltaUsage( + input_tokens=2254, + cache_creation_input_tokens=17269, + cache_read_input_tokens=0, + output_tokens=110, + server_tool_use=ServerToolUsage( + web_search_requests=1, web_fetch_requests=0 + ), + ).model_dump(), + } + ) + + completion = provider.stream_merge_chunks(None, start) + completion = provider.stream_merge_chunks(completion, delta) + + assert completion.usage.input_tokens == 2254 + assert completion.usage.cache_creation_input_tokens == 17269 + assert completion.usage.cache_read_input_tokens == 0 + assert completion.usage.output_tokens == 110 + assert completion.usage.server_tool_use == ServerToolUsage( + web_search_requests=1, web_fetch_requests=0 + ) + assert provider.value_tokens(completion) == (2254 + int(17269 * 1.25), 110, 0) + + +def test_anthropic_stream_usage_keeps_message_start_counts_omitted_by_delta(): + provider = AnthropicProvider(model="claude-haiku-4-5", api_key="dummy", kwargs=None) + start = RawMessageStartEvent( + type="message_start", + message=Message( + id="msg", + type="message", + role="assistant", + model="claude-haiku-4-5", + content=[], + stop_reason=None, + stop_sequence=None, + usage=Usage( + input_tokens=3, + cache_creation_input_tokens=37, + cache_read_input_tokens=17269, + output_tokens=1, + ), + ), + ) + delta = RawMessageDeltaEvent.model_validate( + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 10}, + } + ) + + completion = provider.stream_merge_chunks(None, start) + completion = provider.stream_merge_chunks(completion, delta) + + assert provider.value_tokens(completion) == (3 + int(37 * 1.25), 10, 17269) + + def test_anthropic_web_fetch_citation_stream_uses_document_index_for_source(): provider = AnthropicProvider( model="claude-sonnet-4-5", api_key="dummy", kwargs=None @@ -1166,9 +1256,7 @@ def test_anthropic_message_params_convert_unsigned_thinking_blocks(): turns = [ UserTurn("hi"), - AssistantTurn( - [ContentThinking(thinking="hmm"), ContentText(text="hello")] - ), + AssistantTurn([ContentThinking(thinking="hmm"), ContentText(text="hello")]), UserTurn("again"), ] messages = provider._as_message_params(turns)