From 7d0a98837e1ec2ad6148bb64e6d8fd30955f0dae Mon Sep 17 00:00:00 2001 From: Veniamin Vynohradov Date: Sat, 31 Oct 2026 22:37:16 +0200 Subject: [PATCH 1/2] fix: make Anthropic requests valid for Claude Sonnet 5.5 and other adaptive-only models Running Artemis with claude-sonnet-5-5 failed mid-task with: 400 messages.3.content.0: Invalid `signature` in `thinking` block. The block is bound to a different conversation. Sonnet 5.5 (like Opus 5.5, Haiku 5.5 and Fable 5.1) binds each thinking block to the exact history that produced it. The transcript ledger scrubs old UI lists and screenshots from earlier turns, so the next request replays a thinking block whose prefix changed and the API rejects it. The Anthropic request builder also sent shapes these models reject: temperature 0.0 (non-default sampling is a 400), thinking.budget_tokens from thinking_budget / reasoning_effort (removed, 400), and forced tool_choice from with_structured_output (400). langchain-anthropic also falls back to max_tokens=4096 for models it has no profile for, which thinking can exhaust before the tool call. For adaptive-only Claude models (Opus 4.7+, Sonnet 5+, Haiku 5+, Fable, Mythos): - omit temperature and set max_tokens to 16000 - map reasoning_effort / thinking_budget to output_config.effort with adaptive thinking - on models that bind thinking blocks, send thinking.block_binding.prefix_mismatch_behavior="drop_block" with the thinking-binding-controls-2026-08-01 beta, so blocks invalidated by history scrubbing are dropped instead of failing the request - default with_structured_output to method="json_schema" Older Claude ids (claude-3-7-sonnet, Sonnet/Opus 4.6 and earlier) keep the existing request shape. Co-Authored-By: Claude Opus 5.5 --- artemis/llm/claude.py | 114 ++++++++++++++++ artemis/llm/router.py | 49 +++++++ artemis/services/llm.py | 11 ++ tests/unit/test_claude_models.py | 216 +++++++++++++++++++++++++++++++ 4 files changed, 390 insertions(+) create mode 100644 artemis/llm/claude.py create mode 100644 tests/unit/test_claude_models.py diff --git a/artemis/llm/claude.py b/artemis/llm/claude.py new file mode 100644 index 000000000..de5e3adb7 --- /dev/null +++ b/artemis/llm/claude.py @@ -0,0 +1,114 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Claude model capability checks for the Anthropic request builder. + +Newer Claude models reject request shapes older ones accept: + +* Opus 4.7+, Sonnet 5+, Haiku 5+ and every Fable / Mythos model are + *adaptive-only*: ``thinking.budget_tokens`` and non-default sampling + parameters (``temperature``) are a 400, and thinking depth is set with + ``output_config.effort``. The Fable 5.1 / Opus 5.5 / Sonnet 5.5 + generation also rejects forced ``tool_choice``. +* Fable 5.1, Opus 5.5, Sonnet 5.5 and Haiku 5.5 bind each thinking block to + the exact history that produced it ("preserved thinking"). Replaying a + block after an earlier message changed is a 400 unless the request opts + into ``prefix_mismatch_behavior: "drop_block"``. + +Model ids with no parsable family/version (``claude-3-7-sonnet-...``, +custom deployments) are treated as legacy and keep the old request shape. +""" + +import re +from typing import Any + +#: Beta header that unlocks ``thinking.block_binding``. +THINKING_BINDING_BETA = "thinking-binding-controls-2026-08-01" + +#: Accepted values of ``output_config.effort``. +EFFORT_LEVELS = ("low", "medium", "high", "xhigh", "max") + +_MODEL_RE = re.compile(r"claude-(opus|sonnet|haiku|fable|mythos)-(\d+)(?:-(\d{1,2}))?(?!\d)") + +# Lowest version in each family that is adaptive-only. +_ADAPTIVE_ONLY_MIN = { + "opus": (4, 7), + "sonnet": (5, 0), + "haiku": (5, 0), + "fable": (0, 0), + "mythos": (0, 0), +} + +# Lowest version in each family that binds thinking blocks to the history. +_BLOCK_BINDING_MIN = { + "opus": (5, 5), + "sonnet": (5, 5), + "haiku": (5, 5), + "fable": (5, 1), + "mythos": (5, 1), +} + + +def claude_version(model_name: Any) -> tuple[str, tuple[int, int]] | None: + """Parses ``claude-sonnet-5-5`` into ``("sonnet", (5, 5))``. + + Provider prefixes (``anthropic.``, ``anthropic/``) and date or context + suffixes are ignored. Returns ``None`` for ids in the legacy + ``claude---`` order or anything unrecognised. + """ + if not isinstance(model_name, str): + return None + match = _MODEL_RE.search(model_name.lower()) + if not match: + return None + # The minor group stops at two digits, so a date snapshot + # (``claude-opus-4-20250514``) leaves the minor version at 0. + family, major, minor = match.groups() + return family, (int(major), int(minor) if minor else 0) + + +def _at_least(model_name: Any, minimums: dict[str, tuple[int, int]]) -> bool: + parsed = claude_version(model_name) + if parsed is None: + return False + family, version = parsed + return version >= minimums[family] + + +def is_adaptive_only(model_name: Any) -> bool: + """Whether the model rejects ``budget_tokens`` and non-default sampling.""" + return _at_least(model_name, _ADAPTIVE_ONLY_MIN) + + +def binds_thinking_blocks(model_name: Any) -> bool: + """Whether replayed thinking blocks must match their original history.""" + return _at_least(model_name, _BLOCK_BINDING_MIN) + + +def effort_for(reasoning_effort: str | None, thinking_budget: int | None) -> str | None: + """Maps the endpoint's effort / budget settings onto ``output_config.effort``. + + An explicit ``reasoning_effort`` wins; a legacy token budget is bucketed + with the same thresholds the budget path uses (2048 / 8192 / 32768). + """ + if reasoning_effort: + effort = reasoning_effort.lower() + return effort if effort in EFFORT_LEVELS else None + if thinking_budget: + if thinking_budget <= 2048: + return "low" + if thinking_budget <= 8192: + return "medium" + return "high" + return None diff --git a/artemis/llm/router.py b/artemis/llm/router.py index f2e6b42b6..8fc7c71a0 100644 --- a/artemis/llm/router.py +++ b/artemis/llm/router.py @@ -28,6 +28,12 @@ from pydantic import BaseModel, Field from artemis.config.settings import settings +from artemis.llm.claude import ( + THINKING_BINDING_BETA, + binds_thinking_blocks, + effort_for, + is_adaptive_only, +) from artemis.llm.google.provider import supports_thinking_level from third_party.mobile_use.utils.logger import get_logger @@ -195,6 +201,47 @@ def patched_process_tool_config(self, tool_choice, tool_config, formatted_tools) logger.warning(f"Could not patch ChatGoogleGenerativeAI._process_tool_config: {e}") +_ADAPTIVE_CLAUDE_MAX_TOKENS = 16000 + + +def _adaptive_claude_kwargs(endpoint: ModelEndpoint, kwargs: dict[str, Any]) -> dict[str, Any]: + """Builds ChatAnthropic kwargs for adaptive-only Claude models (Opus 4.7+, Sonnet 5+). + + These models 400 on ``budget_tokens`` and non-default ``temperature``, so + the temperature is dropped and the endpoint's effort / budget becomes + ``output_config.effort`` with adaptive thinking. + + Models that bind thinking blocks to their history (Sonnet 5.5, Opus 5.5, + Haiku 5.5, Fable 5.1) also get ``prefix_mismatch_behavior: "drop_block"``: + the transcript ledger scrubs old screenshots and UI lists from earlier + turns, which would otherwise make every later replayed thinking block a + 400 ("bound to a different conversation"). With ``drop_block`` the API + drops the stale blocks and the request goes through. + + ``max_tokens`` is set explicitly: thinking counts toward it, and + langchain-anthropic falls back to 4096 for models it has no profile for. + 16000 leaves room for thinking and stays under the SDK's non-streaming + limit (structured-output calls do not stream). + """ + kwargs = {k: v for k, v in kwargs.items() if k != "temperature"} + kwargs["max_tokens"] = endpoint.max_tokens or _ADAPTIVE_CLAUDE_MAX_TOKENS + effort = effort_for(endpoint.reasoning_effort, endpoint.thinking_budget) + thinking: dict[str, Any] | None = None + if binds_thinking_blocks(endpoint.model_name): + thinking = { + "type": "adaptive", + "block_binding": {"prefix_mismatch_behavior": "drop_block"}, + } + kwargs["betas"] = [THINKING_BINDING_BETA] + elif effort: + thinking = {"type": "adaptive"} + if thinking: + kwargs["thinking"] = thinking + if effort: + kwargs["effort"] = effort + return {k: v for k, v in kwargs.items() if v is not None} + + class ModelFactory: """Unified factory for instantiating and caching LangChain chat models across providers.""" @@ -321,6 +368,8 @@ def create_model(cls, endpoint: ModelEndpoint) -> BaseChatModel: "api_key": api_key, "timeout": endpoint.timeout_seconds, } + if is_adaptive_only(endpoint.model_name): + return ChatAnthropic(**_adaptive_claude_kwargs(endpoint, kwargs)) budget = endpoint.thinking_budget if not budget and endpoint.reasoning_effort: effort_map = {"low": 2048, "medium": 8192, "high": 32768} diff --git a/artemis/services/llm.py b/artemis/services/llm.py index ec91e6fa0..22de8ffc0 100644 --- a/artemis/services/llm.py +++ b/artemis/services/llm.py @@ -42,6 +42,7 @@ ) from artemis.context import ArtemisContext from artemis.data_engine.trace import CURRENT_TRACE_ID, DataEngineCallbackHandler +from artemis.llm.claude import is_adaptive_only from artemis.llm.google import is_google_chat_model, is_google_provider from artemis.llm.reliability import ( CircuitBreaker, @@ -594,6 +595,16 @@ def with_config(self, *args, **kwargs): ) def with_structured_output(self, *args, **kwargs): + # ChatAnthropic defaults to forced tool calling, which adaptive-only + # Claude models (Sonnet 5.5, Opus 5.5, ...) reject with a 400; native + # structured outputs (output_config.format) work on all of them. + if ( + "method" not in kwargs + and self.endpoint is not None + and ModelProvider.from_string(self.endpoint.provider) == ModelProvider.ANTHROPIC + and is_adaptive_only(self.endpoint.model_name) + ): + kwargs["method"] = "json_schema" if hasattr(self.base_model, "with_structured_output"): return RobustChatModelWrapper( self.base_model.with_structured_output(*args, **kwargs), diff --git a/tests/unit/test_claude_models.py b/tests/unit/test_claude_models.py new file mode 100644 index 000000000..b97a7c1a4 --- /dev/null +++ b/tests/unit/test_claude_models.py @@ -0,0 +1,216 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Request shape for adaptive-only Claude models (Sonnet 5.5, Opus 5.5, ...).""" + +import json +from unittest.mock import MagicMock + +import anthropic +import httpx +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage +import pytest + +from artemis.llm.claude import ( + THINKING_BINDING_BETA, + binds_thinking_blocks, + claude_version, + effort_for, + is_adaptive_only, +) +from artemis.llm.router import ModelEndpoint, ModelFactory, ModelProvider + + +@pytest.mark.parametrize( + "model, expected", + [ + ("claude-sonnet-5-5", ("sonnet", (5, 5))), + ("anthropic.claude-opus-4-8", ("opus", (4, 8))), + ("claude-sonnet-4-5-20250929", ("sonnet", (4, 5))), + ("claude-opus-4-20250514", ("opus", (4, 0))), + ("claude-fable-5", ("fable", (5, 0))), + ("claude-3-7-sonnet-20250219", None), + ("gpt-4o", None), + (None, None), + ], +) +def test_claude_version(model, expected): + assert claude_version(model) == expected + + +@pytest.mark.parametrize( + "model, adaptive_only, binds", + [ + ("claude-sonnet-5-5", True, True), + ("claude-opus-5-5", True, True), + ("claude-haiku-5-5", True, True), + ("claude-fable-5-1", True, True), + ("claude-fable-5", True, False), + ("claude-opus-5", True, False), + ("claude-sonnet-5", True, False), + ("claude-opus-4-7", True, False), + ("claude-opus-4-6", False, False), + ("claude-sonnet-4-6", False, False), + ("claude-haiku-4-5", False, False), + ("claude-3-7-sonnet-20250219", False, False), + ], +) +def test_capabilities(model, adaptive_only, binds): + assert is_adaptive_only(model) is adaptive_only + assert binds_thinking_blocks(model) is binds + + +def test_effort_for(): + assert effort_for("HIGH", 2048) == "high" + assert effort_for("bogus", None) is None + assert effort_for(None, 1024) == "low" + assert effort_for(None, 8192) == "medium" + assert effort_for(None, 32768) == "high" + assert effort_for(None, None) is None + + +def _capture_request(model): + """Sends one tool-use continuation through ``model`` and returns the wire request.""" + captured = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["headers"] = dict(request.headers) + captured["body"] = json.loads(request.content) + return httpx.Response( + 200, + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": model.model, + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + + model.__dict__["_client"] = anthropic.Anthropic( + api_key="sk-ant-test", + http_client=httpx.Client(transport=httpx.MockTransport(handler)), + ) + history = [ + HumanMessage("open the app"), + AIMessage( + content=[ + {"type": "thinking", "thinking": "", "signature": "sig"}, + {"type": "tool_use", "id": "t1", "name": "tap", "input": {}}, + ], + tool_calls=[{"name": "tap", "args": {}, "id": "t1"}], + ), + ToolMessage("tapped", tool_call_id="t1"), + ] + model.invoke(history) + return captured["body"], captured["headers"] + + +def test_sonnet_5_5_request_drops_stale_thinking_blocks(): + """Sonnet 5.5: no temperature, budget mapped to effort, drop_block enabled.""" + endpoint = ModelEndpoint( + provider=ModelProvider.ANTHROPIC, + model_name="claude-sonnet-5-5", + api_key="sk-ant-test", + thinking_budget=8192, + ) + body, headers = _capture_request(ModelFactory.create_model(endpoint)) + + assert "temperature" not in body + assert body["thinking"] == { + "type": "adaptive", + "block_binding": {"prefix_mismatch_behavior": "drop_block"}, + } + assert body["output_config"] == {"effort": "medium"} + assert THINKING_BINDING_BETA in headers["anthropic-beta"] + # The earlier thinking block is still replayed unchanged. + assert body["messages"][1]["content"][0] == { + "type": "thinking", + "thinking": "", + "signature": "sig", + } + + +def test_sonnet_5_5_defaults_send_no_sampling_or_budget(): + endpoint = ModelEndpoint( + provider=ModelProvider.ANTHROPIC, model_name="claude-sonnet-5-5", api_key="sk-ant-test" + ) + body, _ = _capture_request(ModelFactory.create_model(endpoint)) + assert "temperature" not in body + assert "budget_tokens" not in body["thinking"] + assert "output_config" not in body + assert body["max_tokens"] == 16000 + + +def test_opus_4_8_effort_uses_adaptive_thinking_without_binding(): + endpoint = ModelEndpoint( + provider=ModelProvider.ANTHROPIC, + model_name="claude-opus-4-8", + api_key="sk-ant-test", + reasoning_effort="high", + ) + body, headers = _capture_request(ModelFactory.create_model(endpoint)) + assert "temperature" not in body + assert body["thinking"] == {"type": "adaptive"} + assert body["output_config"] == {"effort": "high"} + assert "anthropic-beta" not in headers + + +def test_opus_4_8_without_effort_leaves_thinking_unset(): + endpoint = ModelEndpoint( + provider=ModelProvider.ANTHROPIC, model_name="claude-opus-4-8", api_key="sk-ant-test" + ) + model = ModelFactory.create_model(endpoint) + assert model.thinking is None + assert model.temperature is None + + +def test_legacy_claude_keeps_budget_tokens(): + endpoint = ModelEndpoint( + provider=ModelProvider.ANTHROPIC, + model_name="claude-sonnet-4-6", + api_key="sk-ant-test", + reasoning_effort="medium", + ) + model = ModelFactory.create_model(endpoint) + assert model.thinking == {"type": "enabled", "budget_tokens": 8192} + assert model.temperature == 1.0 + + +@pytest.mark.parametrize( + "model_name, expected_method", + [("claude-sonnet-5-5", "json_schema"), ("claude-sonnet-4-6", None)], +) +def test_structured_output_avoids_forced_tool_choice(model_name, expected_method): + """Adaptive-only Claude models reject forced tool_choice, so use json_schema.""" + from artemis.services.llm import RobustChatModelWrapper + + base = MagicMock() + endpoint = ModelEndpoint(provider=ModelProvider.ANTHROPIC, model_name=model_name) + RobustChatModelWrapper(base, None, endpoint=endpoint).with_structured_output(dict) + assert base.with_structured_output.call_args.kwargs.get("method") == expected_method + + +def test_structured_output_keeps_explicit_method(): + from artemis.services.llm import RobustChatModelWrapper + + base = MagicMock() + endpoint = ModelEndpoint(provider=ModelProvider.ANTHROPIC, model_name="claude-sonnet-5-5") + RobustChatModelWrapper(base, None, endpoint=endpoint).with_structured_output( + dict, method="function_calling" + ) + assert base.with_structured_output.call_args.kwargs["method"] == "function_calling" From b315f100741d52790ec2ce03dfde715de975cc0d Mon Sep 17 00:00:00 2001 From: Veniamin Vynohradov Date: Sun, 1 Nov 2026 12:59:45 +0200 Subject: [PATCH 2/2] fix: stop Sonnet 5.5 refusal loop on the reasoning reminder With claude-sonnet-5-5, a multi-turn Flash task looped from turn 3 on: every request returned HTTP 200 with stop_reason "refusal" and no content, and the runner nudged and re-sent the same prompt each turn. Sonnet 5.5 returns its between-tool narration as thinking blocks, so the ledger sees every turn as silent and appends REASONING_REMINDER. Its wording ("internal thinking is not shown ... write ...") trips the reasoning_extraction safety classifier (reproduced 1/1 live; reworded prompt: 4/4 normal tool calls, with visible text). - Reword REASONING_REMINDER as a request for a short progress note. - Raise LLMPermanentError with a new REFUSAL failure category when a response has stop_reason "refusal", so the fallback endpoint is tried and a repeated refusal fails the task instead of looping. Co-Authored-By: Claude Opus 5.5 --- artemis/agents/operator/prompts.py | 12 ++++++--- artemis/llm/reliability.py | 4 +++ artemis/services/llm.py | 22 ++++++++++++++++ tests/unit/test_claude_models.py | 41 ++++++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 4 deletions(-) diff --git a/artemis/agents/operator/prompts.py b/artemis/agents/operator/prompts.py index e800de234..cbac574d2 100644 --- a/artemis/agents/operator/prompts.py +++ b/artemis/agents/operator/prompts.py @@ -694,12 +694,16 @@ def _clip(text: str, limit: int = 80) -> str: #: One-line tail notice for the turn after a turn whose replies carried no visible #: reasoning text. Thinking-mode models satisfy "reason first" inside their thought #: channel and emit bare tool calls; the reminder costs no extra model call. +# Phrased as a request for a progress note, not for the model's reasoning: +# Claude Sonnet 5.5's safety classifiers decline a request that asks the model +# to surface its internal thinking (stop_reason "refusal", category +# "reasoning_extraction"), which the old wording ("internal thinking is not +# shown") triggered on every silent turn. REASONING_REMINDER = ( "--- Reminder ---\n" - "Your previous turn carried no visible reasoning text (internal thinking is not" - " shown). This turn, write one short paragraph of plain prose before calling a" - " Turn-Ending Action: what the screen shows, whether the last action worked, and" - " what you do next and why." + "Your previous turn went straight to a tool call. This turn, start with one short" + " progress note in plain prose before calling a Turn-Ending Action: what the screen" + " shows, whether the last action worked, and what you do next." ) diff --git a/artemis/llm/reliability.py b/artemis/llm/reliability.py index c35ba6b86..d2e4bd397 100644 --- a/artemis/llm/reliability.py +++ b/artemis/llm/reliability.py @@ -42,6 +42,10 @@ class FailureCategory(StrEnum): CONNECTION = "connection" AUTHENTICATION = "authentication" BAD_REQUEST = "bad_request" + #: The model's safety classifier declined the request (Claude + #: ``stop_reason: "refusal"``). Retrying the same endpoint returns the same + #: refusal; a fallback model may answer. + REFUSAL = "refusal" CANCELLED = "cancelled" UNKNOWN = "unknown" diff --git a/artemis/services/llm.py b/artemis/services/llm.py index 22de8ffc0..a5dc22c4a 100644 --- a/artemis/services/llm.py +++ b/artemis/services/llm.py @@ -46,6 +46,7 @@ from artemis.llm.google import is_google_chat_model, is_google_provider from artemis.llm.reliability import ( CircuitBreaker, + Failure, FailureCategory, LLMCallError, LLMExhaustedError, @@ -642,6 +643,25 @@ def _traced_call(self, args: tuple, kwargs: dict) -> tuple[tuple, dict, Any]: args, kwargs = _inject_parent_trace_id(trace_id, *args, **kwargs) return args, kwargs, trace_id + @staticmethod + def _raise_on_refusal(response) -> None: + """Turns a model refusal into a typed error instead of an empty reply. + + Claude's safety classifiers decline with HTTP 200, ``stop_reason: + "refusal"`` and no content. Returned as-is, the agent loop sees a turn + without a tool call, nudges and re-sends the same prompt, which is + refused again on every turn. Raising lets the fallback chain try the + next endpoint and otherwise fails the task with the reason. + """ + meta = getattr(response, "response_metadata", None) + if isinstance(meta, dict) and meta.get("stop_reason") == "refusal": + model = meta.get("model_name") or meta.get("model") or "the model" + raise LLMPermanentError( + f"{model} declined the request (stop_reason 'refusal'): its safety" + " classifier flagged the prompt.", + failure=Failure(FailureCategory.REFUSAL, retryable=False, should_fallback=True), + ) + def _meter_usage(self, response) -> None: """Record measured token usage for this call (M0 metering, best-effort).""" engine = self.ctx.data_engine if self.ctx else None @@ -657,6 +677,7 @@ async def ainvoke(self, *args, **kwargs): endpoint_key=self._endpoint_key(), ) self._meter_usage(response) + self._raise_on_refusal(response) return response async def complete(self, *args, **kwargs): @@ -677,6 +698,7 @@ async def complete(self, *args, **kwargs): endpoint_key=self._endpoint_key(), ) self._meter_usage(response) + self._raise_on_refusal(response) return response def _emit_stream_delta(self, stream_exec_id, chunk) -> None: diff --git a/tests/unit/test_claude_models.py b/tests/unit/test_claude_models.py index b97a7c1a4..2fbe7fb64 100644 --- a/tests/unit/test_claude_models.py +++ b/tests/unit/test_claude_models.py @@ -214,3 +214,44 @@ def test_structured_output_keeps_explicit_method(): dict, method="function_calling" ) assert base.with_structured_output.call_args.kwargs["method"] == "function_calling" + + +def _wrapper_returning(message): + from unittest.mock import AsyncMock + + from artemis.services.llm import RobustChatModelWrapper + + base = MagicMock() + base.ainvoke = AsyncMock(return_value=message) + endpoint = ModelEndpoint(provider=ModelProvider.ANTHROPIC, model_name="claude-sonnet-5-5") + return RobustChatModelWrapper(base, None, endpoint=endpoint) + + +@pytest.mark.asyncio +async def test_refusal_raises_typed_error_that_allows_fallback(): + """A refusal (HTTP 200, empty content) must not reach the agent loop as a silent turn.""" + from artemis.llm.reliability import FailureCategory, LLMPermanentError + + refused = AIMessage( + content=[], + response_metadata={"stop_reason": "refusal", "model_name": "claude-sonnet-5-5"}, + ) + with pytest.raises(LLMPermanentError) as excinfo: + await _wrapper_returning(refused).ainvoke([HumanMessage("hi")]) + assert excinfo.value.failure.category is FailureCategory.REFUSAL + assert excinfo.value.failure.should_fallback + assert not excinfo.value.failure.retryable + + +@pytest.mark.asyncio +async def test_normal_reply_passes_through(): + reply = AIMessage(content="ok", response_metadata={"stop_reason": "end_turn"}) + assert await _wrapper_returning(reply).ainvoke([HumanMessage("hi")]) is reply + + +def test_reasoning_reminder_does_not_ask_for_internal_thinking(): + """Sonnet 5.5 refuses prompts that ask it to surface its thinking (reasoning_extraction).""" + from artemis.agents.operator.prompts import REASONING_REMINDER + + assert "thinking" not in REASONING_REMINDER.lower() + assert "reasoning" not in REASONING_REMINDER.lower()