diff --git a/artemis/agents/operator/prompts.py b/artemis/agents/operator/prompts.py index e800de234..cbac574d2 100644 --- a/artemis/agents/operator/prompts.py +++ b/artemis/agents/operator/prompts.py @@ -694,12 +694,16 @@ def _clip(text: str, limit: int = 80) -> str: #: One-line tail notice for the turn after a turn whose replies carried no visible #: reasoning text. Thinking-mode models satisfy "reason first" inside their thought #: channel and emit bare tool calls; the reminder costs no extra model call. +# Phrased as a request for a progress note, not for the model's reasoning: +# Claude Sonnet 5.5's safety classifiers decline a request that asks the model +# to surface its internal thinking (stop_reason "refusal", category +# "reasoning_extraction"), which the old wording ("internal thinking is not +# shown") triggered on every silent turn. REASONING_REMINDER = ( "--- Reminder ---\n" - "Your previous turn carried no visible reasoning text (internal thinking is not" - " shown). This turn, write one short paragraph of plain prose before calling a" - " Turn-Ending Action: what the screen shows, whether the last action worked, and" - " what you do next and why." + "Your previous turn went straight to a tool call. This turn, start with one short" + " progress note in plain prose before calling a Turn-Ending Action: what the screen" + " shows, whether the last action worked, and what you do next." ) diff --git a/artemis/llm/claude.py b/artemis/llm/claude.py new file mode 100644 index 000000000..de5e3adb7 --- /dev/null +++ b/artemis/llm/claude.py @@ -0,0 +1,114 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Claude model capability checks for the Anthropic request builder. + +Newer Claude models reject request shapes older ones accept: + +* Opus 4.7+, Sonnet 5+, Haiku 5+ and every Fable / Mythos model are + *adaptive-only*: ``thinking.budget_tokens`` and non-default sampling + parameters (``temperature``) are a 400, and thinking depth is set with + ``output_config.effort``. The Fable 5.1 / Opus 5.5 / Sonnet 5.5 + generation also rejects forced ``tool_choice``. +* Fable 5.1, Opus 5.5, Sonnet 5.5 and Haiku 5.5 bind each thinking block to + the exact history that produced it ("preserved thinking"). Replaying a + block after an earlier message changed is a 400 unless the request opts + into ``prefix_mismatch_behavior: "drop_block"``. + +Model ids with no parsable family/version (``claude-3-7-sonnet-...``, +custom deployments) are treated as legacy and keep the old request shape. +""" + +import re +from typing import Any + +#: Beta header that unlocks ``thinking.block_binding``. +THINKING_BINDING_BETA = "thinking-binding-controls-2026-08-01" + +#: Accepted values of ``output_config.effort``. +EFFORT_LEVELS = ("low", "medium", "high", "xhigh", "max") + +_MODEL_RE = re.compile(r"claude-(opus|sonnet|haiku|fable|mythos)-(\d+)(?:-(\d{1,2}))?(?!\d)") + +# Lowest version in each family that is adaptive-only. +_ADAPTIVE_ONLY_MIN = { + "opus": (4, 7), + "sonnet": (5, 0), + "haiku": (5, 0), + "fable": (0, 0), + "mythos": (0, 0), +} + +# Lowest version in each family that binds thinking blocks to the history. +_BLOCK_BINDING_MIN = { + "opus": (5, 5), + "sonnet": (5, 5), + "haiku": (5, 5), + "fable": (5, 1), + "mythos": (5, 1), +} + + +def claude_version(model_name: Any) -> tuple[str, tuple[int, int]] | None: + """Parses ``claude-sonnet-5-5`` into ``("sonnet", (5, 5))``. + + Provider prefixes (``anthropic.``, ``anthropic/``) and date or context + suffixes are ignored. Returns ``None`` for ids in the legacy + ``claude---`` order or anything unrecognised. + """ + if not isinstance(model_name, str): + return None + match = _MODEL_RE.search(model_name.lower()) + if not match: + return None + # The minor group stops at two digits, so a date snapshot + # (``claude-opus-4-20250514``) leaves the minor version at 0. + family, major, minor = match.groups() + return family, (int(major), int(minor) if minor else 0) + + +def _at_least(model_name: Any, minimums: dict[str, tuple[int, int]]) -> bool: + parsed = claude_version(model_name) + if parsed is None: + return False + family, version = parsed + return version >= minimums[family] + + +def is_adaptive_only(model_name: Any) -> bool: + """Whether the model rejects ``budget_tokens`` and non-default sampling.""" + return _at_least(model_name, _ADAPTIVE_ONLY_MIN) + + +def binds_thinking_blocks(model_name: Any) -> bool: + """Whether replayed thinking blocks must match their original history.""" + return _at_least(model_name, _BLOCK_BINDING_MIN) + + +def effort_for(reasoning_effort: str | None, thinking_budget: int | None) -> str | None: + """Maps the endpoint's effort / budget settings onto ``output_config.effort``. + + An explicit ``reasoning_effort`` wins; a legacy token budget is bucketed + with the same thresholds the budget path uses (2048 / 8192 / 32768). + """ + if reasoning_effort: + effort = reasoning_effort.lower() + return effort if effort in EFFORT_LEVELS else None + if thinking_budget: + if thinking_budget <= 2048: + return "low" + if thinking_budget <= 8192: + return "medium" + return "high" + return None diff --git a/artemis/llm/reliability.py b/artemis/llm/reliability.py index c35ba6b86..d2e4bd397 100644 --- a/artemis/llm/reliability.py +++ b/artemis/llm/reliability.py @@ -42,6 +42,10 @@ class FailureCategory(StrEnum): CONNECTION = "connection" AUTHENTICATION = "authentication" BAD_REQUEST = "bad_request" + #: The model's safety classifier declined the request (Claude + #: ``stop_reason: "refusal"``). Retrying the same endpoint returns the same + #: refusal; a fallback model may answer. + REFUSAL = "refusal" CANCELLED = "cancelled" UNKNOWN = "unknown" diff --git a/artemis/llm/router.py b/artemis/llm/router.py index f2e6b42b6..8fc7c71a0 100644 --- a/artemis/llm/router.py +++ b/artemis/llm/router.py @@ -28,6 +28,12 @@ from pydantic import BaseModel, Field from artemis.config.settings import settings +from artemis.llm.claude import ( + THINKING_BINDING_BETA, + binds_thinking_blocks, + effort_for, + is_adaptive_only, +) from artemis.llm.google.provider import supports_thinking_level from third_party.mobile_use.utils.logger import get_logger @@ -195,6 +201,47 @@ def patched_process_tool_config(self, tool_choice, tool_config, formatted_tools) logger.warning(f"Could not patch ChatGoogleGenerativeAI._process_tool_config: {e}") +_ADAPTIVE_CLAUDE_MAX_TOKENS = 16000 + + +def _adaptive_claude_kwargs(endpoint: ModelEndpoint, kwargs: dict[str, Any]) -> dict[str, Any]: + """Builds ChatAnthropic kwargs for adaptive-only Claude models (Opus 4.7+, Sonnet 5+). + + These models 400 on ``budget_tokens`` and non-default ``temperature``, so + the temperature is dropped and the endpoint's effort / budget becomes + ``output_config.effort`` with adaptive thinking. + + Models that bind thinking blocks to their history (Sonnet 5.5, Opus 5.5, + Haiku 5.5, Fable 5.1) also get ``prefix_mismatch_behavior: "drop_block"``: + the transcript ledger scrubs old screenshots and UI lists from earlier + turns, which would otherwise make every later replayed thinking block a + 400 ("bound to a different conversation"). With ``drop_block`` the API + drops the stale blocks and the request goes through. + + ``max_tokens`` is set explicitly: thinking counts toward it, and + langchain-anthropic falls back to 4096 for models it has no profile for. + 16000 leaves room for thinking and stays under the SDK's non-streaming + limit (structured-output calls do not stream). + """ + kwargs = {k: v for k, v in kwargs.items() if k != "temperature"} + kwargs["max_tokens"] = endpoint.max_tokens or _ADAPTIVE_CLAUDE_MAX_TOKENS + effort = effort_for(endpoint.reasoning_effort, endpoint.thinking_budget) + thinking: dict[str, Any] | None = None + if binds_thinking_blocks(endpoint.model_name): + thinking = { + "type": "adaptive", + "block_binding": {"prefix_mismatch_behavior": "drop_block"}, + } + kwargs["betas"] = [THINKING_BINDING_BETA] + elif effort: + thinking = {"type": "adaptive"} + if thinking: + kwargs["thinking"] = thinking + if effort: + kwargs["effort"] = effort + return {k: v for k, v in kwargs.items() if v is not None} + + class ModelFactory: """Unified factory for instantiating and caching LangChain chat models across providers.""" @@ -321,6 +368,8 @@ def create_model(cls, endpoint: ModelEndpoint) -> BaseChatModel: "api_key": api_key, "timeout": endpoint.timeout_seconds, } + if is_adaptive_only(endpoint.model_name): + return ChatAnthropic(**_adaptive_claude_kwargs(endpoint, kwargs)) budget = endpoint.thinking_budget if not budget and endpoint.reasoning_effort: effort_map = {"low": 2048, "medium": 8192, "high": 32768} diff --git a/artemis/services/llm.py b/artemis/services/llm.py index ec91e6fa0..a5dc22c4a 100644 --- a/artemis/services/llm.py +++ b/artemis/services/llm.py @@ -42,9 +42,11 @@ ) from artemis.context import ArtemisContext from artemis.data_engine.trace import CURRENT_TRACE_ID, DataEngineCallbackHandler +from artemis.llm.claude import is_adaptive_only from artemis.llm.google import is_google_chat_model, is_google_provider from artemis.llm.reliability import ( CircuitBreaker, + Failure, FailureCategory, LLMCallError, LLMExhaustedError, @@ -594,6 +596,16 @@ def with_config(self, *args, **kwargs): ) def with_structured_output(self, *args, **kwargs): + # ChatAnthropic defaults to forced tool calling, which adaptive-only + # Claude models (Sonnet 5.5, Opus 5.5, ...) reject with a 400; native + # structured outputs (output_config.format) work on all of them. + if ( + "method" not in kwargs + and self.endpoint is not None + and ModelProvider.from_string(self.endpoint.provider) == ModelProvider.ANTHROPIC + and is_adaptive_only(self.endpoint.model_name) + ): + kwargs["method"] = "json_schema" if hasattr(self.base_model, "with_structured_output"): return RobustChatModelWrapper( self.base_model.with_structured_output(*args, **kwargs), @@ -631,6 +643,25 @@ def _traced_call(self, args: tuple, kwargs: dict) -> tuple[tuple, dict, Any]: args, kwargs = _inject_parent_trace_id(trace_id, *args, **kwargs) return args, kwargs, trace_id + @staticmethod + def _raise_on_refusal(response) -> None: + """Turns a model refusal into a typed error instead of an empty reply. + + Claude's safety classifiers decline with HTTP 200, ``stop_reason: + "refusal"`` and no content. Returned as-is, the agent loop sees a turn + without a tool call, nudges and re-sends the same prompt, which is + refused again on every turn. Raising lets the fallback chain try the + next endpoint and otherwise fails the task with the reason. + """ + meta = getattr(response, "response_metadata", None) + if isinstance(meta, dict) and meta.get("stop_reason") == "refusal": + model = meta.get("model_name") or meta.get("model") or "the model" + raise LLMPermanentError( + f"{model} declined the request (stop_reason 'refusal'): its safety" + " classifier flagged the prompt.", + failure=Failure(FailureCategory.REFUSAL, retryable=False, should_fallback=True), + ) + def _meter_usage(self, response) -> None: """Record measured token usage for this call (M0 metering, best-effort).""" engine = self.ctx.data_engine if self.ctx else None @@ -646,6 +677,7 @@ async def ainvoke(self, *args, **kwargs): endpoint_key=self._endpoint_key(), ) self._meter_usage(response) + self._raise_on_refusal(response) return response async def complete(self, *args, **kwargs): @@ -666,6 +698,7 @@ async def complete(self, *args, **kwargs): endpoint_key=self._endpoint_key(), ) self._meter_usage(response) + self._raise_on_refusal(response) return response def _emit_stream_delta(self, stream_exec_id, chunk) -> None: diff --git a/tests/unit/test_claude_models.py b/tests/unit/test_claude_models.py new file mode 100644 index 000000000..2fbe7fb64 --- /dev/null +++ b/tests/unit/test_claude_models.py @@ -0,0 +1,257 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Request shape for adaptive-only Claude models (Sonnet 5.5, Opus 5.5, ...).""" + +import json +from unittest.mock import MagicMock + +import anthropic +import httpx +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage +import pytest + +from artemis.llm.claude import ( + THINKING_BINDING_BETA, + binds_thinking_blocks, + claude_version, + effort_for, + is_adaptive_only, +) +from artemis.llm.router import ModelEndpoint, ModelFactory, ModelProvider + + +@pytest.mark.parametrize( + "model, expected", + [ + ("claude-sonnet-5-5", ("sonnet", (5, 5))), + ("anthropic.claude-opus-4-8", ("opus", (4, 8))), + ("claude-sonnet-4-5-20250929", ("sonnet", (4, 5))), + ("claude-opus-4-20250514", ("opus", (4, 0))), + ("claude-fable-5", ("fable", (5, 0))), + ("claude-3-7-sonnet-20250219", None), + ("gpt-4o", None), + (None, None), + ], +) +def test_claude_version(model, expected): + assert claude_version(model) == expected + + +@pytest.mark.parametrize( + "model, adaptive_only, binds", + [ + ("claude-sonnet-5-5", True, True), + ("claude-opus-5-5", True, True), + ("claude-haiku-5-5", True, True), + ("claude-fable-5-1", True, True), + ("claude-fable-5", True, False), + ("claude-opus-5", True, False), + ("claude-sonnet-5", True, False), + ("claude-opus-4-7", True, False), + ("claude-opus-4-6", False, False), + ("claude-sonnet-4-6", False, False), + ("claude-haiku-4-5", False, False), + ("claude-3-7-sonnet-20250219", False, False), + ], +) +def test_capabilities(model, adaptive_only, binds): + assert is_adaptive_only(model) is adaptive_only + assert binds_thinking_blocks(model) is binds + + +def test_effort_for(): + assert effort_for("HIGH", 2048) == "high" + assert effort_for("bogus", None) is None + assert effort_for(None, 1024) == "low" + assert effort_for(None, 8192) == "medium" + assert effort_for(None, 32768) == "high" + assert effort_for(None, None) is None + + +def _capture_request(model): + """Sends one tool-use continuation through ``model`` and returns the wire request.""" + captured = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["headers"] = dict(request.headers) + captured["body"] = json.loads(request.content) + return httpx.Response( + 200, + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": model.model, + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + + model.__dict__["_client"] = anthropic.Anthropic( + api_key="sk-ant-test", + http_client=httpx.Client(transport=httpx.MockTransport(handler)), + ) + history = [ + HumanMessage("open the app"), + AIMessage( + content=[ + {"type": "thinking", "thinking": "", "signature": "sig"}, + {"type": "tool_use", "id": "t1", "name": "tap", "input": {}}, + ], + tool_calls=[{"name": "tap", "args": {}, "id": "t1"}], + ), + ToolMessage("tapped", tool_call_id="t1"), + ] + model.invoke(history) + return captured["body"], captured["headers"] + + +def test_sonnet_5_5_request_drops_stale_thinking_blocks(): + """Sonnet 5.5: no temperature, budget mapped to effort, drop_block enabled.""" + endpoint = ModelEndpoint( + provider=ModelProvider.ANTHROPIC, + model_name="claude-sonnet-5-5", + api_key="sk-ant-test", + thinking_budget=8192, + ) + body, headers = _capture_request(ModelFactory.create_model(endpoint)) + + assert "temperature" not in body + assert body["thinking"] == { + "type": "adaptive", + "block_binding": {"prefix_mismatch_behavior": "drop_block"}, + } + assert body["output_config"] == {"effort": "medium"} + assert THINKING_BINDING_BETA in headers["anthropic-beta"] + # The earlier thinking block is still replayed unchanged. + assert body["messages"][1]["content"][0] == { + "type": "thinking", + "thinking": "", + "signature": "sig", + } + + +def test_sonnet_5_5_defaults_send_no_sampling_or_budget(): + endpoint = ModelEndpoint( + provider=ModelProvider.ANTHROPIC, model_name="claude-sonnet-5-5", api_key="sk-ant-test" + ) + body, _ = _capture_request(ModelFactory.create_model(endpoint)) + assert "temperature" not in body + assert "budget_tokens" not in body["thinking"] + assert "output_config" not in body + assert body["max_tokens"] == 16000 + + +def test_opus_4_8_effort_uses_adaptive_thinking_without_binding(): + endpoint = ModelEndpoint( + provider=ModelProvider.ANTHROPIC, + model_name="claude-opus-4-8", + api_key="sk-ant-test", + reasoning_effort="high", + ) + body, headers = _capture_request(ModelFactory.create_model(endpoint)) + assert "temperature" not in body + assert body["thinking"] == {"type": "adaptive"} + assert body["output_config"] == {"effort": "high"} + assert "anthropic-beta" not in headers + + +def test_opus_4_8_without_effort_leaves_thinking_unset(): + endpoint = ModelEndpoint( + provider=ModelProvider.ANTHROPIC, model_name="claude-opus-4-8", api_key="sk-ant-test" + ) + model = ModelFactory.create_model(endpoint) + assert model.thinking is None + assert model.temperature is None + + +def test_legacy_claude_keeps_budget_tokens(): + endpoint = ModelEndpoint( + provider=ModelProvider.ANTHROPIC, + model_name="claude-sonnet-4-6", + api_key="sk-ant-test", + reasoning_effort="medium", + ) + model = ModelFactory.create_model(endpoint) + assert model.thinking == {"type": "enabled", "budget_tokens": 8192} + assert model.temperature == 1.0 + + +@pytest.mark.parametrize( + "model_name, expected_method", + [("claude-sonnet-5-5", "json_schema"), ("claude-sonnet-4-6", None)], +) +def test_structured_output_avoids_forced_tool_choice(model_name, expected_method): + """Adaptive-only Claude models reject forced tool_choice, so use json_schema.""" + from artemis.services.llm import RobustChatModelWrapper + + base = MagicMock() + endpoint = ModelEndpoint(provider=ModelProvider.ANTHROPIC, model_name=model_name) + RobustChatModelWrapper(base, None, endpoint=endpoint).with_structured_output(dict) + assert base.with_structured_output.call_args.kwargs.get("method") == expected_method + + +def test_structured_output_keeps_explicit_method(): + from artemis.services.llm import RobustChatModelWrapper + + base = MagicMock() + endpoint = ModelEndpoint(provider=ModelProvider.ANTHROPIC, model_name="claude-sonnet-5-5") + RobustChatModelWrapper(base, None, endpoint=endpoint).with_structured_output( + dict, method="function_calling" + ) + assert base.with_structured_output.call_args.kwargs["method"] == "function_calling" + + +def _wrapper_returning(message): + from unittest.mock import AsyncMock + + from artemis.services.llm import RobustChatModelWrapper + + base = MagicMock() + base.ainvoke = AsyncMock(return_value=message) + endpoint = ModelEndpoint(provider=ModelProvider.ANTHROPIC, model_name="claude-sonnet-5-5") + return RobustChatModelWrapper(base, None, endpoint=endpoint) + + +@pytest.mark.asyncio +async def test_refusal_raises_typed_error_that_allows_fallback(): + """A refusal (HTTP 200, empty content) must not reach the agent loop as a silent turn.""" + from artemis.llm.reliability import FailureCategory, LLMPermanentError + + refused = AIMessage( + content=[], + response_metadata={"stop_reason": "refusal", "model_name": "claude-sonnet-5-5"}, + ) + with pytest.raises(LLMPermanentError) as excinfo: + await _wrapper_returning(refused).ainvoke([HumanMessage("hi")]) + assert excinfo.value.failure.category is FailureCategory.REFUSAL + assert excinfo.value.failure.should_fallback + assert not excinfo.value.failure.retryable + + +@pytest.mark.asyncio +async def test_normal_reply_passes_through(): + reply = AIMessage(content="ok", response_metadata={"stop_reason": "end_turn"}) + assert await _wrapper_returning(reply).ainvoke([HumanMessage("hi")]) is reply + + +def test_reasoning_reminder_does_not_ask_for_internal_thinking(): + """Sonnet 5.5 refuses prompts that ask it to surface its thinking (reasoning_extraction).""" + from artemis.agents.operator.prompts import REASONING_REMINDER + + assert "thinking" not in REASONING_REMINDER.lower() + assert "reasoning" not in REASONING_REMINDER.lower()