diff --git a/pyproject.toml b/pyproject.toml index b0eee33539..23ee12fd1d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ dependencies = [ "tomli-w>=1.0.0", "renderers[multimodal]>=0.1.13.dev17", "typing_extensions>=4.12.2", - "mcp==2.0.0", + "mcp==2.2.0", ] [dependency-groups] diff --git a/tests/integration/fixtures/tool_error.py b/tests/integration/fixtures/tool_error.py new file mode 100644 index 0000000000..a18a1b08d3 --- /dev/null +++ b/tests/integration/fixtures/tool_error.py @@ -0,0 +1,64 @@ +"""tool-error (v1, MCP tool): a tool's exception text reaches the model. + +Task tools raise to tell the model what went wrong (an unknown id, an invalid argument), +so the exception's message must come back as the tool result. The tool always raises +with a token the prompt never reveals, so the reward is 1.0 only if the error text +reached the trace. +""" + +import verifiers.v1 as vf +from verifiers.v1.types import content_text + +ERROR_TOKEN = "err-5c2" # only the raised message carries this + + +class OrdersToolset(vf.Toolset[vf.ToolsetConfig]): + TOOL_PREFIX = "orders" + + @vf.tool + def lookup(self, order_id: str) -> str: + """Look up an order by its id.""" + raise ValueError(f"Order {order_id} not found [{ERROR_TOKEN}]") + + +class ToolErrorTaskConfig(vf.TaskConfig): + tools: vf.ToolsetConfig = vf.ToolsetConfig() + + +class ToolErrorTask(vf.Task[vf.TaskData, vf.State, ToolErrorTaskConfig]): + @classmethod + def toolsets(cls, config: ToolErrorTaskConfig) -> list[vf.Toolset]: + return [OrdersToolset(config.tools)] + + @vf.reward(weight=1.0) + async def error_reported(self, trace: vf.Trace) -> float: + results = (content_text(m.content) for m in trace.tool_messages) + return float(any(ERROR_TOKEN in r for r in results)) + + +class ToolErrorConfig(vf.TasksetConfig): + task: ToolErrorTaskConfig = ToolErrorTaskConfig() + + +class ToolErrorTaskset(vf.Taskset[ToolErrorTask, ToolErrorConfig]): + def load(self) -> list[ToolErrorTask]: + return [ + ToolErrorTask( + vf.TaskData( + idx=0, + prompt=( + "Call the `lookup` tool from the `orders` MCP server with the order " + 'id "A-17", then reply with exactly what it returns inside ' + " tags." + ), + ), + self.config.task, + ) + ] + + +__all__ = ["ToolErrorTaskset"] + + +if __name__ == "__main__": + OrdersToolset.run() diff --git a/tests/integration/test_multi_turn_tool.py b/tests/integration/test_multi_turn_tool.py index b3a61c7504..7c02a416f3 100644 --- a/tests/integration/test_multi_turn_tool.py +++ b/tests/integration/test_multi_turn_tool.py @@ -2,7 +2,8 @@ never reveals (echo-tool), so reward 1.0 proves the tool ran. Every MCP harness reaches it colocated across the harness x runtime grid; the chat loop reaches it across its placements (colocated in the harness's box, in a Compose service beside it, or its own -runtime) on the runtime x runtime grid.""" +runtime) on the runtime x runtime grid. A tool that raises (tool-error) reports its message +to the model; that is server-side, so the chat loop covers it.""" import pytest from grid import assert_one_branch, grid, placements @@ -64,6 +65,16 @@ async def test_tool_placement(run, harness_runtime, tool_runtime): assert "message" in echo_tool.parameters.get("properties", {}) +async def test_tool_error(run): + """A tool's exception message comes back to the model, not just that the call failed.""" + (episode,) = await run("tool-error", max_turns=4) + (trace,) = episode.traces + assert trace.ok, [error.message for error in trace.errors] + assert trace.reward == 1.0, [ + message.model_dump_json(exclude_none=True) for message in trace.tool_messages + ] + + @pytest.mark.parametrize( "placement", [ diff --git a/uv.lock b/uv.lock index ef901111b2..838ccf2d60 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 4 +revision = 5 requires-python = ">=3.11, <3.15" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -21,15 +21,15 @@ exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for exclude-newer-span = "P7D" [options.exclude-newer-package] -prime-pydantic-config = false +certifi = false harbor = "2026-10-08T00:00:00Z" -ty = "2026-07-28T00:00:00Z" -ruff = "2026-07-28T00:00:00Z" +prime-pydantic-config = false prime-runs = false -prime-tunnel = false -certifi = false prime-sandboxes = false +prime-tunnel = false renderers = false +ruff = "2026-07-28T00:00:00Z" +ty = "2026-07-28T00:00:00Z" [[package]] name = "aiofile" @@ -2278,7 +2278,7 @@ wheels = [ [[package]] name = "mcp" -version = "2.0.0" +version = "2.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -2296,22 +2296,22 @@ dependencies = [ { name = "typing-inspection" }, { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/74/33/32d4dff2c95bb5d897c3ef4c83649a08996b17b58f0a326d2495d4c81179/mcp-2.0.0.tar.gz", hash = "sha256:0f440e735c13ece8bb19bc62cf0b86f4313448432fbb77d35e14034f4e050728", size = 1662284, upload-time = "2026-07-28T13:45:32.346Z" } +sdist = { url = "https://files.pythonhosted.org/packages/76/31/ac54fb0fdd5b37de704486e288bba4fbbb463f24cfcfedbede407b854513/mcp-2.2.0.tar.gz", hash = "sha256:2dc37ecb1974becdcebdbf7561e7c15a07dbbf20ba21ba16c3593b3038b3afbd", size = 4084129, upload-time = "2026-09-07T16:06:23.439Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/67/72/7d7897418912c1d12e87556630dfb7bf0eac71160e9bef8b447960804ee3/mcp-2.0.0-py3-none-any.whl", hash = "sha256:1cb4c75d2d2c7b8c1d756355e5d82a39f2822cc7f13e22a2051d7ca3592349d6", size = 349980, upload-time = "2026-07-28T13:45:28.853Z" }, + { url = "https://files.pythonhosted.org/packages/1b/ff/8e7eade68b8a28f7da0ed1085544341b51f9c935dbf6b95c76b7edfea6a0/mcp-2.2.0-py3-none-any.whl", hash = "sha256:bde982589473a060ae145e3406e9a5333fe538c97229ba841f5a7f92be004f81", size = 365656, upload-time = "2026-09-07T16:06:19.711Z" }, ] [[package]] name = "mcp-types" -version = "2.0.0" +version = "2.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bb/56/9b8e1c152f61f6c6b07c4b5896c88c7d0ae90bac6ee6306f852fcc5c1eb0/mcp_types-2.0.0.tar.gz", hash = "sha256:d7d939b9285c9961ae8866ba75ef85da34d12bafe276efbf4eb6a131786d8379", size = 66632, upload-time = "2026-07-28T13:45:33.804Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/91/762d7755d971aff8a28d75f7961656148edf27875c8026e6385aaab08ae7/mcp_types-2.2.0.tar.gz", hash = "sha256:d3ed53703ddd10d9c6399f29d322bb66f3f67ab41348ac8556ba23e07fedefad", size = 65892, upload-time = "2026-09-07T16:06:25.187Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f5/4c/c78d78c3d52b0ac594ad7cc8ef5972adfe070e3597a8a4c6ce0cd39196ea/mcp_types-2.0.0-py3-none-any.whl", hash = "sha256:6b2de797ca2797f568b79529e1b25948e34de511bcc0bd82fef1039a6d1b8eb0", size = 69649, upload-time = "2026-07-28T13:45:30.713Z" }, + { url = "https://files.pythonhosted.org/packages/8f/d7/6ffba5d8cd5dd9b8a19478875c50e04945314ba5074e84d749283f27f62d/mcp_types-2.2.0-py3-none-any.whl", hash = "sha256:ea476b73ee86709ab5abc9452385ed36cc05907e582355622e294595c9a04f13", size = 69106, upload-time = "2026-09-07T16:06:21.461Z" }, ] [[package]] @@ -5173,6 +5173,7 @@ wheels = [ [[package]] name = "verifiers" source = { editable = "." } +default-groups = ["dev", "examples"] dependencies = [ { name = "aiohttp" }, { name = "anthropic" }, diff --git a/verifiers/v1/mcp/server.py b/verifiers/v1/mcp/server.py index 2ea101e338..85c0e54588 100644 --- a/verifiers/v1/mcp/server.py +++ b/verifiers/v1/mcp/server.py @@ -233,9 +233,19 @@ async def wrapper(*args, **kwargs): token = _call_state.set(state) try: before = self._state_adapter.dump_json(state) if sync_state else None - result = fn(*args, **kwargs) - if inspect.isawaitable(result): - result = await result + try: + result = fn(*args, **kwargs) + if inspect.isawaitable(result): + result = await result + except Exception as e: + from mcp.server.mcpserver.exceptions import ToolError + from mcp.shared.exceptions import MCPError + + if isinstance(e, (ToolError, MCPError)): + raise + # MCPServer withholds an unexpected exception's text from the client, but + # tools raise to tell the model what went wrong. + raise ToolError(str(e)) from e if before is not None: await self._push_state(before) return result