From 43130d7dc8abfdc94e004f3d41de094be796b3a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 13:32:48 +0000 Subject: [PATCH 1/3] chore(deps): bump mcp from 2.0.0 to 2.2.0 Bumps [mcp](https://github.com/modelcontextprotocol/python-sdk) from 2.0.0 to 2.2.0. - [Release notes](https://github.com/modelcontextprotocol/python-sdk/releases) - [Changelog](https://github.com/modelcontextprotocol/python-sdk/blob/main/RELEASE.md) - [Commits](https://github.com/modelcontextprotocol/python-sdk/compare/v2.0.0...v2.2.0) --- updated-dependencies: - dependency-name: mcp dependency-version: 2.2.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- uv.lock | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b0eee33539..23ee12fd1d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ dependencies = [ "tomli-w>=1.0.0", "renderers[multimodal]>=0.1.13.dev17", "typing_extensions>=4.12.2", - "mcp==2.0.0", + "mcp==2.2.0", ] [dependency-groups] diff --git a/uv.lock b/uv.lock index ef901111b2..838ccf2d60 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 4 +revision = 5 requires-python = ">=3.11, <3.15" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -21,15 +21,15 @@ exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for exclude-newer-span = "P7D" [options.exclude-newer-package] -prime-pydantic-config = false +certifi = false harbor = "2026-10-08T00:00:00Z" -ty = "2026-07-28T00:00:00Z" -ruff = "2026-07-28T00:00:00Z" +prime-pydantic-config = false prime-runs = false -prime-tunnel = false -certifi = false prime-sandboxes = false +prime-tunnel = false renderers = false +ruff = "2026-07-28T00:00:00Z" +ty = "2026-07-28T00:00:00Z" [[package]] name = "aiofile" @@ -2278,7 +2278,7 @@ wheels = [ [[package]] name = "mcp" -version = "2.0.0" +version = "2.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -2296,22 +2296,22 @@ dependencies = [ { name = "typing-inspection" }, { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/74/33/32d4dff2c95bb5d897c3ef4c83649a08996b17b58f0a326d2495d4c81179/mcp-2.0.0.tar.gz", hash = "sha256:0f440e735c13ece8bb19bc62cf0b86f4313448432fbb77d35e14034f4e050728", size = 1662284, upload-time = "2026-07-28T13:45:32.346Z" } +sdist = { url = "https://files.pythonhosted.org/packages/76/31/ac54fb0fdd5b37de704486e288bba4fbbb463f24cfcfedbede407b854513/mcp-2.2.0.tar.gz", hash = "sha256:2dc37ecb1974becdcebdbf7561e7c15a07dbbf20ba21ba16c3593b3038b3afbd", size = 4084129, upload-time = "2026-09-07T16:06:23.439Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/67/72/7d7897418912c1d12e87556630dfb7bf0eac71160e9bef8b447960804ee3/mcp-2.0.0-py3-none-any.whl", hash = "sha256:1cb4c75d2d2c7b8c1d756355e5d82a39f2822cc7f13e22a2051d7ca3592349d6", size = 349980, upload-time = "2026-07-28T13:45:28.853Z" }, + { url = "https://files.pythonhosted.org/packages/1b/ff/8e7eade68b8a28f7da0ed1085544341b51f9c935dbf6b95c76b7edfea6a0/mcp-2.2.0-py3-none-any.whl", hash = "sha256:bde982589473a060ae145e3406e9a5333fe538c97229ba841f5a7f92be004f81", size = 365656, upload-time = "2026-09-07T16:06:19.711Z" }, ] [[package]] name = "mcp-types" -version = "2.0.0" +version = "2.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bb/56/9b8e1c152f61f6c6b07c4b5896c88c7d0ae90bac6ee6306f852fcc5c1eb0/mcp_types-2.0.0.tar.gz", hash = "sha256:d7d939b9285c9961ae8866ba75ef85da34d12bafe276efbf4eb6a131786d8379", size = 66632, upload-time = "2026-07-28T13:45:33.804Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/91/762d7755d971aff8a28d75f7961656148edf27875c8026e6385aaab08ae7/mcp_types-2.2.0.tar.gz", hash = "sha256:d3ed53703ddd10d9c6399f29d322bb66f3f67ab41348ac8556ba23e07fedefad", size = 65892, upload-time = "2026-09-07T16:06:25.187Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f5/4c/c78d78c3d52b0ac594ad7cc8ef5972adfe070e3597a8a4c6ce0cd39196ea/mcp_types-2.0.0-py3-none-any.whl", hash = "sha256:6b2de797ca2797f568b79529e1b25948e34de511bcc0bd82fef1039a6d1b8eb0", size = 69649, upload-time = "2026-07-28T13:45:30.713Z" }, + { url = "https://files.pythonhosted.org/packages/8f/d7/6ffba5d8cd5dd9b8a19478875c50e04945314ba5074e84d749283f27f62d/mcp_types-2.2.0-py3-none-any.whl", hash = "sha256:ea476b73ee86709ab5abc9452385ed36cc05907e582355622e294595c9a04f13", size = 69106, upload-time = "2026-09-07T16:06:21.461Z" }, ] [[package]] @@ -5173,6 +5173,7 @@ wheels = [ [[package]] name = "verifiers" source = { editable = "." } +default-groups = ["dev", "examples"] dependencies = [ { name = "aiohttp" }, { name = "anthropic" }, From 58279abcb5d2543e7d364a694f1e55afdea34c7f Mon Sep 17 00:00:00 2001 From: hallerite Date: Mon, 26 Oct 2026 09:25:42 +0000 Subject: [PATCH 2/3] fix(v1): keep tool exception text for the model on mcp 2.2 mcp 2.1 (modelcontextprotocol/python-sdk#3314) treats any exception from a tool other than ToolError/MCPError as a crash and sends the client only "Error executing tool ". Our tools raise to tell the model what went wrong (e.g. general-agent's "Order X not found"), so re-raise those as ToolError in the toolset wrapper, which keeps the 2.0 text exactly. Adds a tool-error integration cell: a tool that raises a stamped message, rewarded only if the stamp reaches the trace. Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/integration/fixtures/tool_error.py | 64 ++++++++++++++++++++++++ tests/integration/test_tool_error.py | 12 +++++ verifiers/v1/mcp/server.py | 16 ++++-- 3 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 tests/integration/fixtures/tool_error.py create mode 100644 tests/integration/test_tool_error.py diff --git a/tests/integration/fixtures/tool_error.py b/tests/integration/fixtures/tool_error.py new file mode 100644 index 0000000000..a18a1b08d3 --- /dev/null +++ b/tests/integration/fixtures/tool_error.py @@ -0,0 +1,64 @@ +"""tool-error (v1, MCP tool): a tool's exception text reaches the model. + +Task tools raise to tell the model what went wrong (an unknown id, an invalid argument), +so the exception's message must come back as the tool result. The tool always raises +with a token the prompt never reveals, so the reward is 1.0 only if the error text +reached the trace. +""" + +import verifiers.v1 as vf +from verifiers.v1.types import content_text + +ERROR_TOKEN = "err-5c2" # only the raised message carries this + + +class OrdersToolset(vf.Toolset[vf.ToolsetConfig]): + TOOL_PREFIX = "orders" + + @vf.tool + def lookup(self, order_id: str) -> str: + """Look up an order by its id.""" + raise ValueError(f"Order {order_id} not found [{ERROR_TOKEN}]") + + +class ToolErrorTaskConfig(vf.TaskConfig): + tools: vf.ToolsetConfig = vf.ToolsetConfig() + + +class ToolErrorTask(vf.Task[vf.TaskData, vf.State, ToolErrorTaskConfig]): + @classmethod + def toolsets(cls, config: ToolErrorTaskConfig) -> list[vf.Toolset]: + return [OrdersToolset(config.tools)] + + @vf.reward(weight=1.0) + async def error_reported(self, trace: vf.Trace) -> float: + results = (content_text(m.content) for m in trace.tool_messages) + return float(any(ERROR_TOKEN in r for r in results)) + + +class ToolErrorConfig(vf.TasksetConfig): + task: ToolErrorTaskConfig = ToolErrorTaskConfig() + + +class ToolErrorTaskset(vf.Taskset[ToolErrorTask, ToolErrorConfig]): + def load(self) -> list[ToolErrorTask]: + return [ + ToolErrorTask( + vf.TaskData( + idx=0, + prompt=( + "Call the `lookup` tool from the `orders` MCP server with the order " + 'id "A-17", then reply with exactly what it returns inside ' + " tags." + ), + ), + self.config.task, + ) + ] + + +__all__ = ["ToolErrorTaskset"] + + +if __name__ == "__main__": + OrdersToolset.run() diff --git a/tests/integration/test_tool_error.py b/tests/integration/test_tool_error.py new file mode 100644 index 0000000000..2d87a7c34a --- /dev/null +++ b/tests/integration/test_tool_error.py @@ -0,0 +1,12 @@ +"""A tool that raises: its exception text reaches the model as the tool result +(tool-error). The server side is the same for every harness, so the chat loop covers it.""" + + +async def test_tool_error(run): + """A tool's exception message comes back to the model, not just that the call failed.""" + (episode,) = await run("tool-error", max_turns=4) + (trace,) = episode.traces + assert trace.ok, [error.message for error in trace.errors] + assert trace.reward == 1.0, [ + message.model_dump_json(exclude_none=True) for message in trace.tool_messages + ] diff --git a/verifiers/v1/mcp/server.py b/verifiers/v1/mcp/server.py index 2ea101e338..85c0e54588 100644 --- a/verifiers/v1/mcp/server.py +++ b/verifiers/v1/mcp/server.py @@ -233,9 +233,19 @@ async def wrapper(*args, **kwargs): token = _call_state.set(state) try: before = self._state_adapter.dump_json(state) if sync_state else None - result = fn(*args, **kwargs) - if inspect.isawaitable(result): - result = await result + try: + result = fn(*args, **kwargs) + if inspect.isawaitable(result): + result = await result + except Exception as e: + from mcp.server.mcpserver.exceptions import ToolError + from mcp.shared.exceptions import MCPError + + if isinstance(e, (ToolError, MCPError)): + raise + # MCPServer withholds an unexpected exception's text from the client, but + # tools raise to tell the model what went wrong. + raise ToolError(str(e)) from e if before is not None: await self._push_state(before) return result From ef809b5dbdd00ebec58690741a47df000cc47a86 Mon Sep 17 00:00:00 2001 From: hallerite Date: Mon, 26 Oct 2026 09:27:33 +0000 Subject: [PATCH 3/3] test: run the tool-error cell in the Multi-Turn Tool job The integration matrix lists its files, so a new file would not run in CI. Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/integration/test_multi_turn_tool.py | 13 ++++++++++++- tests/integration/test_tool_error.py | 12 ------------ 2 files changed, 12 insertions(+), 13 deletions(-) delete mode 100644 tests/integration/test_tool_error.py diff --git a/tests/integration/test_multi_turn_tool.py b/tests/integration/test_multi_turn_tool.py index b3a61c7504..7c02a416f3 100644 --- a/tests/integration/test_multi_turn_tool.py +++ b/tests/integration/test_multi_turn_tool.py @@ -2,7 +2,8 @@ never reveals (echo-tool), so reward 1.0 proves the tool ran. Every MCP harness reaches it colocated across the harness x runtime grid; the chat loop reaches it across its placements (colocated in the harness's box, in a Compose service beside it, or its own -runtime) on the runtime x runtime grid.""" +runtime) on the runtime x runtime grid. A tool that raises (tool-error) reports its message +to the model; that is server-side, so the chat loop covers it.""" import pytest from grid import assert_one_branch, grid, placements @@ -64,6 +65,16 @@ async def test_tool_placement(run, harness_runtime, tool_runtime): assert "message" in echo_tool.parameters.get("properties", {}) +async def test_tool_error(run): + """A tool's exception message comes back to the model, not just that the call failed.""" + (episode,) = await run("tool-error", max_turns=4) + (trace,) = episode.traces + assert trace.ok, [error.message for error in trace.errors] + assert trace.reward == 1.0, [ + message.model_dump_json(exclude_none=True) for message in trace.tool_messages + ] + + @pytest.mark.parametrize( "placement", [ diff --git a/tests/integration/test_tool_error.py b/tests/integration/test_tool_error.py deleted file mode 100644 index 2d87a7c34a..0000000000 --- a/tests/integration/test_tool_error.py +++ /dev/null @@ -1,12 +0,0 @@ -"""A tool that raises: its exception text reaches the model as the tool result -(tool-error). The server side is the same for every harness, so the chat loop covers it.""" - - -async def test_tool_error(run): - """A tool's exception message comes back to the model, not just that the call failed.""" - (episode,) = await run("tool-error", max_turns=4) - (trace,) = episode.traces - assert trace.ok, [error.message for error in trace.errors] - assert trace.reward == 1.0, [ - message.model_dump_json(exclude_none=True) for message in trace.tool_messages - ]