[TRTLLM-17263][refactor] Converge the three Anthropic orchestrations into one serving class

.pre-commit-config.yaml

@@ -417,6 +417,9 @@ common-files: &common_files |
         tensorrt_llm/scaffolding/worker.py |
         tensorrt_llm/scheduling_params.py |
         tensorrt_llm/serve/__init__.py |
+        tensorrt_llm/serve/_base/msgspec_route.py |
+        tensorrt_llm/serve/_base/sse.py |
+        tensorrt_llm/serve/api/anthropic/serving.py |
         tensorrt_llm/serve/chat_utils.py |
         tensorrt_llm/serve/cluster_storage.py |
         tensorrt_llm/serve/disagg_auto_scaling.py |
@@ -1144,6 +1147,9 @@ legacy-files: &legacy_files |
         tensorrt_llm/scaffolding/worker.py |
         tensorrt_llm/scheduling_params.py |
         tensorrt_llm/serve/__init__.py |
+        tensorrt_llm/serve/_base/msgspec_route.py |
+        tensorrt_llm/serve/_base/sse.py |
+        tensorrt_llm/serve/api/anthropic/serving.py |
         tensorrt_llm/serve/chat_utils.py |
         tensorrt_llm/serve/cluster_storage.py |
         tensorrt_llm/serve/disagg_auto_scaling.py |
@@ -1620,6 +1626,7 @@ repos:
         # Run mypy with no explicit paths so it type-checks the set declared in
         # pyproject.toml [tool.mypy] `files` (single source of truth). This
         # `files:` pattern only decides when the hook triggers — any change under
-        # the sampler source/test dirs.
+        # the sampler source/test dirs or the serving frontends. A PR that adds
+        # a typed module elsewhere extends this pattern in the same PR.
         pass_filenames: false
-        files: ^(tensorrt_llm/_torch/pyexecutor/sampler/|tests/unittest/_torch/sampler/)
+        files: ^(tensorrt_llm/_torch/pyexecutor/sampler/|tests/unittest/_torch/sampler/|tensorrt_llm/serve/|tensorrt_llm/commands/serve_cli/)

jenkins/L0_MergeRequest.groovy

@@ -1948,6 +1948,13 @@ def getMultiGpuFileChanged(pipeline, testFilter, globalVars)
         "tensorrt_llm/models/llama/",
         "tensorrt_llm/parameter.py",
         "tensorrt_llm/serve/__init__.py",
+        // _MsgspecRoute decodes the disaggregated frontend's msgpack bodies.
+        "tensorrt_llm/serve/_base/",
+        // The disaggregated /v1/messages runs through the Anthropic serving class.
+        "tensorrt_llm/serve/api/anthropic/api_router.py",
+        "tensorrt_llm/serve/api/anthropic/serving.py",
+        // /health is the disaggregated readiness probe.
+        "tensorrt_llm/serve/api/ops/",
         "tensorrt_llm/serve/cluster_storage.py",
         "tensorrt_llm/serve/disagg_auto_scaling.py",
         "tensorrt_llm/serve/extensions/",

legacy-files.txt

@@ -409,6 +409,9 @@ tensorrt_llm/scaffolding/task_collection.py
 tensorrt_llm/scaffolding/worker.py
 tensorrt_llm/scheduling_params.py
 tensorrt_llm/serve/__init__.py
+tensorrt_llm/serve/_base/msgspec_route.py
+tensorrt_llm/serve/_base/sse.py
+tensorrt_llm/serve/api/anthropic/serving.py
 tensorrt_llm/serve/chat_utils.py
 tensorrt_llm/serve/cluster_storage.py
 tensorrt_llm/serve/disagg_auto_scaling.py

pyproject.toml

@@ -474,6 +474,9 @@ exclude = [
     "tensorrt_llm/scaffolding/worker.py",
     "tensorrt_llm/scheduling_params.py",
     "tensorrt_llm/serve/__init__.py",
+    "tensorrt_llm/serve/_base/msgspec_route.py",
+    "tensorrt_llm/serve/_base/sse.py",
+    "tensorrt_llm/serve/api/anthropic/serving.py",
     "tensorrt_llm/serve/chat_utils.py",
     "tensorrt_llm/serve/cluster_storage.py",
     "tensorrt_llm/serve/disagg_auto_scaling.py",
@@ -862,6 +865,13 @@ files = [
     "tensorrt_llm/_torch/pyexecutor/sampler/token_ban.py",
     "tensorrt_llm/_torch/pyexecutor/sampler/ops/flashinfer.py",
     "tensorrt_llm/_torch/pyexecutor/sampler/ops/vanilla.py",
+    "tensorrt_llm/serve/_base/chat_backend.py",
+    "tensorrt_llm/serve/_base/errors.py",
+    "tensorrt_llm/serve/_base/request_context.py",
+    "tensorrt_llm/serve/_base/routing.py",
+    "tensorrt_llm/serve/api/anthropic/api_router.py",
+    "tensorrt_llm/serve/api/ops/system/api_router.py",
+    "tensorrt_llm/serve/api/ops/system/serving.py",
     "tests/unittest/_torch/sampler/test_torch_sampler.py",
     "tests/unittest/_torch/sampler/test_token_ban.py",
     "tests/unittest/_torch/sampler/test_beam_search.py",

ruff-legacy-baseline.json

@@ -528,6 +528,9 @@
   "tensorrt_llm/scaffolding/task.py": {
     "F821": 1
   },
+  "tensorrt_llm/serve/_base/sse.py": {
+    "F821": 1
+  },
   "tensorrt_llm/serve/disagg_auto_scaling.py": {
     "D205": 2,
     "D212": 2

ruff-legacy.toml

@@ -426,6 +426,9 @@ include = [
     "tensorrt_llm/scaffolding/worker.py",
     "tensorrt_llm/scheduling_params.py",
     "tensorrt_llm/serve/__init__.py",
+    "tensorrt_llm/serve/_base/msgspec_route.py",
+    "tensorrt_llm/serve/_base/sse.py",
+    "tensorrt_llm/serve/api/anthropic/serving.py",
     "tensorrt_llm/serve/chat_utils.py",
     "tensorrt_llm/serve/cluster_storage.py",
     "tensorrt_llm/serve/disagg_auto_scaling.py",

tensorrt_llm/commands/AGENTS.md

@@ -0,0 +1,6 @@
+<!--
+SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+-->
+
+Rules for the trtllm-serve CLI (`commands/serve.py`) are in `tensorrt_llm/serve/AGENTS.md`.

tensorrt_llm/commands/CLAUDE.md

@@ -0,0 +1,2 @@
+# In ./CLAUDE.md
+@AGENTS.md

tensorrt_llm/serve/AGENTS.md

@@ -0,0 +1,107 @@
+<!--
+SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+-->
+
+# AGENTS.md - tensorrt_llm/serve
+
+Scope: `tensorrt_llm/serve/` and everything under it. Supplements the repo-root `AGENTS.md`
+and `tests/AGENTS.md`; `tensorrt_llm/commands/AGENTS.md` points here for the serve CLI.
+Where a check enforces a rule, the rule names it; a failing check says what to change.
+Name files and symbols here, never line numbers.
+
+## Where code goes
+
+- New endpoint: a router and a serving class under `serve/api/<dialect>/<endpoint>/`
+  (`api_router.py` + `serving.py`). Create the router with `make_router()`
+  (`serve/_base/routing.py`) and add a `RouterSpec` to `ROUTERS`. Never add a method or a
+  route to `OpenAIServer`. Check: `tests/unittest/api_stability/test_serve_routes.py`.
+- Attach a router with `app.include_router(router)` and no `prefix=`; give the prefix to
+  `make_router(prefix=...)` instead. The composition root sets each serving object on
+  `app.state.<bucket>_serving_<endpoint>` before attaching; routers read it from
+  `request.app.state`, never from a closure.
+- Anthropic Messages: `serve/api/anthropic/`. One `AnthropicMessagesServing` serves both
+  servers through a `ChatBackend`; the aggregated routes are in `api_router.py`, the
+  disaggregated frontend keeps its own `/v1/messages` route and calls the same class.
+  Check: `tests/unittest/serve/api/anthropic/test_messages_behavior.py`.
+- Model-specific behavior: a hook on `ServingExtension` (`serve/serving_extensions.py`,
+  `serve/extensions/`). Do not branch on a model name in a handler.
+- Prompt rendering: `serve/render/` only. Handlers do not call `apply_chat_template`.
+- Request and response models: a new field needs a row in
+  `tests/unittest/api_stability/references/trtllm_serve_api.yaml`.
+  Check: `tests/unittest/api_stability/test_serve_api.py`.
+- Routes: every server configuration's routes (path, method, route class, and the order of
+  routes that can match the same request) are pinned in
+  `tests/unittest/api_stability/references/trtllm_serve_routes.yaml`. A route change
+  regenerates it and needs the API review committee.
+  Check: `tests/unittest/api_stability/test_serve_routes.py`.
+- Middleware order matters: `add_middleware` inserts at the front, so the last one added is
+  outermost. Do not reorder. Check: `tests/unittest/serve/test_middleware_order.py`.
+- Import directions between layers are tests; a PR that creates a package adds its contract.
+  Check: `tests/unittest/serve/test_layering.py`.
+- VisualGen serve tests live under a path that contains `visual_gen/`
+  (`tests/unittest/serve/api/visual_gen/`), or CI selection misses them.
+
+## Moving code
+
+- A move PR only moves code: no reformatting, no logic change.
+- A move PR is redone on the latest `main` right before merge instead of being rebased,
+  and its PR body lists every moved definition with its old and new location, and every
+  definition the PR changes on purpose.
+- In the same PR update the multi-GPU file list (`getMultiGpuFileChanged` in
+  `jenkins/L0_MergeRequest.groovy`), `.github/CODEOWNERS` and `legacy-files.txt`. A new file
+  takes the formatting group of the file its code comes from.
+- Old import paths: a move deletes the old module (or the moved names) without a shim and
+  rewrites every import of it in the repository, tests included; module paths written as
+  strings (patch targets, subprocess command lines, `_LAZY_ATTRS` in `serve/__init__.py`) are
+  fixed by hand. The PR body lists "old path → new path" for the release notes. Only five old
+  paths keep a shim, because code outside the repository imports them:
+  `tensorrt_llm/llmapi/reasoning_parser.py`, `serve/tool_parser/`,
+  `serve/openai_disagg_server.py`, `serve/openai_server.py` and `commands/serve.py`. A shim
+  re-exports public names only, lists them in `__all__` (or `_LAZY_NAMES` in
+  `commands/serve.py`), warns once with `FutureWarning`, and has its rows in
+  `tests/unittest/serve/test_shim_identity.py`.
+- `OpenAIDisaggServer` (`serve/openai_disagg_server.py`) is subclassed outside the repository
+  (NeMo-RL). Keep its constructor `(config, req_timeout_secs=..., coordinator_url=None)`,
+  `self.app` with every route registered when `__init__` returns,
+  `__call__(host, port, sockets=None)`, `_wrap_entry_point`, `_handle_exception` and
+  `_perf_metrics_collector.http_exceptions`, or agree the change with the NeMo-RL owners
+  first. Check: `tests/unittest/disaggregated/test_disagg_server_contract.py`.
+- `mock.patch("old.module.name")` does not follow moved code. Grep the tests for every moved
+  name, including `monkeypatch.setattr(module, "name", ...)` and dotted paths split across lines.
+
+## Tests
+
+- Register every new test file in a CI list in the same PR (`tests/AGENTS.md`). Serve tests
+  that need no GPU go in `tests/integration/test_lists/test-db/l0_cpu.yml` and must contain
+  `pytest.mark.cpu_only`, or the CPU stage collects nothing from them.
+- New serve tests mirror the package: `tests/unittest/serve/` for `tensorrt_llm/serve/`.
+- Build servers with the factory and fake generators in `tests/test_common/serve/`
+  (`isolated_serve_env`), never `object.__new__(OpenAIServer)` or
+  `object.__new__(OpenAIDisaggServer)`. Check: `tests/unittest/serve/test_no_object_new.py`.
+- List an app's routes with `effective_routes()` from `tests/test_common/serve/routes.py`, not
+  `app.routes`: FastAPI 0.139 keeps included routers as objects without `.path`.
+- The harmony tests guard whole files with try-imports. After renaming anything they import,
+  run them with `pytest -rs` and check that nothing is skipped as "not importable": a green run
+  can mean every test was skipped.
+
+## Checklists
+
+- Endpoint: package under `serve/api/`; `RouterSpec` in `ROUTERS`; regenerate the route
+  snapshot (API review); multi-GPU list entry if it runs on the disaggregated path; CPU test
+  registered in `tests/integration/test_lists/test-db/l0_cpu.yml`.
+
+## Commands
+
+- All serve CPU tests (the runner exits when `LLM_MODELS_ROOT` is unset):
+  `LLM_MODELS_ROOT=/path/to/models PYTHONPATH=tests python3 -m test_common.serve.cpu_tests`
+- Route snapshot: `pytest -m cpu_only tests/unittest/api_stability/test_serve_routes.py tests/unittest/serve/api/visual_gen/test_routes_visual_gen.py`
+- Regenerate the route snapshot: `PYTHONPATH=tests python3 -m test_common.serve.routes --write`
+- API gates: `pytest tests/unittest/api_stability/test_serve_api.py tests/unittest/api_stability/test_serve_cli.py`
+- Extensions and parsers: `pytest tests/unittest/llmapi/apps/test_serving_extensions.py tests/unittest/llmapi/apps/test_tool_parsers.py`
+- Disaggregated router and coordinator (CPU): `pytest tests/unittest/disaggregated -m cpu_only -q`
+- Which CI stage runs a test: `python scripts/test_to_stage_mapping.py --tests "<test>"`. It
+  matches the text of list entries and does not expand directories: a file listed through its
+  directory maps only when you pass the directory entry, e.g. `--tests "unittest/serve"` for
+  anything under `tests/unittest/serve/`.
+- Tests that load model files need `LLM_MODELS_ROOT` (the harmony tests' autouse fixture does).

tensorrt_llm/serve/CLAUDE.md

@@ -0,0 +1,2 @@
+# In ./CLAUDE.md
+@AGENTS.md

tensorrt_llm/serve/_base/__init__.py

@@ -0,0 +1,15 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Shared serving infrastructure with no dependency on endpoint packages."""

tensorrt_llm/serve/_base/chat_backend.py

@@ -0,0 +1,64 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The chat implementation a wire-dialect serving class runs on."""
+
+from __future__ import annotations
+
+from typing import AsyncIterator, Protocol, Union
+
+from tensorrt_llm.serve._base.errors import ServingError
+from tensorrt_llm.serve._base.request_context import RequestContext
+from tensorrt_llm.serve.openai_protocol import ChatCompletionRequest, ChatCompletionResponse
+
+# Non-streaming: the typed response. Streaming: the OpenAI SSE stream, as str
+# frames (aggregated) or as the worker's raw bytes (disaggregated).
+ChatResult = Union[ChatCompletionResponse, AsyncIterator[str], AsyncIterator[bytes]]
+
+
+class ChatBackend(Protocol):
+    """The chat implementation a wire-dialect serving class runs on.
+
+    The aggregated server implements it over its typed chat core, the
+    disaggregated frontend over its prepare, run and finish steps. Each side
+    keeps its own streamed model name and error-message extraction, which is
+    why both are methods of the backend and not of the serving class.
+    """
+
+    async def create_chat_completion(
+        self, request: ChatCompletionRequest, ctx: RequestContext
+    ) -> ChatResult:
+        """Run one chat completion.
+
+        Non-streaming: the typed response. Streaming: the OpenAI SSE stream, as
+        str frames (aggregated) or the worker's raw bytes (disaggregated); one
+        stream never mixes the two.
+
+        Raises:
+            ServingError: The request was rejected; the HTTP status travels
+                apart from the body.
+            EngineFatalError: The engine failed; SIGINT has been sent.
+        """
+        ...
+
+    def stream_model(self, request: ChatCompletionRequest) -> str:
+        """The model a translated stream announces when no chunk names one."""
+        ...
+
+    def error_message(self, error: ServingError) -> str:
+        """The client-facing message of an error.
+
+        Each backend extracts it from the error body its own way.
+        """
+        ...

tensorrt_llm/serve/_base/errors.py

@@ -0,0 +1,76 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Typed errors of the serving core.
+
+The core raises them; HTTP wrappers and wire dialects render them. ServingError keeps the HTTP
+status apart from the error body, because ErrorResponse.code is not always a status (an
+over-length prompt carries "context_length_exceeded").
+"""
+
+from http import HTTPStatus
+from typing import Union
+
+from tensorrt_llm.executor.utils import (
+    CONTEXT_LENGTH_EXCEEDED_CODE,
+    is_context_length_exceeded_message,
+)
+from tensorrt_llm.serve.openai_protocol import ErrorResponse
+
+
+def error_body(
+    message: str,
+    err_type: str = "BadRequestError",
+    status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
+) -> ErrorResponse:
+    """Build the OpenAI error body for a rejected request.
+
+    ``code`` is the HTTP status, except for an over-length prompt, which is recognized by its
+    text because it crosses the executor IPC boundary as a plain string (RequestError), so no
+    exception type survives to here.
+    """
+    code: Union[int, str] = status_code.value
+    if is_context_length_exceeded_message(message):
+        code = CONTEXT_LENGTH_EXCEEDED_CODE
+    return ErrorResponse(message=message, type=err_type, code=code)
+
+
+class ServingError(Exception):
+    """A request the serving core rejects, with the HTTP status to answer it with.
+
+    Attributes:
+        status: The HTTP status code.
+        body: The OpenAI error body; ``body.code`` is not necessarily ``status``.
+    """
+
+    def __init__(self, status: int, body: ErrorResponse) -> None:
+        super().__init__(body.message)
+        self.status = status
+        self.body = body
+
+
+def serving_error(
+    message: str,
+    err_type: str = "BadRequestError",
+    status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
+) -> ServingError:
+    """Return the ServingError for what OpenAIServer.create_error_response would render."""
+    return ServingError(status_code.value, error_body(message, err_type, status_code))
+
+
+class EngineFatalError(Exception):
+    """The engine failed fatally; SIGINT has been sent and the server is shutting down.
+
+    It carries no response; each caller decides what to answer.
+    """