From 5b71394432eb8bab2364449a0b820ef9c69fd32a Mon Sep 17 00:00:00 2001 From: Abhay Kumar Date: Mon, 26 Oct 2026 22:24:33 +0530 Subject: [PATCH] Docs: vLLM backend needs a vLLM nightly; it supports Qwen3.5 only - README: install the vLLM nightly for the vLLM backend (vLLM 0.32.0 on PyPI predates its decision-model support) and note the GPU requirement (Ampere or newer for non-causal checkpoints such as vllm-sr/d3-edge). - Qwen3.5/3.8 -> Qwen3.5 in the backend docstring, its refusal message and the --backend help: only Qwen3_5Model checkpoints are accepted. - The [vllm] extra's comment says the nightly is needed. --- README.md | 9 ++++++--- pyproject.toml | 2 +- src/decisionformer/cli.py | 2 +- src/decisionformer/vllm_backend.py | 8 ++++---- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 20d000b..71e4746 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Optional extras, combined as `pip install "decisionformer[serve,qlora]"`: |---|---| | `serve` | the `/v1/systemone` API server (FastAPI, uvicorn) | | `qlora` | bitsandbytes, for QLoRA | -| `vllm` | the vLLM serving backend (a vLLM build from 2026-10-15 or later) | +| `vllm` | vLLM, for the vLLM serving backend; Decision 3.0 models need its nightly build (see [Serve as an API](#serve-as-an-api)) | | `fla` | fast Gated DeltaNet kernels for Qwen3.5 backbones | | `data` | datasets, for Hub datasets (`hf:` inputs) and the example converters | @@ -126,10 +126,13 @@ schemas are at `/docs`. Over-long requests are refused with HTTP 422, never trun runs at once; requests that arrive while it is busy are answered together, up to `--max-batch` (16) at a time. For code-readout checkpoints on Qwen3.5 (the Decision 3.0 layout), vLLM can run the model behind the same API. -Other checkpoints fail with a clear error that points to the default backend: +This needs a vLLM nightly build from 2026-10-15 or later (vLLM 0.32.0 on PyPI predates its decision-model support) +and a CUDA GPU, Ampere or newer for non-causal checkpoints such as `vllm-sr/d3-edge`. Other checkpoints fail with a +clear error that points to the default backend: ```bash -pip install "decisionformer[vllm]" +pip install "decisionformer[serve]" +pip install -U vllm --pre --extra-index-url https://wheels.vllm.ai/nightly decisionformer serve vllm-sr/d3-edge --backend vllm ``` diff --git a/pyproject.toml b/pyproject.toml index d22f81b..4566e8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ Dataset = "https://huggingface.co/datasets/FrontiersMind/deciformer-15k" [project.optional-dependencies] qlora = ["bitsandbytes>=0.45"] serve = ["fastapi>=0.110", "uvicorn>=0.29"] -vllm = ["fastapi>=0.110", "uvicorn>=0.29", "vllm>=0.12"] # decision-model support: vLLM nightly from 2026-10-15 +vllm = ["fastapi>=0.110", "uvicorn>=0.29", "vllm>=0.12"] # Decision 3.0 models need a vLLM nightly from 2026-10-15 (0.32.0 predates it) fla = ["flash-linear-attention>=0.5"] # fast Gated DeltaNet kernels for Qwen3.5/3.8 backbones data = ["datasets>=3"] dev = ["pytest>=8", "ruff>=0.6", "mypy>=1.11", "fastapi>=0.110", "uvicorn>=0.29", "httpx>=0.27"] diff --git a/src/decisionformer/cli.py b/src/decisionformer/cli.py index 5c22a3e..4a149c4 100644 --- a/src/decisionformer/cli.py +++ b/src/decisionformer/cli.py @@ -279,7 +279,7 @@ def build_parser() -> argparse.ArgumentParser: serve.add_argument("--host", default="127.0.0.1") serve.add_argument("--port", type=int, default=8000) serve.add_argument("--backend", choices=("transformers", "vllm"), default="transformers", - help="vllm: vLLM computes the readout (Qwen3.5/3.8 code-readout checkpoints only)") + help="vllm: vLLM computes the readout (Qwen3.5 code-readout checkpoints; needs a vLLM nightly)") serve.add_argument("--max-batch", type=int, default=16, help="requests arriving together are answered in one batch of up to this many") serve.add_argument("--api-key-env", default="DECISIONFORMER_API_KEY", diff --git a/src/decisionformer/vllm_backend.py b/src/decisionformer/vllm_backend.py index 0f643ee..dbb0932 100644 --- a/src/decisionformer/vllm_backend.py +++ b/src/decisionformer/vllm_backend.py @@ -2,11 +2,11 @@ vLLM computes the answer-code logits; decisionformer renders the prompt (byte-identical to the transformers backend) and turns logits into answers with the checkpoint's per-type temperatures, so both backends answer the -same way. vLLM's decision-model support covers code-readout checkpoints whose backbone is a bare Qwen3.5/3.8 model +same way. vLLM's decision-model support covers code-readout checkpoints whose backbone is a bare Qwen3.5 model (`Qwen3_5Model` plus `decision_config.json` and `readout.safetensors`): decisionformer exports on those bases, Decision 3.0 (`vllm-sr/d3*`) and AutoJev-format exports such as FrontiersMind/Lumma-fev-27B. Anything else raises -`UnsupportedBackend`; serve it with the default transformers backend. Needs a vLLM build with decision-model -support (`pip install decisionformer[vllm]`, vLLM nightly from 2026-10-15 on) and a CUDA or ROCm GPU. +`UnsupportedBackend`; serve it with the default transformers backend. Needs a vLLM nightly from 2026-10-15 +on (vLLM 0.32.0 on PyPI predates its decision-model support) and a CUDA or ROCm GPU. """ from __future__ import annotations @@ -34,7 +34,7 @@ def check_vllm_support(root: Path, config: dict[str, Any]) -> None: """Raise UnsupportedBackend unless vLLM's decision-model path can load this checkpoint as it is.""" def refuse(reason: str) -> None: raise UnsupportedBackend(f"vLLM cannot serve {root}: {reason}. vLLM serves code-readout decision checkpoints " - "with a bare Qwen3.5/3.8 backbone (Qwen3_5Model). Use the transformers backend " + "with a bare Qwen3.5 backbone (Qwen3_5Model). Use the transformers backend " "(the default) for this model.") if not (root / READOUT).is_file() or (root / HEADS).is_file():