Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions docs/source/_static/config_db.json
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,18 @@
"model_url": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4",
"scenario": "Max Throughput"
},
{
"command": "trtllm-serve nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 --config ${TRTLLM_DIR}/examples/configs/curated/nemotron-super-marlin.yaml",
"config_filename": "nemotron-super-marlin.yaml",
"config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/curated/nemotron-super-marlin.yaml",
"config_path": "examples/configs/curated/nemotron-super-marlin.yaml",
"config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/curated/nemotron-super-marlin.yaml",
"gpu_compatibility": "H100, H200",
"model": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4",
"model_display_name": "Nemotron v3 Super (NVFP4)",
"model_url": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4",
"scenario": "NVFP4 on Hopper (Marlin)"
},
{
"command": "trtllm-serve Qwen/Qwen3-Next-80B-A3B-Thinking --config ${TRTLLM_DIR}/examples/configs/curated/qwen3-next.yaml",
"config_filename": "qwen3-next.yaml",
Expand Down
17 changes: 17 additions & 0 deletions examples/configs/curated/lookup.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,11 @@
config_path: examples/configs/curated/nemotron-3-ultra-throughput.yaml
scenario: Max Throughput
gpu_compatibility: "B200, B300, GB200, GB300, H100, H200"
- model: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4
arch: NemotronHForCausalLM
config_path: examples/configs/curated/nemotron-super-marlin.yaml
scenario: NVFP4 on Hopper (Marlin)
gpu_compatibility: "H100, H200"
- model: Qwen/Qwen3-Next-80B-A3B-Thinking
arch: Qwen3NextForCausalLM
config_path: examples/configs/curated/qwen3-next.yaml
Expand Down Expand Up @@ -56,6 +61,18 @@
config_path: examples/configs/curated/qwen3.5.yaml
scenario: Max Throughput
gpu_compatibility: "B200, B300, GB200, GB300"
- model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2
arch: Qwen3_5MoeForCausalLM
config_path: examples/configs/curated/qwen3.5-gb300-agentic-prefill.yaml
scenario: Agentic Disaggregated Prefill
gpu_compatibility: "GB300"
disagg: true
- model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2
arch: Qwen3_5MoeForCausalLM
config_path: examples/configs/curated/qwen3.5-gb300-agentic-decode.yaml
scenario: Agentic Disaggregated Decode
gpu_compatibility: "GB300"
disagg: true
- model: Qwen/Qwen3-30B-A3B
arch: Qwen3MoeForCausalLM
config_path: examples/configs/curated/qwen3.yaml
Expand Down
18 changes: 17 additions & 1 deletion tests/unittest/llmapi/test_config_database.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,14 @@
CURATED_DIR = CONFIG_ROOT / "curated"
DATABASE_DIR = CONFIG_ROOT / "database"

CURATED_CONFIGS = collect_yaml_files(CURATED_DIR, "**/*.yaml", exclude_names={"lookup.yaml"})
# curated/eplb/ holds MoE expert-placement tables that curated configs reference through
# moe_config.load_balancer; like lookup.yaml they are not standalone serving configs.
CURATED_EPLB_DIR = CURATED_DIR / "eplb"
CURATED_CONFIGS = [
path
for path in collect_yaml_files(CURATED_DIR, "**/*.yaml", exclude_names={"lookup.yaml"})
if CURATED_EPLB_DIR not in path.parents
]
DATABASE_CONFIGS = collect_yaml_files(DATABASE_DIR, "**/*.yaml", exclude_names={"lookup.yaml"})
ALL_CONFIGS = sorted(CURATED_CONFIGS + DATABASE_CONFIGS)
ALL_LOOKUP_PATHS = sorted((DATABASE_DIR / "lookup.yaml", CURATED_DIR / "lookup.yaml"))
Expand Down Expand Up @@ -207,6 +214,13 @@ def _serve_cli_args(config_path: Path, port: int = 17999):
str(config_path),
"--port",
str(port),
# With more than one frontend, launch_server spawns attached frontend
# processes by re-executing sys.argv (here: the pytest command line);
# a single frontend keeps the real launch_server in-process. Passing
# the flag skips the default frontend-count resolution, so this test
# covers the single-frontend path only.
"--num_serve_frontends",
Comment thread
JunyiXu-nv marked this conversation as resolved.
Resolved
"1",
]


Expand All @@ -218,6 +232,8 @@ class _MockOpenAIServer:
def __init__(self, generator, model, **kwargs):
self.generator = generator
self.model = model
# launch_server registers --middleware objects on the server's app.
self.app = FastAPI()

def __call__(self, host, port, sockets=None):
return _noop_serve(host, port, sockets)
Expand Down
Loading