From 4a8b4acfcfcff7fd657f673e44846bd7ceda2a5a Mon Sep 17 00:00:00 2001 From: aas008 Date: Thu, 1 Oct 2026 11:12:04 -0400 Subject: [PATCH 1/7] feat(rhaiis): onboard Athena and Janus H200 clusters Add cluster presets for psap-fire-athena (Athena) and psap-de-h200-cluster (Janus) to the RHAIIS orchestration layer. Both clusters have H200 GPUs and follow the same pattern as Hera/Zeus: - cluster_tag for dashboard labelling - npalaska-image-pull for image authentication - fs_group: 0 for guidellm benchmark compatibility Also adds both to cpt-rhaiis-3.6-h200 __clusters so the Fournos UI offers them as valid targets for H200 CPT pipeline jobs. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- projects/rhaiis/orchestration/cpt.d/cpt.yaml | 2 ++ .../rhaiis/orchestration/presets.d/clusters.yaml | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/projects/rhaiis/orchestration/cpt.d/cpt.yaml b/projects/rhaiis/orchestration/cpt.d/cpt.yaml index 7578b4429..a7a73eb75 100644 --- a/projects/rhaiis/orchestration/cpt.d/cpt.yaml +++ b/projects/rhaiis/orchestration/cpt.d/cpt.yaml @@ -133,6 +133,8 @@ cpt-rhaiis-3.6-h200: - hera - zeus - old-zeus + - athena + - janus __models: qwen3-vl-30b/tp1: rhaiis.engines.vllm.args.tensor-parallel-size: 1 diff --git a/projects/rhaiis/orchestration/presets.d/clusters.yaml b/projects/rhaiis/orchestration/presets.d/clusters.yaml index 024933d06..338cd665f 100644 --- a/projects/rhaiis/orchestration/presets.d/clusters.yaml +++ b/projects/rhaiis/orchestration/presets.d/clusters.yaml @@ -29,6 +29,16 @@ b200: rhaiis.env_vars: FLASHINFER_CUBIN_DIR: "/tmp/flashinfer" +athena: + rhaiis.cluster_tag: "athena" + rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull"] + benchmarks.guidellm.fs_group: 0 + +janus: + rhaiis.cluster_tag: "janus" + rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull"] + benchmarks.guidellm.fs_group: 0 + # ── Cluster + accelerator combos ──────────────────────────── # Usage: /test fournos rhaiis # /cluster hera @@ -41,3 +51,9 @@ cluster_zeus: cluster_old_zeus: extends: [nvidia, old-zeus] + +cluster_athena: + extends: [nvidia, athena] + +cluster_janus: + extends: [nvidia, janus] From 36c268e4342f4766f02d47d79b9bc7395617a8de Mon Sep 17 00:00:00 2001 From: aas008 Date: Fri, 2 Oct 2026 00:08:59 -0400 Subject: [PATCH 2/7] fix(rhaiis): set gpu_types.nvidia=nvidia for Janus cluster Janus's Kueue ResourceFlavor is fournos/gpu-nvidia (16 GPUs), not fournos/gpu-h200. Without this override the default gpu_types.nvidia=h200 causes FournosJob admission to fail with quota=0. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- projects/rhaiis/orchestration/presets.d/clusters.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/rhaiis/orchestration/presets.d/clusters.yaml b/projects/rhaiis/orchestration/presets.d/clusters.yaml index 338cd665f..3cb135884 100644 --- a/projects/rhaiis/orchestration/presets.d/clusters.yaml +++ b/projects/rhaiis/orchestration/presets.d/clusters.yaml @@ -36,6 +36,7 @@ athena: janus: rhaiis.cluster_tag: "janus" + rhaiis.gpu_types.nvidia: "nvidia" rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull"] benchmarks.guidellm.fs_group: 0 From b18e479fc0d4d5a74ef082dbe8a9b1cf87c75420 Mon Sep 17 00:00:00 2001 From: aas008 Date: Mon, 5 Oct 2026 14:36:41 -0400 Subject: [PATCH 3/7] fix(rhaiis): add toleration support to ISVC and configure for Athena Adds optional tolerations parameter to build_inferenceservice() and wires it from deploy_cfg.tolerations. The athena preset sets a toleration for benchmark.llm-d.ai/test-gpu-amd64:NoSchedule so ISVC pods can schedule on Athena's second GPU node (which has that taint). Without this, Athena jobs hang indefinitely waiting for GPUs on the only untainted node, which is frequently occupied by other workloads. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- projects/rhaiis/orchestration/manifests.py | 4 ++++ projects/rhaiis/orchestration/presets.d/clusters.yaml | 4 ++++ projects/rhaiis/orchestration/test_phase.py | 1 + 3 files changed, 9 insertions(+) diff --git a/projects/rhaiis/orchestration/manifests.py b/projects/rhaiis/orchestration/manifests.py index 3fbf6b67a..4f4e216bc 100644 --- a/projects/rhaiis/orchestration/manifests.py +++ b/projects/rhaiis/orchestration/manifests.py @@ -95,6 +95,7 @@ def build_inferenceservice( model_id: str, service_account_name: str = "", supplemental_groups: list[int] | None = None, + tolerations: list[dict] | None = None, labels: dict | None = None, profiler_ranges: str | None = None, ) -> dict[str, Any]: @@ -158,6 +159,9 @@ def build_inferenceservice( if supplemental_groups: predictor["securityContext"] = {"supplementalGroups": supplemental_groups} + if tolerations: + predictor["tolerations"] = tolerations + return { "apiVersion": "serving.kserve.io/v1beta1", "kind": "InferenceService", diff --git a/projects/rhaiis/orchestration/presets.d/clusters.yaml b/projects/rhaiis/orchestration/presets.d/clusters.yaml index 3cb135884..9b498293e 100644 --- a/projects/rhaiis/orchestration/presets.d/clusters.yaml +++ b/projects/rhaiis/orchestration/presets.d/clusters.yaml @@ -32,6 +32,10 @@ b200: athena: rhaiis.cluster_tag: "athena" rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull"] + rhaiis.deploy.tolerations: + - key: "benchmark.llm-d.ai/test-gpu-amd64" + value: "true" + effect: "NoSchedule" benchmarks.guidellm.fs_group: 0 janus: diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 6ce33244f..69c3683e7 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -269,6 +269,7 @@ def _run_test( model_id=model_cfg["hf_model_id"], service_account_name=deploy_cfg.get("service_account_name", ""), supplemental_groups=deploy_cfg.get("supplemental_groups"), + tolerations=deploy_cfg.get("tolerations"), labels=isvc_labels, profiler_ranges=profiler_ranges, ) From 38614533061ef9ee47cb5d789600412e7013a1f6 Mon Sep 17 00:00:00 2001 From: aas008 Date: Mon, 5 Oct 2026 14:37:41 -0400 Subject: [PATCH 4/7] Revert "fix(rhaiis): add toleration support to ISVC and configure for Athena" This reverts commit b18e479fc0d4d5a74ef082dbe8a9b1cf87c75420. --- projects/rhaiis/orchestration/manifests.py | 4 ---- projects/rhaiis/orchestration/presets.d/clusters.yaml | 4 ---- projects/rhaiis/orchestration/test_phase.py | 1 - 3 files changed, 9 deletions(-) diff --git a/projects/rhaiis/orchestration/manifests.py b/projects/rhaiis/orchestration/manifests.py index 4f4e216bc..3fbf6b67a 100644 --- a/projects/rhaiis/orchestration/manifests.py +++ b/projects/rhaiis/orchestration/manifests.py @@ -95,7 +95,6 @@ def build_inferenceservice( model_id: str, service_account_name: str = "", supplemental_groups: list[int] | None = None, - tolerations: list[dict] | None = None, labels: dict | None = None, profiler_ranges: str | None = None, ) -> dict[str, Any]: @@ -159,9 +158,6 @@ def build_inferenceservice( if supplemental_groups: predictor["securityContext"] = {"supplementalGroups": supplemental_groups} - if tolerations: - predictor["tolerations"] = tolerations - return { "apiVersion": "serving.kserve.io/v1beta1", "kind": "InferenceService", diff --git a/projects/rhaiis/orchestration/presets.d/clusters.yaml b/projects/rhaiis/orchestration/presets.d/clusters.yaml index 9b498293e..3cb135884 100644 --- a/projects/rhaiis/orchestration/presets.d/clusters.yaml +++ b/projects/rhaiis/orchestration/presets.d/clusters.yaml @@ -32,10 +32,6 @@ b200: athena: rhaiis.cluster_tag: "athena" rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull"] - rhaiis.deploy.tolerations: - - key: "benchmark.llm-d.ai/test-gpu-amd64" - value: "true" - effect: "NoSchedule" benchmarks.guidellm.fs_group: 0 janus: diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 69c3683e7..6ce33244f 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -269,7 +269,6 @@ def _run_test( model_id=model_cfg["hf_model_id"], service_account_name=deploy_cfg.get("service_account_name", ""), supplemental_groups=deploy_cfg.get("supplemental_groups"), - tolerations=deploy_cfg.get("tolerations"), labels=isvc_labels, profiler_ranges=profiler_ranges, ) From ae79212a738440300ba0d2c2d0e1d5b77aa117ba Mon Sep 17 00:00:00 2001 From: aas008 Date: Mon, 5 Oct 2026 14:41:54 -0400 Subject: [PATCH 5/7] chore(rhaiis): remove athena/janus from cpt-rhaiis-3.6-h200 __clusters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review feedback — these clusters should not be in the CPT pipeline yet. The cluster presets (cluster_athena, cluster_janus) remain for one-off job submissions via Fournos UI. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- projects/rhaiis/orchestration/cpt.d/cpt.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/projects/rhaiis/orchestration/cpt.d/cpt.yaml b/projects/rhaiis/orchestration/cpt.d/cpt.yaml index a7a73eb75..7578b4429 100644 --- a/projects/rhaiis/orchestration/cpt.d/cpt.yaml +++ b/projects/rhaiis/orchestration/cpt.d/cpt.yaml @@ -133,8 +133,6 @@ cpt-rhaiis-3.6-h200: - hera - zeus - old-zeus - - athena - - janus __models: qwen3-vl-30b/tp1: rhaiis.engines.vllm.args.tensor-parallel-size: 1 From 096556b43f4b9cc7fe7d7c297f3c14716d7f0173 Mon Sep 17 00:00:00 2001 From: aas008 Date: Mon, 5 Oct 2026 14:42:29 -0400 Subject: [PATCH 6/7] chore(rhaiis): restore athena/janus in cpt-rhaiis-3.6-h200 __clusters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Approved to include — reverts the previous removal. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- projects/rhaiis/orchestration/cpt.d/cpt.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/rhaiis/orchestration/cpt.d/cpt.yaml b/projects/rhaiis/orchestration/cpt.d/cpt.yaml index 7578b4429..a7a73eb75 100644 --- a/projects/rhaiis/orchestration/cpt.d/cpt.yaml +++ b/projects/rhaiis/orchestration/cpt.d/cpt.yaml @@ -133,6 +133,8 @@ cpt-rhaiis-3.6-h200: - hera - zeus - old-zeus + - athena + - janus __models: qwen3-vl-30b/tp1: rhaiis.engines.vllm.args.tensor-parallel-size: 1 From 4888236fea03cd2ff71c8ffaa6115376bf747dad Mon Sep 17 00:00:00 2001 From: aas008 Date: Mon, 5 Oct 2026 15:01:14 -0400 Subject: [PATCH 7/7] fix(rhaiis): rename athena preset to athena-fire to match Fournos cluster name The Fournos cluster is registered as 'athena-fire', so the combo preset must be cluster_athena-fire for it to be auto-applied when using /cluster athena-fire in CI triggers. Also updates the CPT __clusters entry. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- projects/rhaiis/orchestration/cpt.d/cpt.yaml | 2 +- projects/rhaiis/orchestration/presets.d/clusters.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/rhaiis/orchestration/cpt.d/cpt.yaml b/projects/rhaiis/orchestration/cpt.d/cpt.yaml index a7a73eb75..a49337731 100644 --- a/projects/rhaiis/orchestration/cpt.d/cpt.yaml +++ b/projects/rhaiis/orchestration/cpt.d/cpt.yaml @@ -133,7 +133,7 @@ cpt-rhaiis-3.6-h200: - hera - zeus - old-zeus - - athena + - athena-fire - janus __models: qwen3-vl-30b/tp1: diff --git a/projects/rhaiis/orchestration/presets.d/clusters.yaml b/projects/rhaiis/orchestration/presets.d/clusters.yaml index 3cb135884..261d93214 100644 --- a/projects/rhaiis/orchestration/presets.d/clusters.yaml +++ b/projects/rhaiis/orchestration/presets.d/clusters.yaml @@ -29,7 +29,7 @@ b200: rhaiis.env_vars: FLASHINFER_CUBIN_DIR: "/tmp/flashinfer" -athena: +athena-fire: rhaiis.cluster_tag: "athena" rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull"] benchmarks.guidellm.fs_group: 0 @@ -53,8 +53,8 @@ cluster_zeus: cluster_old_zeus: extends: [nvidia, old-zeus] -cluster_athena: - extends: [nvidia, athena] +cluster_athena-fire: + extends: [nvidia, athena-fire] cluster_janus: extends: [nvidia, janus]