diff --git a/console/fit.py b/console/fit.py index 41211e77..beb14151 100644 --- a/console/fit.py +++ b/console/fit.py @@ -1094,7 +1094,13 @@ def fit(host, cfg, weights, engine=None, ctx=None, seqs=None, gpus=None, kv_dtyp doc = {"schema": FIT_SCHEMA, "model": name, "format": m["format"], "arch": m["arch"], "fits": bool(picked), "degraded": degraded, "host_reserve": hres, "explain": ex} if not picked: - why = tried[-1]["why"] if tried else (cands[0][1] if cands else "no engine serves %s weights" % m["format"]) + if tried: + why = tried[-1]["why"] + elif any(e for e, _ in cands): + # No placement was tried: no card left after the busy ones, and the CPU not allowed. + why = "no Intel GPU with device-local memory is free" + else: + why = cands[0][1] if cands else "no engine serves %s weights" % m["format"] doc.update(reason=why, summary="fit: %s does not fit this host: %s" % (name, why)) return doc rungs = guard_rungs(picked["ctx"], picked["seqs"]) @@ -1172,7 +1178,11 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, compile_host = m["serve_bytes"] if eng == "ovms" else 0 gpu_w_all = m["serve_bytes"] - on_host hostrt = HOST_RUNTIME[eng] + (VLLM_SHM if eng == "vllm" else 0) # times the rank count for vLLM, below - if cards: + if cards and eng == "ovms" and want_gpus and want_gpus > 1: + # The agent's OVMS pod holds one gpu.intel.com/xe; multi-GPU tensor parallel on Arc + # is not documented (model_server docs/parameters.md). + yield _res(eng, "gpu", False, "OVMS serves one GPU: %d asked" % want_gpus, gpus=want_gpus) + elif cards: if eng == "vllm": counts = _tp_sizes(m, len(cards)) elif eng == "llama.cpp": diff --git a/docs/fit.md b/docs/fit.md index ab06e5f1..a304003f 100644 --- a/docs/fit.md +++ b/docs/fit.md @@ -57,7 +57,7 @@ in this order, and the first that fits wins: 1. one GPU; 2. several GPUs: vLLM tensor parallel (heads divisible by the size), or llama.cpp's - layer split; + layer split (OVMS serves one GPU); 3. MoE experts in DRAM (llama.cpp `--n-cpu-moe N`): only when the GPU placements do not fit, the CPU has AMX or AVX-512, and the DRAM side's bandwidth bound per token stays above a floor. The fit takes the fewest layers whose experts leave the GPU; @@ -81,6 +81,9 @@ Context first: the largest context, halving from the model's cap, at which 4 seq fit at once and that is at least 32768 tokens (or the cap). Only when no placement reaches that does the fit take less, down to one sequence of 4096 tokens, and the summary says it is below the target. Then as many sequences as fit at that context. +When no placement can be tried at all (every card busy or none with device-local memory, +and the CPU not allowed), the refusal says that no Intel GPU with device-local memory is +free. llama.cpp keeps the token embeddings and the n-gram table on the CPU by design (`src/llama-model.cpp`, input layer "always keep it on the CPU"), so those count as host @@ -100,7 +103,9 @@ The overhead is an estimate from the formula, not a measurement; `agent up` applies them, sets the pod's memory request and limit to the fit, and annotates the pod `k3d-gpu.io/agent-fit`. `K3D_GPU_AGENT_MAX_LEN`, `K3D_GPU_AGENT_MAX_NUM_SEQS` -and `K3D_GPU_AGENT_GPUS` are inputs the fit checks, not values that replace it. A fit +and `K3D_GPU_AGENT_GPUS` (vLLM only: an OVMS pod holds one GPU, and `fit --gpus 2` with +OVMS is refused as "OVMS serves one GPU") are inputs the fit checks, not values that +replace it. A fit that picks llama.cpp is refused by the OVMS and vLLM backends with the llama-server flags to run instead. `K3D_GPU_FIT=off` skips the fit; `on` also sizes the default models, which `auto` (the default) leaves as they are, as it leaves a quickstart RAM-tier diff --git a/scripts/k3d-gpu b/scripts/k3d-gpu index b4f8b143..3b34fc12 100755 --- a/scripts/k3d-gpu +++ b/scripts/k3d-gpu @@ -10383,7 +10383,8 @@ agent_fit() { || { fit_soft "${mode}" "fit: no metadata for ${AGENT_MODEL} (offline, private or gated: set HF_TOKEN)"; return 0; } [ -z "${K3D_GPU_AGENT_MAX_LEN:-}" ] || args+=(--ctx "${K3D_GPU_AGENT_MAX_LEN}") [ -z "${K3D_GPU_AGENT_MAX_NUM_SEQS:-}" ] || args+=(--seqs "${K3D_GPU_AGENT_MAX_NUM_SEQS}") - [ -z "${K3D_GPU_AGENT_GPUS:-}" ] || args+=(--gpus "${K3D_GPU_AGENT_GPUS}") + # K3D_GPU_AGENT_GPUS shards a vLLM model; an OVMS pod holds one GPU whatever it says. + [ -z "${K3D_GPU_AGENT_GPUS:-}" ] || [ "${AGENT_APP}" != vllm ] || args+=(--gpus "${K3D_GPU_AGENT_GPUS}") # agent up replaces a running server of this backend (the Deployment is Recreate): it # is stopped before the probe, so the memory it holds is measured as free, not as # someone else's. A model that then does not fit gets the old server back. diff --git a/tests/fit_test.py b/tests/fit_test.py index 56bf7440..3c40ed93 100755 --- a/tests/fit_test.py +++ b/tests/fit_test.py @@ -439,6 +439,33 @@ def test_kv_scheme_follows_checkpoint(self): self.assertTrue(any("kv_cache_scheme" in s for s in d["explain"])) +class OvmsOneGpu(unittest.TestCase): + """OVMS serves one GPU: --gpus 2 is refused, never sized as a layer split of two cards.""" + + def test_two_gpus_asked_is_refused(self): + for mname in ("OpenVINO__gemma-4-26b-a4b-it-int4-ov", "OpenVINO__Qwen3.8-27B-int4-ov"): + d = plan("xeon6741p-4xb70", mname, gpus=2, device="gpu") + self.assertFalse(d["fits"], (mname, d["summary"])) + self.assertIn("OVMS serves one GPU", d["reason"]) + + def test_cache_size_within_one_card(self): + one = max(c["budget"] for c in fit.gpu_budget(host("xeon6741p-4xb70"))[0]) + for gpus in (None, 1, 2): + d = plan("xeon6741p-4xb70", "OpenVINO__gemma-4-26b-a4b-it-int4-ov", gpus=gpus, device="gpu") + if not d["fits"]: + continue + self.assertEqual((d["placement"], d["gpus"]), ("gpu", 1), gpus) + cache = [int(f.split("=")[1]) for f in d["flags"] if f.startswith("--cache_size=")] + self.assertLessEqual(cache[0] * GIB, one, (gpus, d["flags"])) + + def test_no_free_gpu_says_so(self): + # device gpu on a host with no Intel GPU of its own memory: no placement is tried at all + d = plan("desktop-i9-nogpu", "OpenVINO__Qwen3.8-27B-int4-ov", device="gpu") + self.assertFalse(d["fits"]) + self.assertIn("no Intel GPU with device-local memory is free", d["reason"]) + self.assertNotIn("release notes", d["reason"]) + + class Guard(unittest.TestCase): """guard_step under simulated PSI: a CPU-placed server sheds rungs; a VRAM one holds or stops.""" diff --git a/tests/fit_test.sh b/tests/fit_test.sh index b655baf5..5b7230b2 100755 --- a/tests/fit_test.sh +++ b/tests/fit_test.sh @@ -156,6 +156,18 @@ if render ovms STUB_ALLOC='cpu 32' K3D_GPU_FIT_HOST="${DESK}" K3D_GPU_AGENT_MODE ok "agent manifest (ovms, CPU): static KV pool, seqs and pod memory limit from the fit" else fail "agent manifest (ovms, CPU) from the fit" "$(cat "${WORK}/ovms.err" "${WORK}/ovms.yaml" 2>/dev/null | head -60)"; fi +# OVMS serves one GPU: K3D_GPU_AGENT_GPUS=2 is a vLLM knob, so the OVMS pod (one +# gpu.intel.com/xe) gets a KV pool that fits one card, not two cards' worth. +one_card=$(python3 -c 'import json, sys; sys.path.insert(0, sys.argv[1]); import fit +print(max(c["budget"] for c in fit.gpu_budget(json.load(open(sys.argv[2])))[0]) // fit.GIB)' "${ROOT}/console" "${XEON}") +if render ovms2 STUB_ALLOC=$'cpu 64\ngpu.intel.com/xe 4' STUB_NODE_LABELS=gpu.intel.com/xmx=true K3D_GPU_FIT_HOST="${XEON}" \ + K3D_GPU_AGENT_MODEL=OpenVINO/gemma-4-26b-a4b-it-int4-ov K3D_GPU_AGENT_GPUS=2 -- \ + && cache=$(sed -n 's/^ *- --cache_size=\([0-9]*\)$/\1/p' "${WORK}/ovms2.yaml") && [ -n "${cache}" ] \ + && [ "${cache}" -le "${one_card}" ] && has ovms2 '^ gpu.intel.com/xe: 1$' && ! has ovms2 'gpu-split'; then + ok "agent manifest (ovms, K3D_GPU_AGENT_GPUS=2): --cache_size=${cache} fits one card's ${one_card} GiB budget" +else fail "agent manifest (ovms, K3D_GPU_AGENT_GPUS=2) sized for one card (cache ${cache:-?}, one card ${one_card})" \ + "$(cat "${WORK}/ovms2.err"; grep -E 'cache_size|agent-fit|gpu.intel.com/xe' "${WORK}/ovms2.yaml")"; fi + if render tier STUB_ALLOC='cpu 32' K3D_GPU_FIT_HOST="${LAPTOP}" K3D_GPU_AGENT_MODEL=OpenVINO/Qwen3.8-27B-int8-ov \ K3D_GPU_AGENT_MODEL_TIER=1 -- && ! has tier 'agent-fit'; then ok "a quickstart tier pick (K3D_GPU_AGENT_MODEL_TIER=1) is not sized: it is no user choice"