From 9482b3281ddabe2049efb8bb9b56fdd0dac8a210 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Wed, 28 Oct 2026 22:38:13 -0700 Subject: [PATCH 01/13] fit: OVMS serves one GPU With K3D_GPU_AGENT_GPUS=2 the launcher passed --gpus 2 to the fit for an OVMS agent too. The fit then sized a layer split over two cards and rendered a KV pool of two cards' worth (--cache_size=41 for the Gemma 4 int4 IR on the Xeon fixture) into a pod that holds one gpu.intel.com/xe. The fit now refuses a multi-GPU OVMS placement with "OVMS serves one GPU", and agent_fit passes --gpus only when the agent app is vLLM. When no placement can be tried at all, the refusal says no Intel GPU with device-local memory is free instead of echoing the engine note. Tests: OVMS with --gpus 2 is refused, the GPU cache_size stays within one card's budget, the no-free-GPU reason, and an agent manifest with K3D_GPU_AGENT_GPUS=2 under OVMS whose --cache_size fits one card. --- console/fit.py | 14 ++++++++++++-- docs/fit.md | 9 +++++++-- scripts/k3d-gpu | 3 ++- tests/fit_test.py | 27 +++++++++++++++++++++++++++ tests/fit_test.sh | 12 ++++++++++++ 5 files changed, 60 insertions(+), 5 deletions(-) diff --git a/console/fit.py b/console/fit.py index dd9448f9..f7a2f02c 100644 --- a/console/fit.py +++ b/console/fit.py @@ -1141,7 +1141,13 @@ def fit(host, cfg, weights, engine=None, ctx=None, seqs=None, gpus=None, kv_dtyp doc = {"schema": FIT_SCHEMA, "model": name, "format": m["format"], "arch": m["arch"], "fits": bool(picked), "degraded": degraded, "host_reserve": hres, "explain": ex} if not picked: - why = tried[-1]["why"] if tried else (cands[0][1] if cands else "no engine serves %s weights" % m["format"]) + if tried: + why = tried[-1]["why"] + elif any(e for e, _ in cands): + # No placement was tried: no card left after the busy ones, and the CPU not allowed. + why = "no Intel GPU with device-local memory is free" + else: + why = cands[0][1] if cands else "no engine serves %s weights" % m["format"] doc.update(reason=why, summary="fit: %s does not fit this host: %s" % (name, why)) return doc rungs = guard_rungs(picked["ctx"], picked["seqs"]) @@ -1227,7 +1233,11 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, compile_host = m["serve_bytes"] if eng == "ovms" else 0 gpu_w_all = m["serve_bytes"] - on_host hostrt = HOST_RUNTIME[eng] + (VLLM_SHM if eng == "vllm" else 0) # times the rank count for vLLM, below - if cards: + if cards and eng == "ovms" and want_gpus and want_gpus > 1: + # The agent's OVMS pod holds one gpu.intel.com/xe; multi-GPU tensor parallel on Arc + # is not documented (model_server docs/parameters.md). + yield _res(eng, "gpu", False, "OVMS serves one GPU: %d asked" % want_gpus, gpus=want_gpus) + elif cards: if eng == "vllm": counts = _tp_sizes(m, len(cards)) elif eng == "llama.cpp": diff --git a/docs/fit.md b/docs/fit.md index 36358501..3dfc62b5 100644 --- a/docs/fit.md +++ b/docs/fit.md @@ -68,7 +68,7 @@ in this order, and the first that fits wins: 1. one GPU; 2. several GPUs: vLLM tensor parallel (heads divisible by the size), or llama.cpp's - layer split; + layer split (OVMS serves one GPU); 3. MoE experts in DRAM (llama.cpp `--n-cpu-moe N`): only when the GPU placements do not fit, the CPU has AMX or AVX-512, and the DRAM side's bandwidth bound per token stays above a floor. The fit takes the fewest layers whose experts leave the GPU; @@ -92,6 +92,9 @@ Context first: the largest context, halving from the model's cap, at which 4 seq fit at once and that is at least 32768 tokens (or the cap). Only when no placement reaches that does the fit take less, down to one sequence of 4096 tokens, and the summary says it is below the target. Then as many sequences as fit at that context. +When no placement can be tried at all (every card busy or none with device-local memory, +and the CPU not allowed), the refusal says that no Intel GPU with device-local memory is +free. llama.cpp keeps the token embeddings and the n-gram table on the CPU by design (`src/llama-model.cpp`, input layer "always keep it on the CPU"), so those count as host @@ -111,7 +114,9 @@ The overhead is an estimate from the formula, not a measurement; `agent up` applies them, sets the pod's memory request and limit to the fit, and annotates the pod `k3d-gpu.io/agent-fit`. `K3D_GPU_AGENT_MAX_LEN`, `K3D_GPU_AGENT_MAX_NUM_SEQS` -and `K3D_GPU_AGENT_GPUS` are inputs the fit checks, not values that replace it. With +and `K3D_GPU_AGENT_GPUS` (vLLM only: an OVMS pod holds one GPU, and `fit --gpus 2` with +OVMS is refused as "OVMS serves one GPU") are inputs the fit checks, not values that +replace it. With `K3D_GPU_AGENT_BACKEND` unset, the engine the fit picks serves the model: OVMS, vLLM or llama-server. With `--model`, the model optimizer first picks the variant for the default engine (OVMS: Gemma 4's OpenVINO IR, for example). When that engine has nothing to serve, diff --git a/scripts/k3d-gpu b/scripts/k3d-gpu index 847b5c14..5ef513bc 100755 --- a/scripts/k3d-gpu +++ b/scripts/k3d-gpu @@ -11200,7 +11200,8 @@ agent_fit() { || { fit_soft "${mode}" "fit: no metadata for ${AGENT_MODEL} (offline, private or gated: set HF_TOKEN)"; return 0; } [ -z "${K3D_GPU_AGENT_MAX_LEN:-}" ] || args+=(--ctx "${K3D_GPU_AGENT_MAX_LEN}") [ -z "${K3D_GPU_AGENT_MAX_NUM_SEQS:-}" ] || args+=(--seqs "${K3D_GPU_AGENT_MAX_NUM_SEQS}") - [ -z "${K3D_GPU_AGENT_GPUS:-}" ] || args+=(--gpus "${K3D_GPU_AGENT_GPUS}") + # K3D_GPU_AGENT_GPUS shards a vLLM model; an OVMS pod holds one GPU whatever it says. + [ -z "${K3D_GPU_AGENT_GPUS:-}" ] || [ "${AGENT_APP}" != vllm ] || args+=(--gpus "${K3D_GPU_AGENT_GPUS}") # agent up replaces the running server (the Deployment is Recreate, and a backend # switch deletes the other one): it is stopped before the probe, so the memory it holds # is measured as free, not as someone else's. A model that then does not fit gets the diff --git a/tests/fit_test.py b/tests/fit_test.py index 341c6e4a..df5d56f9 100755 --- a/tests/fit_test.py +++ b/tests/fit_test.py @@ -501,6 +501,33 @@ def test_kv_scheme_follows_checkpoint(self): self.assertTrue(any("kv_cache_scheme" in s for s in d["explain"])) +class OvmsOneGpu(unittest.TestCase): + """OVMS serves one GPU: --gpus 2 is refused, never sized as a layer split of two cards.""" + + def test_two_gpus_asked_is_refused(self): + for mname in ("OpenVINO__gemma-4-26b-a4b-it-int4-ov", "OpenVINO__Qwen3.8-27B-int4-ov"): + d = plan("xeon6741p-4xb70", mname, gpus=2, device="gpu") + self.assertFalse(d["fits"], (mname, d["summary"])) + self.assertIn("OVMS serves one GPU", d["reason"]) + + def test_cache_size_within_one_card(self): + one = max(c["budget"] for c in fit.gpu_budget(host("xeon6741p-4xb70"))[0]) + for gpus in (None, 1, 2): + d = plan("xeon6741p-4xb70", "OpenVINO__gemma-4-26b-a4b-it-int4-ov", gpus=gpus, device="gpu") + if not d["fits"]: + continue + self.assertEqual((d["placement"], d["gpus"]), ("gpu", 1), gpus) + cache = [int(f.split("=")[1]) for f in d["flags"] if f.startswith("--cache_size=")] + self.assertLessEqual(cache[0] * GIB, one, (gpus, d["flags"])) + + def test_no_free_gpu_says_so(self): + # device gpu on a host with no Intel GPU of its own memory: no placement is tried at all + d = plan("desktop-i9-nogpu", "OpenVINO__Qwen3.8-27B-int4-ov", device="gpu") + self.assertFalse(d["fits"]) + self.assertIn("no Intel GPU with device-local memory is free", d["reason"]) + self.assertNotIn("release notes", d["reason"]) + + class Guard(unittest.TestCase): """guard_step under simulated PSI: a CPU-placed server sheds rungs; a VRAM one holds or stops.""" diff --git a/tests/fit_test.sh b/tests/fit_test.sh index bdb42439..f7398d47 100755 --- a/tests/fit_test.sh +++ b/tests/fit_test.sh @@ -156,6 +156,18 @@ if render ovms STUB_ALLOC='cpu 32' K3D_GPU_FIT_HOST="${DESK}" K3D_GPU_AGENT_MODE ok "agent manifest (ovms, CPU): static KV pool, seqs and pod memory limit from the fit" else fail "agent manifest (ovms, CPU) from the fit" "$(cat "${WORK}/ovms.err" "${WORK}/ovms.yaml" 2>/dev/null | head -60)"; fi +# OVMS serves one GPU: K3D_GPU_AGENT_GPUS=2 is a vLLM knob, so the OVMS pod (one +# gpu.intel.com/xe) gets a KV pool that fits one card, not two cards' worth. +one_card=$(python3 -c 'import json, sys; sys.path.insert(0, sys.argv[1]); import fit +print(max(c["budget"] for c in fit.gpu_budget(json.load(open(sys.argv[2])))[0]) // fit.GIB)' "${ROOT}/console" "${XEON}") +if render ovms2 STUB_ALLOC=$'cpu 64\ngpu.intel.com/xe 4' STUB_NODE_LABELS=gpu.intel.com/xmx=true K3D_GPU_FIT_HOST="${XEON}" \ + K3D_GPU_AGENT_MODEL=OpenVINO/gemma-4-26b-a4b-it-int4-ov K3D_GPU_AGENT_GPUS=2 -- \ + && cache=$(sed -n 's/^ *- --cache_size=\([0-9]*\)$/\1/p' "${WORK}/ovms2.yaml") && [ -n "${cache}" ] \ + && [ "${cache}" -le "${one_card}" ] && has ovms2 '^ gpu.intel.com/xe: 1$' && ! has ovms2 'gpu-split'; then + ok "agent manifest (ovms, K3D_GPU_AGENT_GPUS=2): --cache_size=${cache} fits one card's ${one_card} GiB budget" +else fail "agent manifest (ovms, K3D_GPU_AGENT_GPUS=2) sized for one card (cache ${cache:-?}, one card ${one_card})" \ + "$(cat "${WORK}/ovms2.err"; grep -E 'cache_size|agent-fit|gpu.intel.com/xe' "${WORK}/ovms2.yaml")"; fi + if render tier STUB_ALLOC='cpu 32' K3D_GPU_FIT_HOST="${LAPTOP}" K3D_GPU_AGENT_MODEL=OpenVINO/Qwen3.8-27B-int8-ov \ K3D_GPU_AGENT_MODEL_TIER=1 -- && ! has tier 'agent-fit'; then ok "a quickstart tier pick (K3D_GPU_AGENT_MODEL_TIER=1) is not sized: it is no user choice" From 6f3ffa8d4f21867f041967a191892dcafed1d035 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Wed, 28 Oct 2026 22:44:06 -0700 Subject: [PATCH 02/13] fit: size sliding-window KV per engine The fit capped every engine's sliding-window layers at the window. OpenVINO GenAI 2026.4 has no sliding-window cache type (only KV_CACHE and LINEAR_ATTENTION_CACHE in continuous_batching/cache/cache_type.hpp), and one KV block spans every attention layer, so OVMS holds every token in those layers too. Gemma 4 costs 225280 B per token under OVMS, not 20480 B plus a 1024-token window; on the Xeon fixture the old fit set --cache_size=13 for 131072 x 5 sequences, which needs about 138 GiB. kv_profile and per_seq are now engine-aware through swa_tokens: OVMS counts sliding-window layers at the whole context, vLLM keeps the window cap, and llama.cpp counts its SWA cache of GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch 512), 256) cells per slot (src/llama-kv-cache-iswa.cpp at b11749), 1536 cells for Gemma 4's 1024 window. OVMS --cache_size is whole GiB and was floored, so the pool could hold fewer tokens than the fit counted. Whole-GiB pools (OVMS, and vLLM's CPU KV space) are now sized inside whole GiB of room and rounded up, on every guard rung too. Tests: per-engine SWA bytes for Gemma 4; the Gemma 4 OVMS cache_size holds context x sequences at 225280 B per token on a GPU, on the Xeon CPU and on the desktop, at every guard rung; the agent manifest renders such a pool; the Gemma QAT GGUF plan drops from 6 to 5 slots for the 1536-cell SWA cache. --- console/fit.py | 74 ++++++++++++++++++++++++++++++++++++++--------- docs/fit.md | 15 ++++++++-- tests/fit_test.py | 33 ++++++++++++++++++++- tests/fit_test.sh | 11 +++++++ 4 files changed, 116 insertions(+), 17 deletions(-) diff --git a/console/fit.py b/console/fit.py index f7a2f02c..6c9bb2ca 100644 --- a/console/fit.py +++ b/console/fit.py @@ -118,6 +118,12 @@ # KV cache types each engine takes and the fit sets a flag for (vLLM --kv-cache-dtype, # llama.cpp -ctk/-ctv; OVMS's KV_CACHE_PRECISION is not set by the fit). KV_DTYPES_OF = {"vllm": ("fp8", "fp16", "bf16"), "llama.cpp": ("f16", "fp16", "bf16", "q8_0", "q4_0")} +# Tokens a sliding-window layer's cache holds per sequence (swa_tokens). vLLM: the window. +# llama.cpp b11749: GGML_PAD(n_swa + n_ubatch, 256) cells per stream (src/llama-kv-cache-iswa.cpp). +# OVMS: OpenVINO GenAI 2026.4 has only KV_CACHE and LINEAR_ATTENTION_CACHE types +# (src/cpp/src/continuous_batching/cache/cache_type.hpp), and a KV block spans every +# attention layer (kv_cache_manager.hpp), so sliding-window layers hold the whole context. +SWA_CACHE = {"vllm": "window", "llama.cpp": "llama.cpp", "ovms": "full"} # Tensor classes by name: the parts a placement moves separately. CLASSES = ( @@ -789,9 +795,10 @@ def model_type(cfg): return str(text_config(cfg).get("model_type") or cfg.get("model_type") or "").replace("_text", "") -def kv_profile(cfg, kv_bytes): +def kv_profile(cfg, kv_bytes, engine=None): """Per-token KV bytes for full-attention layers, per-token-within-window bytes for - sliding-window layers, and per-sequence state bytes for linear-attention layers.""" + sliding-window layers, and per-sequence state bytes for linear-attention layers. + engine sets how many tokens a sliding-window layer holds (swa_tokens).""" tc = text_config(cfg) n = tc.get("num_hidden_layers") types = tc.get("layer_types") or ["full_attention"] * n @@ -817,9 +824,16 @@ def kv_profile(cfg, kv_bytes): % (full, tc["indexer_kv_heads"], tc["indexer_head_dim"], _num(kv_bytes), full * idx)) window = tc.get("sliding_window") or 0 per_swa = 2 * kvh * hd * kv_bytes + swa_cache = SWA_CACHE.get(engine, "window") if swa: - lines.append("sliding window: %d layers x 2 x %d kv heads x %d head dim x %s B = %d B/token, at most %d tokens" - % (swa, kvh, hd, _num(kv_bytes), swa * per_swa, window)) + lines.append("sliding window: %d layers x 2 x %d kv heads x %d head dim x %s B = %d B/token, %s" + % (swa, kvh, hd, _num(kv_bytes), swa * per_swa, { + "full": "for the whole context: OpenVINO GenAI 2026.4 has no sliding-window cache type" + " (continuous_batching/cache/cache_type.hpp), every attention layer gets every token", + "llama.cpp": "for GGML_PAD(%d window + %d ubatch, 256) = %d tokens per sequence at most" + " (src/llama-kv-cache-iswa.cpp)" % (window, LLAMA_UBATCH, + _pad(window + LLAMA_UBATCH, 256)), + "window": "at most %d tokens" % window}[swa_cache])) state = 0 if lin: vh, kd = tc.get("linear_num_value_heads", 0), tc.get("linear_key_head_dim", 0) @@ -833,10 +847,15 @@ def kv_profile(cfg, kv_bytes): lines.append("linear attention state: %d layers x (%d v heads x %d x %d x %d B + conv (%d-1) x %d ch x 2 B)" " = %s per sequence, any length" % (lin, vh, kd, vd, ssm_b, ck, ch, _mb(state))) return {"full_layers": full, "full_per_token": full * (per_full + idx), "swa_layers": swa, - "swa_per_token": swa * per_swa, "window": window, "linear_layers": lin, + "swa_per_token": swa * per_swa, "window": window, "swa_cache": swa_cache, "linear_layers": lin, "state_per_seq": state, "kv_heads": gkvh, "lines": lines} +def _pad(n, to): + """GGML_PAD: n rounded up to a multiple of to.""" + return (n + to - 1) // to * to + + def _num(x): return ("%g" % x) if isinstance(x, float) else str(x) @@ -1013,8 +1032,21 @@ def act_bytes(m, tokens, seqs): return tokens * width * 2 * 2 + seqs * m["vocab"] * 4 +def swa_tokens(kv, ctx): + """Tokens one sequence's sliding-window layers hold at context ctx. vLLM: the window. + llama.cpp: its SWA cache of GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch), 256) cells per + stream (one stream per slot with -np set). OVMS: the whole context.""" + w = kv["window"] or ctx + mode = kv.get("swa_cache", "window") + if mode == "full": + return ctx + if mode == "llama.cpp": + return min(_pad(ctx, 256), _pad(w + LLAMA_UBATCH, 256)) + return min(ctx, w) + + def per_seq(kv, ctx, scale=1.0): - return int((kv["full_per_token"] * ctx + kv["swa_per_token"] * min(ctx, kv["window"] or ctx)) * scale + return int((kv["full_per_token"] * ctx + kv["swa_per_token"] * swa_tokens(kv, ctx)) * scale + kv["state_per_seq"] * scale) @@ -1110,7 +1142,7 @@ def fit(host, cfg, weights, engine=None, ctx=None, seqs=None, gpus=None, kv_dtyp raise ValueError("--kv-dtype %s is not a KV cache type of %s (%s)" % (kv_dtype, eng, ", ".join(KV_DTYPES_OF.get(eng, ())) or "auto only")) kvb = KV_DTYPE_BYTES.get(kv_dtype) or (1 if eng == "vllm" and m["kv_scheme_bits"] == 8 else m["dtype_bytes"]) - kvs[eng] = kv_profile(cfg, kvb) + kvs[eng] = kv_profile(cfg, kvb, eng) kvs[eng]["dtype"] = kv_dtype if kv_dtype != "auto" else ("fp8" if kvb == 1 else "auto") ex.append(" KV cache (%s B per element%s):" % (_num(kvb), ", the checkpoint's kv_cache_scheme" if kvb == 1 and kv_dtype == "auto" else "")) @@ -1155,7 +1187,7 @@ def fit(host, cfg, weights, engine=None, ctx=None, seqs=None, gpus=None, kv_dtyp rung_kv, rung_limit = [], [] for rs, rc in rungs: kb = per_seq(kvp, rc, picked["kv_scale"]) * rs - if picked["engine"] == "vllm" and picked["placement"] == "cpu": + if _whole_gib_pool(picked["engine"], picked["placement"]): kb = int(math.ceil(kb / GIB)) * GIB hb = picked["host_bytes"] - picked["kv_bytes"] + kb if picked["placement"] == "cpu" else picked["host_bytes"] rung_kv.append(int(kb)) @@ -1212,6 +1244,22 @@ def _res(eng, place, fits, why="", math_lines=(), **kw): return r +def _whole_gib_pool(eng, place): + """KV pools set in whole GiB: OVMS --cache_size, vLLM's CPU VLLM_CPU_KVCACHE_SPACE.""" + return eng == "ovms" or (eng == "vllm" and place == "cpu") + + +def _kv_room(eng, place, left): + """The KV room sequences are sized in: whole GiB for a whole-GiB pool, so the pool + rounded up to hold them stays inside left.""" + return left // GIB * GIB if _whole_gib_pool(eng, place) else left + + +def _kv_pool(eng, place, kvbytes): + """The KV bytes the engine sets aside for kvbytes of sequences.""" + return int(math.ceil(kvbytes / GIB)) * GIB if _whole_gib_pool(eng, place) else kvbytes + + def _dram_rate(bw, nbytes): return bw * 1e9 / nbytes if bw and nbytes > 0 else None @@ -1276,12 +1324,12 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, yield _res(eng, place, False, "weights + overhead %s exceed the GPU budget %s" % (g1(w + ov * (1 if eng == "vllm" else k)), g1(budget)), lines, gpus=k, tp=k if eng == "vllm" else 1) continue - c, n = size_kv(kv, left, ladder, want_seqs, scale, relaxed) + c, n = size_kv(kv, _kv_room(eng, place, left), ladder, want_seqs, scale, relaxed) if not c: yield _res(eng, place, False, "KV room %s holds no %d-token sequence" % (g1(left), ladder[-1]), lines, gpus=k) continue - kvused = per_seq(kv, c, scale) * n + kvused = _kv_pool(eng, place, per_seq(kv, c, scale) * n) lines.append("KV: %d seqs x %s at %d tokens = %s%s" % (n, _mb(per_seq(kv, c, scale)), c, g1(kvused), " per card" if eng == "vllm" else "")) # vLLM runs one worker process per tensor-parallel rank, each with its own runtime. @@ -1372,13 +1420,11 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, yield _res(eng, "cpu", False, "DRAM bandwidth gives %.1f tok/s, under the %g floor" % (toks, MIN_DRAM_TOK_S), lines) return - c, n = size_kv(kv, left, ladder, want_seqs, 1.0, relaxed) + c, n = size_kv(kv, _kv_room(eng, "cpu", left), ladder, want_seqs, 1.0, relaxed) if not c: yield _res(eng, "cpu", False, "KV room %s holds no %d-token sequence" % (g1(left), ladder[-1]), lines) return - kvused = per_seq(kv, c) * n - if eng == "vllm": - kvused = int(math.ceil(kvused / GIB)) * GIB # VLLM_CPU_KVCACHE_SPACE is whole GiB + kvused = _kv_pool(eng, "cpu", per_seq(kv, c) * n) lines.append("KV: %d seqs x %s at %d tokens = %s" % (n, _mb(per_seq(kv, c)), c, g1(kvused))) hb = int(m["serve_bytes"] + act + HOST_RUNTIME[eng] + kvused) yield _res(eng, "cpu", True, "", lines, ctx=c, seqs=n, host_bytes=hb, kv_bytes=int(kvused), diff --git a/docs/fit.md b/docs/fit.md index 3dfc62b5..c71604a1 100644 --- a/docs/fit.md +++ b/docs/fit.md @@ -80,7 +80,10 @@ Per placement: weights = bytes of each tensor class to serve (MTP left out unless the GGUF holds it) KV per token = full layers x 2 x kv heads x head dim x bytes + indexer keys (sparse attention) - + sliding layers x 2 x kv heads x head dim x bytes, for at most the window + + sliding layers x 2 x kv heads x head dim x bytes, for: + vLLM at most the window + llama.cpp at most GGML_PAD(window + 512 ubatch, 256) tokens per slot + OVMS the whole context (OpenVINO GenAI has no sliding-window cache) state per seq = linear layers x (v heads x k dim x v dim x 4 B + conv state) overhead (GPU) = runtime + activations (batch x widest matmul x 2 x 2 B) + logits (seqs x vocab x 4 B) host reserve = max(10% of MemTotal, 4 GiB); host budget = min(MemAvailable, cgroup room) - reserve @@ -101,6 +104,14 @@ llama.cpp keeps the token embeddings and the n-gram table on the CPU by design bytes on every llama.cpp placement. OpenVINO's GPU plugin compiles from a host copy, so an OVMS GPU placement counts the weights as host bytes too. +A sliding-window layer's cache is engine-specific. llama.cpp b11749 gives each slot +`GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch), 256)` cells (`src/llama-kv-cache-iswa.cpp`), +so Gemma 4's 1024-token window holds 1536 cells. OpenVINO GenAI 2026.4 has only a +full-attention KV cache and a linear-attention state cache +(`src/cpp/src/continuous_batching/cache/cache_type.hpp`), and one KV block spans every +attention layer, so OVMS holds every token in the sliding-window layers too: Gemma 4 +costs 225280 B per token there (5 full and 25 sliding layers, K and V at 2 B). + The overhead is an estimate from the formula, not a measurement; `K3D_GPU_FIT_OVERHEAD_GIB` replaces it with a measured value. @@ -109,7 +120,7 @@ The overhead is an estimate from the formula, not a measurement; | Engine | Flags (spellings from each engine's own argument parser) | | --- | --- | | vLLM v0.32.0 | `--tensor-parallel-size`, `--max-model-len`, `--max-num-seqs`, `--gpu-memory-utilization` (vLLM still profiles activations inside that share), `--kv-cache-dtype fp8` when the checkpoint's `kv_cache_scheme` made the fit count one byte. CPU build (EAT): `VLLM_CPU_KVCACHE_SPACE` | -| OVMS 2026.4.1 | `--max_num_seqs`, `--cache_size` (a static pool: the dynamic one grows past the fit) | +| OVMS 2026.4.1 | `--max_num_seqs`, `--cache_size` (a static pool in whole GiB, rounded up to hold every sequence: the dynamic one grows past the fit) | | llama.cpp v0.8.0 | `-c` (context x slots), `-np`, `-ngl`, `-ts`, `--n-cpu-moe` or `--cpu-moe`, `-ctk`/`-ctv` | `agent up` applies them, sets the pod's memory request and limit to the fit, and diff --git a/tests/fit_test.py b/tests/fit_test.py index df5d56f9..f18873c1 100755 --- a/tests/fit_test.py +++ b/tests/fit_test.py @@ -258,6 +258,19 @@ def test_gemma4_hybrid_swa(self): self.assertEqual(kv["state_per_seq"], 0) self.assertEqual(fit.per_seq(kv, 131072), 5 * 2 * 2 * 512 * 2 * 131072 + 25 * 2 * 8 * 256 * 2 * 1024) + def test_swa_cache_per_engine(self): + cfg = config("google__gemma-4-26B-A4B-it") + full, swa = 5 * 2 * 2 * 512 * 2, 25 * 2 * 8 * 256 * 2 + self.assertEqual(full + swa, 225280) + # vLLM: the window + self.assertEqual(fit.per_seq(fit.kv_profile(cfg, 2, "vllm"), 131072), full * 131072 + swa * 1024) + # llama.cpp b11749: GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch 512), 256) cells per stream + kv = fit.kv_profile(cfg, 2, "llama.cpp") + self.assertEqual(fit.per_seq(kv, 131072), full * 131072 + swa * 1536) + self.assertEqual(fit.per_seq(kv, 1024), (full + swa) * 1024) + # OVMS (OpenVINO GenAI 2026.4): no sliding-window cache type, every layer at full context + self.assertEqual(fit.per_seq(fit.kv_profile(cfg, 2, "ovms"), 131072), 225280 * 131072) + def test_qwen38_gated_deltanet(self): kv = fit.kv_profile(config("Qwen__Qwen3.8-27B"), 2) self.assertEqual(kv["full_per_token"], 16 * 2 * 4 * 256 * 2) @@ -378,10 +391,28 @@ def test_gemma_qat_gguf(self): d = plan("xeon6741p-4xb70", "google__gemma-4-26B-A4B-it-qat-q4_0-gguf") self.assertTrue(d["fits"], d["summary"]) self.assertEqual((d["engine"], d["placement"], d["gpus"]), ("llama.cpp", "gpu", 1)) - self.assertEqual((d["ctx"], d["seqs"]), (131072, 6)) + # each slot's SWA cache is GGML_PAD(1024 + 512, 256) = 1536 cells, not the 1024 window + self.assertEqual((d["ctx"], d["seqs"]), (131072, 5)) self.assertFalse(d["degraded"]) self.assertIn("-ngl", d["flags"]) + def test_gemma_ovms_cache_holds_every_token(self): + # OVMS gives Gemma 4's 25 sliding-window layers the whole context: 225280 B per token, + # and --cache_size (whole GiB) holds context x sequences at that, on every guard rung. + one = max(c["budget"] for c in fit.gpu_budget(host("xeon6741p-4xb70"))[0]) + for hname, dev in (("xeon6741p-4xb70", "gpu"), ("xeon6741p-4xb70", "cpu"), ("desktop-i9-nogpu", None)): + d = plan(hname, "OpenVINO__gemma-4-26b-a4b-it-int4-ov", device=dev) + self.assertTrue(d["fits"], (hname, d["summary"])) + self.assertEqual(d["engine"], "ovms") + cache = [int(f.split("=")[1]) for f in d["flags"] if f.startswith("--cache_size=")][0] + self.assertGreaterEqual(cache * GIB, d["ctx"] * d["seqs"] * 225280, (hname, d["flags"])) + self.assertEqual(cache * GIB, d["kv_bytes"], hname) + if d["placement"] == "gpu": + self.assertLessEqual(d["gpu_bytes"], one) + for r, (s, c) in enumerate(d["guard"]["rungs"]): + k = dict(line.split("=", 1) for line in fit.knobs(d, r).splitlines()) + self.assertGreaterEqual(int(k["cache_gib"]) * GIB, s * c * 225280, (hname, r)) + def test_vllm_host_side_per_rank(self): d = plan("xeon6741p-4xb70", "Qwen__Qwen3.8-27B") # one runtime per tensor-parallel rank, plus the /dev/shm emptyDir diff --git a/tests/fit_test.sh b/tests/fit_test.sh index f7398d47..436e348b 100755 --- a/tests/fit_test.sh +++ b/tests/fit_test.sh @@ -168,6 +168,17 @@ if render ovms2 STUB_ALLOC=$'cpu 64\ngpu.intel.com/xe 4' STUB_NODE_LABELS=gpu.in else fail "agent manifest (ovms, K3D_GPU_AGENT_GPUS=2) sized for one card (cache ${cache:-?}, one card ${one_card})" \ "$(cat "${WORK}/ovms2.err"; grep -E 'cache_size|agent-fit|gpu.intel.com/xe' "${WORK}/ovms2.yaml")"; fi +# OVMS has no sliding-window cache: Gemma 4's 25 SWA layers hold the whole context, so the +# static pool holds seqs x context x 225280 B (5 full + 25 SWA layers' K and V at 2 B). +if render ovms-swa STUB_ALLOC=$'cpu 64\ngpu.intel.com/xe 4' STUB_NODE_LABELS=gpu.intel.com/xmx=true K3D_GPU_FIT_HOST="${XEON}" \ + K3D_GPU_AGENT_MODEL=OpenVINO/gemma-4-26b-a4b-it-int4-ov -- \ + && cache=$(sed -n 's/^ *- --cache_size=\([0-9]*\)$/\1/p' "${WORK}/ovms-swa.yaml") \ + && sc=$(sed -n 's/^ *k3d-gpu.io\/agent-fit: "gpu \([0-9]*x[0-9]*\)"$/\1/p' "${WORK}/ovms-swa.yaml") \ + && [ -n "${cache}" ] && [ -n "${sc}" ] && [ $(( cache * 1073741824 )) -ge $(( ${sc%x*} * ${sc#*x} * 225280 )) ]; then + ok "agent manifest (ovms, Gemma 4 on a GPU): --cache_size=${cache} holds ${sc} tokens at 225280 B each" +else fail "agent manifest (ovms, Gemma 4): --cache_size holds seqs x context at 225280 B/token (cache ${cache:-?}, ${sc:-?})" \ + "$(cat "${WORK}/ovms-swa.err"; grep -E 'cache_size|agent-fit' "${WORK}/ovms-swa.yaml")"; fi + if render tier STUB_ALLOC='cpu 32' K3D_GPU_FIT_HOST="${LAPTOP}" K3D_GPU_AGENT_MODEL=OpenVINO/Qwen3.8-27B-int8-ov \ K3D_GPU_AGENT_MODEL_TIER=1 -- && ! has tier 'agent-fit'; then ok "a quickstart tier pick (K3D_GPU_AGENT_MODEL_TIER=1) is not sized: it is no user choice" From 57cf38d893ae7928e85775f1d9e152a70c6c3ee0 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Wed, 28 Oct 2026 22:49:18 -0700 Subject: [PATCH 03/13] fit: count llama-server's host caches llama-server b11749 keeps two host caches beside the KV cache that the fit did not count: a prompt cache of up to 8192 MiB (--cache-ram) and up to 32 context checkpoints per slot (-ctxcp), each a host copy of the slot's SWA cache and recurrent state. For the Gemma QAT GGUF on the desktop CPU plan (5 slots at 262144, KV filling the room) that is 8 GiB plus about 47 GiB of checkpoints over a 45Gi pod limit. The llama.cpp flags now carry --cache-ram 0 and -ctxcp 4, and the fit counts slots x 4 x one checkpoint (the SWA cache of GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch), 256) cells plus the recurrent state) in the host bytes, the pod limit and every guard rung. On the CPU placement the checkpoints share the KV room, so the slot count accounts for them; on the GPU and experts-in-DRAM placements they are host bytes. A model with neither sliding-window nor linear-attention layers makes none. Tests: the Gemma QAT CPU plan on desktop-i9-nogpu emits --cache-ram 0 and -ctxcp K and counts seqs x K x 300 MiB in the host bytes, the pod limit and each rung's limit; Flash-Next's checkpoints are its recurrent state; a full-attention model makes none; k3d-gpu fit --json on the desktop shows the same. --- console/fit.py | 67 +++++++++++++++++++++++++++++++++++------------ docs/fit.md | 12 ++++++++- tests/fit_test.py | 52 ++++++++++++++++++++++++++++++++++++ tests/fit_test.sh | 17 ++++++++++++ 4 files changed, 130 insertions(+), 18 deletions(-) diff --git a/console/fit.py b/console/fit.py index 6c9bb2ca..54cf1d80 100644 --- a/console/fit.py +++ b/console/fit.py @@ -94,6 +94,13 @@ # the agent stack sets it, llama.cpp's default -ub 512). VLLM_BATCHED_TOKENS = 2048 LLAMA_UBATCH = 512 +# llama-server context checkpoints per slot (-ctxcp; b11749 default 32, common/common.h). Each +# is a host-RAM copy of the slot's SWA cache and recurrent state (tools/server/server-context.cpp +# create_checkpoint, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY). A request makes them at the last user +# message and 4 + n_ubatch and 4 tokens before its prompt's end; an appended turn restores from +# those, so four keep the latest request's set. The host prompt cache (--cache-ram, default +# 8192 MiB) is turned off with --cache-ram 0: a slot's KV stays in its own cache. +LLAMA_CTX_CHECKPOINTS = 4 # Fixed runtime per GPU process (device context, allocator pools, collective # buffers) and per pod on the host (Python, torch or the server binary). Estimates. GPU_RUNTIME = 1 * GIB @@ -1050,6 +1057,15 @@ def per_seq(kv, ctx, scale=1.0): + kv["state_per_seq"] * scale) +def ckpt_seq(kv, ctx): + """Host bytes of one llama-server slot's context checkpoints at context ctx: kv["ckpts"] + of them, each the slot's SWA cache (GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch), 256) + cells, src/llama-kv-cache-iswa.cpp) and recurrent state. A model with neither makes none.""" + w = kv["window"] or ctx + one = kv["swa_per_token"] * min(_pad(ctx, 256), _pad(w + LLAMA_UBATCH, 256)) + kv["state_per_seq"] + return kv.get("ckpts", 0) * one + + def ctx_ladder(max_ctx, want=None): if want: return [min(want, max_ctx)] @@ -1060,13 +1076,14 @@ def ctx_ladder(max_ctx, want=None): return out or [max_ctx] -def size_kv(kv, budget, ladder, want_seqs, scale=1.0, relaxed=False): +def size_kv(kv, budget, ladder, want_seqs, scale=1.0, relaxed=False, host_ckpt=False): """(ctx, seqs): the most context at which TARGET_SEQS (or want_seqs) whole sequences fit in budget, then as many sequences as fit at that context (capped at MAX_SEQS). - relaxed: at the ladder's last rung one sequence is enough.""" + relaxed: at the ladder's last rung one sequence is enough. host_ckpt: the budget also + holds each sequence's context checkpoints (KV in host RAM).""" need = want_seqs or TARGET_SEQS for ctx in ladder: - s = per_seq(kv, ctx, scale) + s = per_seq(kv, ctx, scale) + (ckpt_seq(kv, ctx) if host_ckpt else 0) n = int(budget // s) if s else MAX_SEQS if n >= need or (relaxed and ctx == ladder[-1] and n >= 1 and not want_seqs): return ctx, min(want_seqs or n, MAX_SEQS) @@ -1144,9 +1161,14 @@ def fit(host, cfg, weights, engine=None, ctx=None, seqs=None, gpus=None, kv_dtyp kvb = KV_DTYPE_BYTES.get(kv_dtype) or (1 if eng == "vllm" and m["kv_scheme_bits"] == 8 else m["dtype_bytes"]) kvs[eng] = kv_profile(cfg, kvb, eng) kvs[eng]["dtype"] = kv_dtype if kv_dtype != "auto" else ("fp8" if kvb == 1 else "auto") + kvs[eng]["ckpts"] = LLAMA_CTX_CHECKPOINTS if eng == "llama.cpp" else 0 ex.append(" KV cache (%s B per element%s):" % (_num(kvb), ", the checkpoint's kv_cache_scheme" if kvb == 1 and kv_dtype == "auto" else "")) ex += [" " + s for s in kvs[eng]["lines"]] + if eng == "llama.cpp": + ex.append(" host prompt cache off (--cache-ram 0); context checkpoints (-ctxcp %d) in host RAM: " + "%d x the slot's SWA cache and recurrent state = %s per slot at %d tokens" + % (LLAMA_CTX_CHECKPOINTS, LLAMA_CTX_CHECKPOINTS, _mb(ckpt_seq(kvs[eng], full[0])), full[0])) tried, picked, degraded = [], None, False for i, (label, ladder, relaxed) in enumerate(passes): ex.append("placements, %s:" % label) @@ -1189,11 +1211,12 @@ def fit(host, cfg, weights, engine=None, ctx=None, seqs=None, gpus=None, kv_dtyp kb = per_seq(kvp, rc, picked["kv_scale"]) * rs if _whole_gib_pool(picked["engine"], picked["placement"]): kb = int(math.ceil(kb / GIB)) * GIB - hb = picked["host_bytes"] - picked["kv_bytes"] + kb if picked["placement"] == "cpu" else picked["host_bytes"] + hb = (picked["host_bytes"] - picked["kv_bytes"] - picked["ckpt_bytes"] + kb + ckpt_seq(kvp, rc) * rs + if picked["placement"] == "cpu" else picked["host_bytes"]) rung_kv.append(int(kb)) rung_limit.append("%dGi" % int(math.ceil(hb * (1 + LIMIT_SLACK) / GIB))) doc.update({k: picked[k] for k in ("engine", "placement", "ctx", "seqs", "tp", "gpus", "gpu_bytes", "host_bytes", - "flags", "dram_tok_s", "note", "kv_bytes")}) + "flags", "dram_tok_s", "note", "kv_bytes", "ckpt_bytes")}) doc["source"] = weights.get("source") doc["weight_bytes"] = weights.get("total_bytes") or m["serve_bytes"] if m["format"] == "gguf": @@ -1239,7 +1262,7 @@ def _overhead(eng, m, seqs, override): def _res(eng, place, fits, why="", math_lines=(), **kw): r = {"engine": eng, "placement": place, "fits": fits, "why": why, "math": list(math_lines), "ctx": None, "seqs": 0, "tp": 1, "gpus": 0, "gpu_bytes": 0, "host_bytes": 0, "flags": [], "dram_tok_s": None, - "kv_bytes": 0, "kv_scale": 1.0} + "kv_bytes": 0, "kv_scale": 1.0, "ckpt_bytes": 0} r.update(kw) return r @@ -1333,13 +1356,16 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, lines.append("KV: %d seqs x %s at %d tokens = %s%s" % (n, _mb(per_seq(kv, c, scale)), c, g1(kvused), " per card" if eng == "vllm" else "")) # vLLM runs one worker process per tensor-parallel rank, each with its own runtime. - hb = int(hostrt + HOST_RUNTIME[eng] * (k - 1) * (eng == "vllm") + on_host + compile_host) + ck = ckpt_seq(kv, c) * n + if ck: + lines.append("context checkpoints: %d seqs x %s = %s in host RAM" % (n, _mb(ckpt_seq(kv, c)), g1(ck))) + hb = int(hostrt + HOST_RUNTIME[eng] * (k - 1) * (eng == "vllm") + on_host + compile_host + ck) if hb > hfree: yield _res(eng, place, False, "host side %s exceeds the host budget %s" % (g1(hb), g1(hfree)), lines, gpus=k) continue gpu_use = int(w + ov + kvused) if eng == "vllm" else int((w + ov * k + kvused) / k) yield _res(eng, place, True, "", lines, ctx=c, seqs=n, tp=k if eng == "vllm" else 1, gpus=k, - gpu_bytes=gpu_use, host_bytes=hb, kv_bytes=int(kvused), kv_scale=scale, + gpu_bytes=gpu_use, host_bytes=hb, kv_bytes=int(kvused), kv_scale=scale, ckpt_bytes=ck, flags=_flags(eng, place, c, n, k, gpu_use, use, kvused, m, 0, kv)) # MoE experts to DRAM (llama.cpp --n-cpu-moe): every free card, fewest CPU layers. if eng == "llama.cpp" and cards and m["moe"]: @@ -1385,9 +1411,10 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, % (bw, g1(dram_w), m["experts_per_tok"], m["experts"], toks, MIN_DRAM_TOK_S)) kvused = per_seq(kv, c) * n lines.append("KV: %d seqs x %s at %d tokens = %s on GPU" % (n, _mb(per_seq(kv, c)), c, g1(kvused))) - hb = int(dram_w + on_host + HOST_RUNTIME[eng]) - lines.append("host: experts %s + lookup tables %s + runtime %s = %s" - % (g1(dram_w), g1(on_host), g1(HOST_RUNTIME[eng]), g1(hb))) + ck = ckpt_seq(kv, c) * n + hb = int(dram_w + on_host + HOST_RUNTIME[eng] + ck) + lines.append("host: experts %s + lookup tables %s + runtime %s + context checkpoints %d x %s = %s" + % (g1(dram_w), g1(on_host), g1(HOST_RUNTIME[eng]), n, _mb(ckpt_seq(kv, c)), g1(hb))) if toks < MIN_DRAM_TOK_S: yield _res(eng, "moe-offload", False, "DRAM bandwidth gives %.1f tok/s, under the %g floor" % (toks, MIN_DRAM_TOK_S), lines, gpus=k) @@ -1397,7 +1424,7 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, else: gpu_use = int((gw + ov * k + kvused) / k) yield _res(eng, "moe-offload", True, "", lines, ctx=c, seqs=n, gpus=k, gpu_bytes=gpu_use, - host_bytes=hb, dram_tok_s=round(toks, 1), kv_bytes=int(kvused), + host_bytes=hb, dram_tok_s=round(toks, 1), kv_bytes=int(kvused), ckpt_bytes=ck, flags=_flags(eng, "moe-offload", c, n, k, gpu_use, use, kvused, m, n_cpu, kv)) # CPU only: weights, KV and activations all in DRAM (vLLM: its CPU build, the KV # pool sized by VLLM_CPU_KVCACHE_SPACE, as EAT's vLLM chart sets it). @@ -1408,8 +1435,9 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, if m["moe"]: active -= m["bytes"]["experts"] * (1 - m["experts_per_tok"] / m["experts"]) toks = _dram_rate(bw, active) - lines = ["host budget / %.2f slack %s - weights %s - activations %s - runtime %s = %s for KV" - % (1 + LIMIT_SLACK, g1(hfree), g1(m["serve_bytes"]), g1(act), g1(HOST_RUNTIME[eng]), g1(max(left, 0)))] + lines = ["host budget / %.2f slack %s - weights %s - activations %s - runtime %s = %s for KV%s" + % (1 + LIMIT_SLACK, g1(hfree), g1(m["serve_bytes"]), g1(act), g1(HOST_RUNTIME[eng]), g1(max(left, 0)), + " and context checkpoints" if kv.get("ckpts") else "")] if toks: lines.append("DRAM decode bound: %g GB/s / %s read per token = %.1f tok/s" % (bw, g1(active), toks)) if left <= 0: @@ -1420,14 +1448,17 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, yield _res(eng, "cpu", False, "DRAM bandwidth gives %.1f tok/s, under the %g floor" % (toks, MIN_DRAM_TOK_S), lines) return - c, n = size_kv(kv, _kv_room(eng, "cpu", left), ladder, want_seqs, 1.0, relaxed) + c, n = size_kv(kv, _kv_room(eng, "cpu", left), ladder, want_seqs, 1.0, relaxed, host_ckpt=True) if not c: yield _res(eng, "cpu", False, "KV room %s holds no %d-token sequence" % (g1(left), ladder[-1]), lines) return kvused = _kv_pool(eng, "cpu", per_seq(kv, c) * n) lines.append("KV: %d seqs x %s at %d tokens = %s" % (n, _mb(per_seq(kv, c)), c, g1(kvused))) - hb = int(m["serve_bytes"] + act + HOST_RUNTIME[eng] + kvused) - yield _res(eng, "cpu", True, "", lines, ctx=c, seqs=n, host_bytes=hb, kv_bytes=int(kvused), + ck = ckpt_seq(kv, c) * n + if ck: + lines.append("context checkpoints: %d seqs x %s = %s" % (n, _mb(ckpt_seq(kv, c)), g1(ck))) + hb = int(m["serve_bytes"] + act + HOST_RUNTIME[eng] + kvused + ck) + yield _res(eng, "cpu", True, "", lines, ctx=c, seqs=n, host_bytes=hb, kv_bytes=int(kvused), ckpt_bytes=ck, dram_tok_s=round(toks, 1) if toks else None, flags=_flags(eng, "cpu", c, n, 1, 0, [], kvused, m, 0, kv)) @@ -1458,6 +1489,8 @@ def _flags(eng, place, ctx, seqs, k, gpu_use, cards, kvused, m, n_cpu_moe, kv=No out += ["-ts", ",".join(str(max(int(c["budget"] // GIB), 1)) for c in cards)] if place == "moe-offload": out += ["--cpu-moe"] if n_cpu_moe >= m["layers"] else ["--n-cpu-moe", str(n_cpu_moe)] + # -cram/--cache-ram, -ctxcp/--ctx-checkpoints: the host caches the fit counts (LLAMA_CTX_CHECKPOINTS) + out += ["--cache-ram", "0", "-ctxcp", str(LLAMA_CTX_CHECKPOINTS)] t = {"fp16": "f16"}.get(kv.get("dtype"), kv.get("dtype")) if t in ("f16", "bf16", "q8_0", "q4_0"): out += ["-ctk", t, "-ctv", t] # -ctk/--cache-type-k, -ctv/--cache-type-v diff --git a/docs/fit.md b/docs/fit.md index c71604a1..321772a2 100644 --- a/docs/fit.md +++ b/docs/fit.md @@ -85,6 +85,7 @@ KV per token = full layers x 2 x kv heads x head dim x bytes llama.cpp at most GGML_PAD(window + 512 ubatch, 256) tokens per slot OVMS the whole context (OpenVINO GenAI has no sliding-window cache) state per seq = linear layers x (v heads x k dim x v dim x 4 B + conv state) +checkpoints = llama.cpp: 4 per slot in host RAM, each the slot's SWA cache and state overhead (GPU) = runtime + activations (batch x widest matmul x 2 x 2 B) + logits (seqs x vocab x 4 B) host reserve = max(10% of MemTotal, 4 GiB); host budget = min(MemAvailable, cgroup room) - reserve GPU reserve = max(5% of VRAM, 1 GiB); GPU budget = VRAM - in use by others - reserve @@ -112,6 +113,15 @@ full-attention KV cache and a linear-attention state cache attention layer, so OVMS holds every token in the sliding-window layers too: Gemma 4 costs 225280 B per token there (5 full and 25 sliding layers, K and V at 2 B). +llama-server b11749 keeps two host caches beside the KV cache: a prompt cache of up to +8192 MiB (`--cache-ram`) and up to 32 context checkpoints per slot (`-ctxcp`), each a +host copy of the slot's SWA cache and recurrent state (`tools/server/server-context.cpp`, +`create_checkpoint`). The fit turns the prompt cache off (`--cache-ram 0`), keeps 4 +checkpoints per slot (a request makes them at the last user message and just before its +prompt's end, which is what the next turn restores from), and counts slots x 4 x one +checkpoint in the host bytes and the pod limit. A model with no sliding-window and no +linear-attention layers makes none. + The overhead is an estimate from the formula, not a measurement; `K3D_GPU_FIT_OVERHEAD_GIB` replaces it with a measured value. @@ -121,7 +131,7 @@ The overhead is an estimate from the formula, not a measurement; | --- | --- | | vLLM v0.32.0 | `--tensor-parallel-size`, `--max-model-len`, `--max-num-seqs`, `--gpu-memory-utilization` (vLLM still profiles activations inside that share), `--kv-cache-dtype fp8` when the checkpoint's `kv_cache_scheme` made the fit count one byte. CPU build (EAT): `VLLM_CPU_KVCACHE_SPACE` | | OVMS 2026.4.1 | `--max_num_seqs`, `--cache_size` (a static pool in whole GiB, rounded up to hold every sequence: the dynamic one grows past the fit) | -| llama.cpp v0.8.0 | `-c` (context x slots), `-np`, `-ngl`, `-ts`, `--n-cpu-moe` or `--cpu-moe`, `-ctk`/`-ctv` | +| llama.cpp v0.8.0 | `-c` (context x slots), `-np`, `-ngl`, `-ts`, `--n-cpu-moe` or `--cpu-moe`, `--cache-ram 0`, `-ctxcp 4`, `-ctk`/`-ctv` | `agent up` applies them, sets the pod's memory request and limit to the fit, and annotates the pod `k3d-gpu.io/agent-fit`. `K3D_GPU_AGENT_MAX_LEN`, `K3D_GPU_AGENT_MAX_NUM_SEQS` diff --git a/tests/fit_test.py b/tests/fit_test.py index f18873c1..7dd9b3e6 100755 --- a/tests/fit_test.py +++ b/tests/fit_test.py @@ -470,6 +470,58 @@ def test_no_moe_offload_without_amx_or_avx512(self): self.assertIn("exceed the host budget", d["reason"]) +class LlamaHostCaches(unittest.TestCase): + """llama-server b11749 keeps an 8192 MiB host prompt cache (--cache-ram) and up to 32 + context checkpoints per slot (-ctxcp), each a slot's SWA cache or recurrent state, in + host RAM. The fit turns the prompt cache off, bounds the checkpoints, and counts them.""" + + def flag(self, d, name): + i = d["flags"].index(name) + return int(d["flags"][i + 1]) + + def test_gemma_qat_cpu_on_the_desktop(self): + d = plan("desktop-i9-nogpu", "google__gemma-4-26B-A4B-it-qat-q4_0-gguf") + self.assertTrue(d["fits"], d["summary"]) + self.assertEqual((d["engine"], d["placement"]), ("llama.cpp", "cpu")) + self.assertEqual(self.flag(d, "--cache-ram"), 0) + k = self.flag(d, "-ctxcp") + self.assertGreaterEqual(k, 1) + # one checkpoint: 25 SWA layers x 2 x 8 kv heads x 256 x 2 B over the slot's SWA + # cache, GGML_PAD(1024 window + 512 ubatch, 256) = 1536 cells + ckpt = 25 * 2 * 8 * 256 * 2 * 1536 + self.assertGreaterEqual(d["ckpt_bytes"], d["seqs"] * k * ckpt) + cfg = config("google__gemma-4-26B-A4B-it-qat-q4_0-gguf") + w = json.load(open(os.path.join(mdir("google__gemma-4-26B-A4B-it-qat-q4_0-gguf"), "weights.json"))) + m = fit.model_profile(cfg, w) + self.assertGreaterEqual(d["host_bytes"], m["serve_bytes"] + fit.HOST_RUNTIME["llama.cpp"] + d["kv_bytes"] + + d["seqs"] * k * ckpt) + # the pod limit holds the checkpoints and stays inside the host budget + budget, _, _ = fit.host_budget(host("desktop-i9-nogpu")) + self.assertGreaterEqual(int(d["k8s"]["memory_limit"][:-2]) * GIB, d["host_bytes"]) + self.assertLessEqual(int(d["k8s"]["memory_limit"][:-2]) * GIB, budget) + # and on every guard rung + for r, (rs, rc) in enumerate(d["guard"]["rungs"]): + self.assertGreaterEqual(int(d["guard"]["rung_memory_limit"][r][:-2]) * GIB, + d["host_bytes"] - d["kv_bytes"] - d["ckpt_bytes"] + + d["guard"]["rung_kv_bytes"][r] + rs * k * ckpt, r) + + def test_recurrent_state_checkpoints_on_the_host(self): + # Flash-Next: 36 gated-DeltaNet layers, each checkpoint the per-sequence state + d = plan("xeon6741p-4xb70", "ggml-org__Qwen3.8-Flash-Next-GGUF") + self.assertTrue(d["fits"], d["summary"]) + k = self.flag(d, "-ctxcp") + state = 36 * (48 * 128 * 128 * 4 + 3 * (2 * 16 * 128 + 48 * 128) * 2) + self.assertEqual(d["ckpt_bytes"], d["seqs"] * k * state) + self.assertIn("--cache-ram", d["flags"]) + + def test_full_attention_makes_no_checkpoints(self): + cfg = config("Qwen__Qwen3.8-27B") + cfg = dict(cfg, text_config=dict(cfg.get("text_config") or cfg, layer_types=None)) + kv = fit.kv_profile(cfg, 2) + kv["ckpts"] = 4 + self.assertEqual(fit.ckpt_seq(kv, 65536), 0) + + class FitLaptop(unittest.TestCase): """Core Ultra 7 258V, 16 GB, 9 GB available, swap in use.""" diff --git a/tests/fit_test.sh b/tests/fit_test.sh index 436e348b..69c2ec32 100755 --- a/tests/fit_test.sh +++ b/tests/fit_test.sh @@ -69,6 +69,23 @@ if [ "${rc}" -eq 0 ] && python3 -c 'import json, sys; d = json.loads(sys.argv[1] ok "fit --json: a fit/v1 document, from a local model directory" else fail "fit --json: a fit/v1 document" "${out}"; fi +# llama-server keeps a host prompt cache (8192 MiB by default) and context checkpoints per +# slot (32 by default, each the slot's SWA cache): the fit turns the first off, bounds the +# second, and counts the checkpoints in the host bytes and the pod limit. +out=$(bash "${K3D_GPU}" fit --host "${DESK}" --model google/gemma-4-26B-A4B-it-qat-q4_0-gguf --json 2>&1); rc=$? +if [ "${rc}" -eq 0 ] && python3 -c ' +import json, sys +d = json.loads(sys.argv[1]) +f = d["flags"] +assert d["placement"] == "cpu" and f[f.index("--cache-ram") + 1] == "0", f +k = int(f[f.index("-ctxcp") + 1]) +ckpt = 25 * 2 * 8 * 256 * 2 * 1536 # 25 SWA layers K and V at 2 B over 1536 cells +assert d["ckpt_bytes"] >= d["seqs"] * k * ckpt > 0, (d["ckpt_bytes"], d["seqs"], k) +assert int(d["k8s"]["memory_limit"][:-2]) * 2 ** 30 >= d["host_bytes"] >= d["kv_bytes"] + d["ckpt_bytes"] +' "${out}"; then + ok "fit (llama.cpp, Gemma QAT on the desktop CPU): --cache-ram 0, -ctxcp K, checkpoints in the pod limit" +else fail "fit (llama.cpp host caches) on the desktop (rc ${rc})" "${out}"; fi + out=$(bash "${K3D_GPU}" fit --host "${XEON}" --model not/in-cache-offline 2>&1 Date: Wed, 28 Oct 2026 22:58:19 -0700 Subject: [PATCH 04/13] fit: size llama.cpp's layer split card by card On several GPUs the fit treated llama.cpp's cards as one pool: the summed budgets less the summed weights, KV and overhead. It then wrote -ts as each card's budget in GiB. llama.cpp places whole layers, though, and with --n-cpu-moe the layers whose experts stay on the GPU are the last ones. The Xeon fixture's Flash-Next IQ4_NL plan, -ts 30,30 --n-cpu-moe 28 at 262144 x 4, therefore put 43.5 GiB on the second 32 GiB card. The fit now sizes each card from the layers llama.cpp b11749 gives it with -ngl 999 (src/llama-model.cpp load_tensors): - layer il goes to the device upper_bound(cumsum(ts) / sum, il / (layers + 1)); - the output layer, holding lm_head, goes the same way and lands on the last card; - the first --n-cpu-moe layers keep their experts in host RAM; - each layer's KV or recurrent state sits on its card; - every card carries its own overhead. -ts is now integer layer counts summing to layers + 1, so each card gets exactly the layers it was sized for. Every card is held to the smallest budget of the cards used, because the device plugin orders them inside the pod. The largest card load is balanced, and a split that cannot fit is refused. The experts-in-DRAM placement uses the same per-card search, so -ts and --n-cpu-moe are chosen together. --explain lists each card, and --json has card_bytes. Flash-Next IQ4_NL on the Xeon is now -ts 35,14 --n-cpu-moe 29 at 262144 x 4: 29.9 and 28.9 GiB on cards with 30.4 and 30.2 GiB budgets. Tests: - llama_layer_devices matches the upper_bound rule. - The old -ts 30,30 --n-cpu-moe 28 plan works out to 43.5 GiB on its second card. - For the Flash-Next IQ4_NL plan and a two-card Gemma QAT layer split, each card's bytes, recomputed in the test from the flags, stay within budget and agree with card_bytes. - A split that cannot fit is refused. - fit_test.sh checks the Flash-Next plan's -ts and card_bytes. --- console/fit.py | 240 +++++++++++++++++++++++++++++++++------------- docs/fit.md | 19 +++- tests/fit_test.py | 101 +++++++++++++++++++ tests/fit_test.sh | 19 ++++ 4 files changed, 312 insertions(+), 67 deletions(-) diff --git a/console/fit.py b/console/fit.py index 54cf1d80..b71104a5 100644 --- a/console/fit.py +++ b/console/fit.py @@ -48,6 +48,7 @@ """ import argparse +import bisect import glob import json import math @@ -812,10 +813,10 @@ def kv_profile(cfg, kv_bytes, engine=None): heads = tc.get("num_attention_heads") kvh = tc.get("num_key_value_heads") or heads hd = tc.get("head_dim") or tc["hidden_size"] // heads - full = sum(1 for t in types if t == "full_attention") - swa = sum(1 for t in types if t in ("sliding_attention", "sliding_window_attention", "local_attention")) - lin = sum(1 for t in types if t in ("linear_attention", "mamba", "gated_delta_net")) - full += len(types) - full - swa - lin # an unknown layer type counts as full attention (upper bound) + # an unknown layer type counts as full attention (upper bound) + kinds = ["swa" if t in ("sliding_attention", "sliding_window_attention", "local_attention") else + "linear" if t in ("linear_attention", "mamba", "gated_delta_net") else "full" for t in types] + full, swa, lin = kinds.count("full"), kinds.count("swa"), kinds.count("linear") gkvh = tc.get("num_global_key_value_heads") or kvh ghd = tc.get("global_head_dim") or hd lines = [] @@ -855,7 +856,7 @@ def kv_profile(cfg, kv_bytes, engine=None): " = %s per sequence, any length" % (lin, vh, kd, vd, ssm_b, ck, ch, _mb(state))) return {"full_layers": full, "full_per_token": full * (per_full + idx), "swa_layers": swa, "swa_per_token": swa * per_swa, "window": window, "swa_cache": swa_cache, "linear_layers": lin, - "state_per_seq": state, "kv_heads": gkvh, "lines": lines} + "state_per_seq": state, "kv_heads": gkvh, "kinds": kinds, "lines": lines} def _pad(n, to): @@ -1057,6 +1058,113 @@ def per_seq(kv, ctx, scale=1.0): + kv["state_per_seq"] * scale) +def kv_parts(kv, ctx): + """(full attention, sliding window, linear state) KV bytes of one sequence at context ctx.""" + return kv["full_per_token"] * ctx, kv["swa_per_token"] * min(ctx, kv["window"] or ctx), kv["state_per_seq"] + + +# ---- llama.cpp over several GPUs ---------------------------------------------------------- +def llama_layer_devices(ts, n_layer): + """Device of each repeating layer and of the output layer (the last entry), as llama.cpp + b11749 assigns them with -ngl past the layer count (src/llama-model.cpp load_tensors): + upper_bound(cumsum(ts) / sum(ts), il / (n_layer + 1)). Integer layer counts summing to + n_layer + 1 give each device exactly its count, the output layer on the last one.""" + total, cum, splits = float(sum(ts)), 0.0, [] + for t in ts: + cum += t + splits.append(cum / total) + return [bisect.bisect_right(splits, il / (n_layer + 1)) for il in range(n_layer + 1)] + + +def llama_layer_bytes(m, kv, ctx, seqs, n_cpu_moe): + """([weights], [KV]) on the GPU per repeating layer, the output layer last: the class + totals spread evenly over the layers, the experts of the first n_cpu_moe layers in host + RAM (--n-cpu-moe), and each layer's own KV (its device holds it) for seqs sequences. + The output layer holds lm_head, and the MTP and vision tensors when the GGUF has them.""" + n = max(m["layers"], 1) + exp, other = m["bytes"]["experts"] / n, m["bytes"]["other"] / n + f, w, st = kv_parts(kv, ctx) + per = {"full": f / max(kv["full_layers"], 1), "swa": w / max(kv["swa_layers"], 1), + "linear": st / max(kv["linear_layers"], 1)} + kinds = kv.get("kinds") or [] + wl = [other + (exp if il >= n_cpu_moe else 0) for il in range(n)] + kl = [per[kinds[il]] * seqs if il < len(kinds) else 0 for il in range(n)] + return wl + [m["bytes"]["lm_head"] + m["bytes"]["mtp"] + m["bytes"]["vision"]], kl + [0] + + +def llama_split(layer_bytes, k, room): + """-ts for k cards as integer layer counts (the last card's count includes the output + layer) that keep every card's layers within room bytes, its largest load as small as it + can be; None when no split does.""" + last = len(layer_bytes) - 1 + + def fill(cap): + ts, i = [], 0 + for _ in range(k - 1): + start, used = i, 0 + while i < last and used + layer_bytes[i] <= cap: + used += layer_bytes[i] + i += 1 + ts.append(i - start) + if sum(layer_bytes[i:]) > cap: + return None + return ts + [len(layer_bytes) - i] + + if fill(room) is None: + return None + lo, hi = max(layer_bytes), room + while hi - lo > MIB: + mid = (lo + hi) / 2 + lo, hi = (lo, mid) if fill(mid) else (mid, hi) + return fill(hi) + + +def llama_cards(ts, layer_bytes): + """Each card's bytes for ts: the layers llama_layer_devices puts on it.""" + out = [0] * len(ts) + for il, dev in enumerate(llama_layer_devices(ts, len(layer_bytes) - 1)): + out[dev] += layer_bytes[il] + return out + + +def _llama_search(m, kv, ladder, relaxed, want_seqs, k, room, n_cpus): + """(ctx, seqs, n_cpu_moe, ts) for llama.cpp on k cards of room bytes each: the most + context at which TARGET_SEQS (or want_seqs) sequences fit, then the fewest layers whose + experts leave the GPU (from n_cpus), then as many sequences as fit (capped at MAX_SEQS). + relaxed: at the ladder's last rung one sequence is enough. None when nothing fits.""" + for ctx in ladder: + need = want_seqs or (1 if relaxed and ctx == ladder[-1] else TARGET_SEQS) + for n_cpu in n_cpus: + def split(n): + wl, kl = llama_layer_bytes(m, kv, ctx, n, n_cpu) + return llama_split([a + b for a, b in zip(wl, kl)], k, room) + if not split(need): + continue + n = want_seqs + if not n: + lo, hi = need, MAX_SEQS + if split(hi): + lo = hi + while hi - lo > 1: + mid = (lo + hi) // 2 + lo, hi = (mid, hi) if split(mid) else (lo, mid) + n = lo + return ctx, n, n_cpu, split(n) + return None + + +def _llama_card_lines(use, ts, wl, kl, ov): + """One explain line per card of a llama.cpp split.""" + lines, first = [], 0 + for c, t, w, kvb in zip(use, ts, llama_cards(ts, wl), llama_cards(ts, kl)): + lines.append(" %s (-ts %d): layers %s, weights %s + KV %s + overhead %s = %s, budget %s" + % (c["card"], t, "%d-%d" % (first, first + t - 1) if t else "none", g1(w), g1(kvb), g1(ov), + g1(w + kvb + ov), g1(c["budget"]))) + first += t + lines.append(" (layer %d is the output layer: lm_head)" % (len(wl) - 1)) + return lines + + def ckpt_seq(kv, ctx): """Host bytes of one llama-server slot's context checkpoints at context ctx: kv["ckpts"] of them, each the slot's SWA cache (GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch), 256) @@ -1216,7 +1324,7 @@ def fit(host, cfg, weights, engine=None, ctx=None, seqs=None, gpus=None, kv_dtyp rung_kv.append(int(kb)) rung_limit.append("%dGi" % int(math.ceil(hb * (1 + LIMIT_SLACK) / GIB))) doc.update({k: picked[k] for k in ("engine", "placement", "ctx", "seqs", "tp", "gpus", "gpu_bytes", "host_bytes", - "flags", "dram_tok_s", "note", "kv_bytes", "ckpt_bytes")}) + "flags", "dram_tok_s", "note", "kv_bytes", "ckpt_bytes", "card_bytes")}) doc["source"] = weights.get("source") doc["weight_bytes"] = weights.get("total_bytes") or m["serve_bytes"] if m["format"] == "gguf": @@ -1262,7 +1370,7 @@ def _overhead(eng, m, seqs, override): def _res(eng, place, fits, why="", math_lines=(), **kw): r = {"engine": eng, "placement": place, "fits": fits, "why": why, "math": list(math_lines), "ctx": None, "seqs": 0, "tp": 1, "gpus": 0, "gpu_bytes": 0, "host_bytes": 0, "flags": [], "dram_tok_s": None, - "kv_bytes": 0, "kv_scale": 1.0, "ckpt_bytes": 0} + "kv_bytes": 0, "kv_scale": 1.0, "ckpt_bytes": 0, "card_bytes": []} r.update(kw) return r @@ -1323,6 +1431,9 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, yield _res(eng, place, False, "%d GPUs asked, %d not busy" % (k, len(cards)), gpus=k) continue use = cards[:k] + if eng == "llama.cpp" and k > 1: + yield _llama_multi(m, kv, ladder, relaxed, want_seqs, use, ov, ovl, [0], on_host, hfree, bw) + continue if eng == "vllm": # Every rank holds 1/k of the weights; KV heads split across ranks, or one # per rank (replicated) with fewer KV heads than ranks. @@ -1333,8 +1444,7 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, lines = ["per card: budget %s - weights %s / %d - overhead %s = %s for KV" % (g1(budget), g1(gpu_w_all), k, ovl, g1(max(left, 0)))] else: - # One card, or llama.cpp's layer split (-sm layer, the default): weights and - # KV spread over the cards in proportion to -ts; each card runs its own buffers. + # One card (llama.cpp on several is sized card by card in _llama_multi). budget = sum(c["budget"] for c in use) w = gpu_w_all left = budget - w - ov * k @@ -1366,66 +1476,19 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, gpu_use = int(w + ov + kvused) if eng == "vllm" else int((w + ov * k + kvused) / k) yield _res(eng, place, True, "", lines, ctx=c, seqs=n, tp=k if eng == "vllm" else 1, gpus=k, gpu_bytes=gpu_use, host_bytes=hb, kv_bytes=int(kvused), kv_scale=scale, ckpt_bytes=ck, + card_bytes=[gpu_use] * k, flags=_flags(eng, place, c, n, k, gpu_use, use, kvused, m, 0, kv)) # MoE experts to DRAM (llama.cpp --n-cpu-moe): every free card, fewest CPU layers. if eng == "llama.cpp" and cards and m["moe"]: - k = want_gpus or len(cards) - use = cards[:k] - budget = sum(c["budget"] for c in use) - exp_layer = m["bytes"]["experts"] / max(m["layers"], 1) - base = gpu_w_all - m["bytes"]["experts"] - lines = ["GPU (%s, budget %s) keeps %s non-expert weights + overhead %d x %s" - % ("+".join(c["card"] for c in use), g1(budget), g1(base), k, ovl), - "lookup tables %s in host RAM" % g1(on_host)] + use = cards[:want_gpus or len(cards)] if not (cpu["amx"] or cpu["avx512"]): - yield _res(eng, "moe-offload", False, "the CPU has neither AMX nor AVX-512 for expert compute", lines, gpus=k) + yield _res(eng, "moe-offload", False, "the CPU has neither AMX nor AVX-512 for expert compute", gpus=len(use)) elif not bw: yield _res(eng, "moe-offload", False, "DRAM bandwidth unknown (no STREAM run cached, SMBIOS unreadable)", - lines, gpus=k) + gpus=len(use)) else: - # Most context first (as everywhere), then the fewest layers whose experts leave the GPU. - best = None - for rung in ladder: - for n_cpu in range(1, m["layers"] + 1): - gw = base + exp_layer * (m["layers"] - n_cpu) - left = budget - gw - ov * k - if left <= 0: - continue - c, n = size_kv(kv, left, [rung], want_seqs, 1.0, relaxed and rung == ladder[-1]) - if c: - best = (n_cpu, gw, c, n) - break - if best: - break - if not best: - yield _res(eng, "moe-offload", False, "non-expert weights + KV do not fit %s of VRAM" % g1(budget), - lines, gpus=k) - else: - n_cpu, gw, c, n = best - dram_w = exp_layer * n_cpu - read = dram_w * m["experts_per_tok"] / m["experts"] - toks = _dram_rate(bw, read) - lines.append("experts of the first %d of %d layers in DRAM: %s; on GPU %s" - % (n_cpu, m["layers"], g1(dram_w), g1(gw))) - lines.append("DRAM decode bound: %g GB/s / (%s x %d/%d read per token) = %.0f tok/s (floor %g)" - % (bw, g1(dram_w), m["experts_per_tok"], m["experts"], toks, MIN_DRAM_TOK_S)) - kvused = per_seq(kv, c) * n - lines.append("KV: %d seqs x %s at %d tokens = %s on GPU" % (n, _mb(per_seq(kv, c)), c, g1(kvused))) - ck = ckpt_seq(kv, c) * n - hb = int(dram_w + on_host + HOST_RUNTIME[eng] + ck) - lines.append("host: experts %s + lookup tables %s + runtime %s + context checkpoints %d x %s = %s" - % (g1(dram_w), g1(on_host), g1(HOST_RUNTIME[eng]), n, _mb(ckpt_seq(kv, c)), g1(hb))) - if toks < MIN_DRAM_TOK_S: - yield _res(eng, "moe-offload", False, "DRAM bandwidth gives %.1f tok/s, under the %g floor" - % (toks, MIN_DRAM_TOK_S), lines, gpus=k) - elif hb > hfree: - yield _res(eng, "moe-offload", False, "DRAM side needs %s, host budget %s" % (g1(hb), g1(hfree)), - lines, gpus=k) - else: - gpu_use = int((gw + ov * k + kvused) / k) - yield _res(eng, "moe-offload", True, "", lines, ctx=c, seqs=n, gpus=k, gpu_bytes=gpu_use, - host_bytes=hb, dram_tok_s=round(toks, 1), kv_bytes=int(kvused), ckpt_bytes=ck, - flags=_flags(eng, "moe-offload", c, n, k, gpu_use, use, kvused, m, n_cpu, kv)) + yield _llama_multi(m, kv, ladder, relaxed, want_seqs, use, ov, ovl, range(1, m["layers"] + 1), on_host, + hfree, bw) # CPU only: weights, KV and activations all in DRAM (vLLM: its CPU build, the KV # pool sized by VLLM_CPU_KVCACHE_SPACE, as EAT's vLLM chart sets it). if cpu_ok: @@ -1463,7 +1526,54 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, flags=_flags(eng, "cpu", c, n, 1, 0, [], kvused, m, 0, kv)) -def _flags(eng, place, ctx, seqs, k, gpu_use, cards, kvused, m, n_cpu_moe, kv=None): +def _llama_multi(m, kv, ladder, relaxed, want_seqs, use, ov, ovl, n_cpus, on_host, hfree, bw): + """llama.cpp on the cards in use, each holding its own layers, their KV and its own + overhead within the smallest budget among them (the device plugin, not the fit, orders + the cards inside the pod): a layer split (n_cpus [0]) or experts in DRAM (n_cpus 1..).""" + k = len(use) + place = "gpu-split" if list(n_cpus) == [0] else "moe-offload" + budget = min(c["budget"] for c in use) + room = budget - ov + lines = ["%s: layer il on card upper_bound(cumsum(-ts) / sum, il / (layers + 1)) (src/llama-model.cpp); " + "each card: budget %s (the smallest of them) - overhead %s = %s for its layers" + % ("+".join(c["card"] for c in use), g1(budget), ovl, g1(max(room, 0)))] + if on_host: + lines.append("lookup tables %s stay in host RAM" % g1(on_host)) + found = _llama_search(m, kv, ladder, relaxed, want_seqs, k, room, n_cpus) if room > 0 else None + if not found: + why = ("weights + KV for %d-token sequences do not fit %d card(s) of %s each, layer by layer" + % (ladder[-1], k, g1(budget))) + return _res("llama.cpp", place, False, why, lines, gpus=k) + c, n, n_cpu, ts = found + wl, kl = llama_layer_bytes(m, kv, c, n, n_cpu) + cards_b = [int(b + ov) for b in llama_cards(ts, [a + b for a, b in zip(wl, kl)])] + lines += _llama_card_lines(use, ts, wl, kl, ov) + kvused = per_seq(kv, c) * n + lines.append("KV: %d seqs x %s at %d tokens = %s over the cards" % (n, _mb(per_seq(kv, c)), c, g1(kvused))) + hb = int(HOST_RUNTIME["llama.cpp"] + on_host) + toks = None + if n_cpu: + exp_layer = m["bytes"]["experts"] / max(m["layers"], 1) + dram_w = exp_layer * n_cpu + toks = _dram_rate(bw, dram_w * m["experts_per_tok"] / m["experts"]) + lines.append("experts of the first %d of %d layers in DRAM: %s" % (n_cpu, m["layers"], g1(dram_w))) + lines.append("DRAM decode bound: %g GB/s / (%s x %d/%d read per token) = %.0f tok/s (floor %g)" + % (bw, g1(dram_w), m["experts_per_tok"], m["experts"], toks, MIN_DRAM_TOK_S)) + hb += int(dram_w) + lines.append("host: experts %s + lookup tables %s + runtime %s = %s" + % (g1(dram_w), g1(on_host), g1(HOST_RUNTIME["llama.cpp"]), g1(hb))) + if toks < MIN_DRAM_TOK_S: + return _res("llama.cpp", place, False, "DRAM bandwidth gives %.1f tok/s, under the %g floor" + % (toks, MIN_DRAM_TOK_S), lines, gpus=k) + if hb > hfree: + return _res("llama.cpp", place, False, "host side %s exceeds the host budget %s" % (g1(hb), g1(hfree)), + lines, gpus=k) + return _res("llama.cpp", place, True, "", lines, ctx=c, seqs=n, gpus=k, gpu_bytes=max(cards_b), card_bytes=cards_b, + host_bytes=hb, dram_tok_s=round(toks, 1) if toks else None, kv_bytes=int(kvused), + flags=_flags("llama.cpp", place, c, n, k, max(cards_b), use, kvused, m, n_cpu, kv, ts)) + + +def _flags(eng, place, ctx, seqs, k, gpu_use, cards, kvused, m, n_cpu_moe, kv=None, ts=None): """Engine memory flags from the fit. Spellings from each engine's own argument parser: vLLM v0.32.0 vllm/engine/arg_utils.py, OVMS v2026.4.1 src/graph_export/graph_cli_parser.cpp, llama.cpp 1242be6 common/arg.cpp.""" @@ -1485,8 +1595,8 @@ def _flags(eng, place, ctx, seqs, k, gpu_use, cards, kvused, m, n_cpu_moe, kv=No # -c is the total context; with -np set explicitly each slot gets c/np (kv-unified is # on only when the slot count is auto). out = ["-c", str(ctx * seqs), "-np", str(seqs), "-ngl", "0" if place == "cpu" else "999"] - if k > 1: - out += ["-ts", ",".join(str(max(int(c["budget"] // GIB), 1)) for c in cards)] + if ts and k > 1: + out += ["-ts", ",".join(str(t) for t in ts)] # layers per card (llama_layer_devices) if place == "moe-offload": out += ["--cpu-moe"] if n_cpu_moe >= m["layers"] else ["--n-cpu-moe", str(n_cpu_moe)] # -cram/--cache-ram, -ctxcp/--ctx-checkpoints: the host caches the fit counts (LLAMA_CTX_CHECKPOINTS) diff --git a/docs/fit.md b/docs/fit.md index 321772a2..ff83426b 100644 --- a/docs/fit.md +++ b/docs/fit.md @@ -68,7 +68,7 @@ in this order, and the first that fits wins: 1. one GPU; 2. several GPUs: vLLM tensor parallel (heads divisible by the size), or llama.cpp's - layer split (OVMS serves one GPU); + layer split, sized card by card (below); OVMS serves one GPU; 3. MoE experts in DRAM (llama.cpp `--n-cpu-moe N`): only when the GPU placements do not fit, the CPU has AMX or AVX-512, and the DRAM side's bandwidth bound per token stays above a floor. The fit takes the fewest layers whose experts leave the GPU; @@ -122,6 +122,21 @@ prompt's end, which is what the next turn restores from), and counts slots x 4 x checkpoint in the host bytes and the pod limit. A model with no sliding-window and no linear-attention layers makes none. +On several GPUs llama.cpp places whole layers, so the fit sizes each card from the +layers it gets, the way llama.cpp b11749 assigns them with `-ngl 999` +(`src/llama-model.cpp`, `load_tensors`). Layer `il` goes to the device +`upper_bound(cumsum(ts) / sum(ts), il / (layers + 1))`, and the output layer +(`il = layers`, holding `lm_head`) goes the same way, which is the last card. Each +layer's weights are its share of the class totals, minus its experts when it is one of +the first `--n-cpu-moe` layers. Each layer's KV and recurrent state sit on its card. Every +card also carries its own overhead. +The fit emits `-ts` as whole layer counts summing to layers + 1, so each card gets exactly +the layers it was sized for. It keeps every card within the smallest budget of the cards +used, because the device plugin, not the fit, decides which card is device 0 inside the +pod, and it balances the largest card's load. When no split fits, the placement is refused. +`--explain` lists each card's layers, weights, KV and total, and `card_bytes` in `--json` +has the same numbers. + The overhead is an estimate from the formula, not a measurement; `K3D_GPU_FIT_OVERHEAD_GIB` replaces it with a measured value. @@ -131,7 +146,7 @@ The overhead is an estimate from the formula, not a measurement; | --- | --- | | vLLM v0.32.0 | `--tensor-parallel-size`, `--max-model-len`, `--max-num-seqs`, `--gpu-memory-utilization` (vLLM still profiles activations inside that share), `--kv-cache-dtype fp8` when the checkpoint's `kv_cache_scheme` made the fit count one byte. CPU build (EAT): `VLLM_CPU_KVCACHE_SPACE` | | OVMS 2026.4.1 | `--max_num_seqs`, `--cache_size` (a static pool in whole GiB, rounded up to hold every sequence: the dynamic one grows past the fit) | -| llama.cpp v0.8.0 | `-c` (context x slots), `-np`, `-ngl`, `-ts`, `--n-cpu-moe` or `--cpu-moe`, `--cache-ram 0`, `-ctxcp 4`, `-ctk`/`-ctv` | +| llama.cpp v0.8.0 | `-c` (context x slots), `-np`, `-ngl`, `-ts` (layers per card), `--n-cpu-moe` or `--cpu-moe`, `--cache-ram 0`, `-ctxcp 4`, `-ctk`/`-ctv` | `agent up` applies them, sets the pod's memory request and limit to the fit, and annotates the pod `k3d-gpu.io/agent-fit`. `K3D_GPU_AGENT_MAX_LEN`, `K3D_GPU_AGENT_MAX_NUM_SEQS` diff --git a/tests/fit_test.py b/tests/fit_test.py index 7dd9b3e6..3b4b9702 100755 --- a/tests/fit_test.py +++ b/tests/fit_test.py @@ -12,6 +12,7 @@ class bytes from the real safetensors/GGUF headers at a pinned revision), Usage: python3 tests/fit_test.py """ +import bisect import gzip import json import os @@ -246,6 +247,11 @@ def config(model): return json.load(f) +def weights(model): + with open(os.path.join(mdir(model), "weights.json"), encoding="utf-8") as f: + return json.load(f) + + class KV(unittest.TestCase): """KV bytes from config.json, worked out by hand from each config's fields.""" @@ -813,5 +819,100 @@ def test_falls_to_fewer_bits_when_int4_does_not_fit(self): self.assertTrue(any("IQ4_NL" in s for s in d["explain"]) and any("Q2_K" in s for s in d["explain"])) +def llama_cards(d, cfg, w, kv_layer): + """Each card's GPU bytes for a llama.cpp plan, worked out here from its flags the way + llama.cpp b11749 assigns them (src/llama-model.cpp load_tensors, -ngl past the layer + count): layer il on device upper_bound(cumsum(ts) / sum(ts), il / (n_layer + 1)), the + output layer (il = n_layer) the same way, the experts of the first --n-cpu-moe layers in + host RAM, each layer's KV on its device, and each card's own overhead. Weights per layer + are the class totals over the layers. kv_layer(il, ctx) is one sequence's KV of layer il.""" + tc = cfg.get("text_config") or cfg + n = tc["num_hidden_layers"] + f = d["flags"] + ts = [int(x) for x in f[f.index("-ts") + 1].split(",")] + ncpu = int(f[f.index("--n-cpu-moe") + 1]) if "--n-cpu-moe" in f else n if "--cpu-moe" in f else 0 + cls = {k: v["bytes"] for k, v in w["classes"].items()} + cum, c = [], 0 + for t in ts: + c += t + cum.append(c / sum(ts)) + cards = [0] * len(ts) + for il in range(n + 1): + dev = bisect.bisect_right(cum, il / (n + 1)) + if il == n: + cards[dev] += cls.get("lm_head", 0) + cls.get("mtp", 0) + cls.get("vision", 0) + else: + cards[dev] += (cls.get("other", 0) / n + (cls.get("experts", 0) / n if il >= ncpu else 0) + + kv_layer(il, d["ctx"]) * d["seqs"]) + m = fit.model_profile(cfg, w) + ov = fit._overhead("llama.cpp", m, fit.TARGET_SEQS, None)[0] + return ts, [int(b + ov) for b in cards] + + +def flash_next_kv(il, ctx): + # every 4th layer full attention: 2 x 2 kv heads x 256 x 2 B + indexer 128 x 2 B per token; + # the rest gated DeltaNet: 48 x 128 x 128 x 4 B state + 3 x 10240 conv ch x 2 B per sequence + return (2 * 2 * 256 * 2 + 128 * 2) * ctx if il % 4 == 3 else 48 * 128 * 128 * 4 + 3 * 10240 * 2 + + +def gemma4_kv(il, ctx): + # every 6th layer global: 2 x 2 kv heads x 512 x 2 B per token; the rest a 1024-token window + return 2 * 2 * 512 * 2 * ctx if il % 6 == 5 else 2 * 8 * 256 * 2 * min(ctx, 1024) + + +class LlamaSplit(unittest.TestCase): + """llama.cpp over several GPUs: each card's own layers, KV and overhead within its budget.""" + + def budget(self, k): + return min(c["budget"] for c in fit.gpu_budget(host("xeon6741p-4xb70"))[0][:k]) + + def test_layer_devices(self): + # upper_bound(cumsum(ts)/sum, il/(n_layer+1)): -ts 30,30 over 48 layers puts 25 on the first card + self.assertEqual(fit.llama_layer_devices([30, 30], 48), [0] * 25 + [1] * 24) + # integer layer counts that sum to n_layer + 1 give exactly those counts, the output last + self.assertEqual(fit.llama_layer_devices([28, 21], 48), [0] * 28 + [1] * 21) + self.assertEqual(fit.llama_layer_devices([10, 0, 21], 30), [0] * 10 + [2] * 21) + + def test_old_split_overflows_the_second_card(self): + # The plan this replaces: -ts 30,30 --n-cpu-moe 28 at 262144 x 4 puts ~43.5 GiB on a 32 GiB card. + cfg = config(FLASH_GGUF) + w = weights(FLASH_GGUF)["groups"]["IQ4_NL"] + old = {"flags": ["-ts", "30,30", "--n-cpu-moe", "28"], "ctx": 262144, "seqs": 4} + _, cards = llama_cards(old, cfg, w, flash_next_kv) + self.assertGreater(cards[1], 32 * GIB) + + def check(self, d, cfg, w, kv_layer): + n = (cfg.get("text_config") or cfg)["num_hidden_layers"] + ts, cards = llama_cards(d, cfg, w, kv_layer) + self.assertEqual(sum(ts), n + 1, ts) + self.assertEqual(len(ts), d["gpus"]) + budget = self.budget(d["gpus"]) + for i, b in enumerate(cards): + self.assertLessEqual(b, budget, (i, ts, d["flags"])) + # the fit's own per-card bytes agree with the ones worked out here + self.assertEqual(len(d["card_bytes"]), len(ts)) + for got, want in zip(d["card_bytes"], cards): + self.assertAlmostEqual(got / GIB, want / GIB, delta=0.01) + self.assertEqual(d["gpu_bytes"], max(d["card_bytes"])) + + def test_flash_next_iq4nl_cards_within_budget(self): + d = plan("xeon6741p-4xb70", FLASH_GGUF) + self.assertTrue(d["fits"], d["summary"]) + self.assertEqual((d["placement"], d["gpus"]), ("moe-offload", 2)) + w = weights(FLASH_GGUF)["groups"]["IQ4_NL"] + self.check(d, config(FLASH_GGUF), w, flash_next_kv) + + def test_gemma_layer_split_cards_within_budget(self): + g = "google__gemma-4-26B-A4B-it-qat-q4_0-gguf" + d = plan("xeon6741p-4xb70", g, ctx=262144, seqs=6) + self.assertTrue(d["fits"], d["summary"]) + self.assertEqual((d["placement"], d["gpus"]), ("gpu-split", 2)) + self.check(d, config(g), weights(g), gemma4_kv) + + def test_too_much_for_any_split_is_refused(self): + d = plan("xeon6741p-4xb70", "google__gemma-4-26B-A4B-it-qat-q4_0-gguf", ctx=262144, seqs=12, device="gpu") + self.assertFalse(d["fits"]) + + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/tests/fit_test.sh b/tests/fit_test.sh index 69c2ec32..85570b2d 100755 --- a/tests/fit_test.sh +++ b/tests/fit_test.sh @@ -109,6 +109,25 @@ out=$(bash "${K3D_GPU}" fit --host "${XEON}" --model "${FLASH}" 2>&1); rc=$? if [ "${rc}" -eq 0 ] && grep -q "^fit: ${FLASH}:IQ4_NL on 2 GPU(s) + experts and lookup tables in DRAM" <<< "${out}"; then ok "fit: a multi-GGUF repo needs no flag (IQ4_NL picked, split shards summed)" else fail "fit: multi-GGUF repo auto-pick (rc ${rc})" "${out}"; fi +# Its two-card plan sizes each card the way llama.cpp assigns layers: -ts is integer layer +# counts (48 layers + the output layer), and no card's own bytes pass its budget. The plan +# this replaces, -ts 30,30 --n-cpu-moe 28, put 43.5 GiB on the second 32 GiB card. +out=$(bash "${K3D_GPU}" fit --host "${XEON}" --model "${FLASH}" --json 2>&1); rc=$? +if [ "${rc}" -eq 0 ] && ts=$(python3 -c ' +import json, sys +sys.path.insert(0, sys.argv[1]) +import fit +d = json.loads(sys.argv[2]) +f = d["flags"] +ts = [int(x) for x in f[f.index("-ts") + 1].split(",")] +budget = min(c["budget"] for c in fit.gpu_budget(json.load(open(sys.argv[3])))[0][:d["gpus"]]) +assert d["placement"] == "moe-offload" and len(ts) == d["gpus"] == 2 and sum(ts) == 49, (ts, d["gpus"]) +assert ts != [30, 30] and "--n-cpu-moe" in f, f +assert len(d["card_bytes"]) == 2 and max(d["card_bytes"]) <= budget, (d["card_bytes"], budget) +print(f[f.index("-ts") + 1]) +' "${ROOT}/console" "${out}" "${XEON}"); then + ok "fit: Flash-Next IQ4_NL on two cards, -ts ${ts} layers, every card within its budget" +else fail "fit: Flash-Next IQ4_NL per-card split (rc ${rc})" "$(grep -E '"-ts"|card_bytes' -A3 <<< "${out}")"; fi out=$(bash "${K3D_GPU}" fit --host "${XEON}" --model "${FLASH}" --file Qwen3.8-Flash-Next-Q8_0-00002-of-00002.gguf --explain 2>&1); rc=$? if [ "${rc}" -eq 0 ] && grep -q ':Q8_0 on' <<< "${out}" && grep -q -- "-hf ${FLASH}:Q8_0" <<< "${out}"; then ok "fit --file: a shard name picks its whole split model" From d666d07c9d84136d871d0300d34b877096d4ea6e Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Wed, 28 Oct 2026 23:28:37 -0700 Subject: [PATCH 05/13] fit: llama.cpp per-card bytes follow the bytes to serve; moe-offload picks its card count The per-layer weights counted the MTP head for every format, so a model sized from safetensors for a GGUF conversion carried 4.9 GiB (Qwen3.8-Flash-Next) on its output layer that model_profile leaves out of the bytes to serve. One-card moe-offload plans therefore no longer matched the old arithmetic: Intel/Qwen3.8-Flash-Next-W4A16-AutoRound with --gpus 1 on the fixture Xeon went from --n-cpu-moe 43 to 47. The output layer now holds the bytes to serve less the lookup tables, experts, other and vision: lm_head, plus MTP only when the GGUF holds it. The one-card plan is --n-cpu-moe 43 again. The vision projector was put on the output layer, the last card. llama.cpp loads the projector through tools/mtmd/clip.cpp, which takes the first GPU backend, so it now counts with layer 0 on the first card, and --explain says so. moe-offload always used every free card. Each card is held to the smallest budget of the cards used, so a smaller card beside two large ones lowers the room on all of them. On the fixture Xeon with 0, 0, 12 and 14 GiB in use on its cards (budgets 30.4, 30.4, 18.4 and 16.4 GiB), Intel/Qwen3.8-Flash-Next-W4A16-AutoRound at 262144 x 6 with --device gpu was refused ("do not fit 4 card(s) of 16.4 GiB each") while --gpus 2 fit. The fit now tries 1, 2, ... of the free cards by budget (or the --gpus count) and keeps the plan with the most context, then the fewest CPU-expert layers, then the most sequences, then the fewest cards; that case is now two cards, -ts 31,18 --n-cpu-moe 41. --gpus above the free card count is refused, as the GPU placements already refuse it, instead of running on fewer cards. The fixture Xeon's GGUF plan is unchanged: -ts 35,14 --n-cpu-moe 29 at 262144 x 4. Tests (each fails on the previous commit): - llama_layer_bytes adds up to the bytes to serve less the lookup tables, its output layer is lm_head alone and layer 0 carries the vision projector, for Qwen3.8-Flash-Next from safetensors. - On the mixed-budget Xeon, the automatic moe-offload plan fits, has no more CPU-expert layers than any --gpus 1..4 plan, uses fewer than four cards and keeps every card in budget. - --gpus 3 on the fixture Xeon, two cards free, is refused for moe-offload. - The test's own per-card arithmetic follows the same vision and MTP rule. --- console/fit.py | 45 ++++++++++++++++++++++++++++++---------- docs/fit.md | 13 +++++++----- tests/fit_test.py | 52 +++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 92 insertions(+), 18 deletions(-) diff --git a/console/fit.py b/console/fit.py index b71104a5..b43ab411 100644 --- a/console/fit.py +++ b/console/fit.py @@ -1080,16 +1080,21 @@ def llama_layer_bytes(m, kv, ctx, seqs, n_cpu_moe): """([weights], [KV]) on the GPU per repeating layer, the output layer last: the class totals spread evenly over the layers, the experts of the first n_cpu_moe layers in host RAM (--n-cpu-moe), and each layer's own KV (its device holds it) for seqs sequences. - The output layer holds lm_head, and the MTP and vision tensors when the GGUF has them.""" + The weights add up to the bytes to serve less the lookup tables (MTP only when the GGUF + holds it). The output layer holds lm_head and the MTP tensors; the vision projector + counts with layer 0, on the first card: tools/mtmd/clip.cpp loads it on the first GPU.""" n = max(m["layers"], 1) - exp, other = m["bytes"]["experts"] / n, m["bytes"]["other"] / n + b = m["bytes"] + exp, other = b["experts"] / n, b["other"] / n f, w, st = kv_parts(kv, ctx) per = {"full": f / max(kv["full_layers"], 1), "swa": w / max(kv["swa_layers"], 1), "linear": st / max(kv["linear_layers"], 1)} kinds = kv.get("kinds") or [] wl = [other + (exp if il >= n_cpu_moe else 0) for il in range(n)] + wl[0] += b["vision"] kl = [per[kinds[il]] * seqs if il < len(kinds) else 0 for il in range(n)] - return wl + [m["bytes"]["lm_head"] + m["bytes"]["mtp"] + m["bytes"]["vision"]], kl + [0] + out = m["serve_bytes"] - m["lookup_bytes"] - b["experts"] - b["other"] - b["vision"] + return wl + [out], kl + [0] def llama_split(layer_bytes, k, room): @@ -1478,17 +1483,32 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, gpu_bytes=gpu_use, host_bytes=hb, kv_bytes=int(kvused), kv_scale=scale, ckpt_bytes=ck, card_bytes=[gpu_use] * k, flags=_flags(eng, place, c, n, k, gpu_use, use, kvused, m, 0, kv)) - # MoE experts to DRAM (llama.cpp --n-cpu-moe): every free card, fewest CPU layers. + # MoE experts to DRAM (llama.cpp --n-cpu-moe). Every card is held to the smallest budget + # of the cards used, so one more card can leave less room per card: the free cards by + # budget, 1, 2, ... of them (or --gpus), and the plan with the most context, then the + # fewest CPU-expert layers, then the most sequences, then the fewest cards. if eng == "llama.cpp" and cards and m["moe"]: - use = cards[:want_gpus or len(cards)] - if not (cpu["amx"] or cpu["avx512"]): - yield _res(eng, "moe-offload", False, "the CPU has neither AMX nor AVX-512 for expert compute", gpus=len(use)) + ks = [want_gpus] if want_gpus else list(range(1, len(cards) + 1)) + if want_gpus and want_gpus > len(cards): + yield _res(eng, "moe-offload", False, "%d GPUs asked, %d not busy" % (want_gpus, len(cards)), gpus=want_gpus) + elif not (cpu["amx"] or cpu["avx512"]): + yield _res(eng, "moe-offload", False, "the CPU has neither AMX nor AVX-512 for expert compute", gpus=ks[-1]) elif not bw: yield _res(eng, "moe-offload", False, "DRAM bandwidth unknown (no STREAM run cached, SMBIOS unreadable)", - gpus=len(use)) + gpus=ks[-1]) else: - yield _llama_multi(m, kv, ladder, relaxed, want_seqs, use, ov, ovl, range(1, m["layers"] + 1), on_host, - hfree, bw) + tries = [_llama_multi(m, kv, ladder, relaxed, want_seqs, cards[:k], ov, ovl, range(1, m["layers"] + 1), + on_host, hfree, bw) for k in ks] + ok = [r for r in tries if r["fits"]] + if not ok: + yield tries[-1] + else: + best = max(ok, key=lambda r: (r["ctx"], -r["n_cpu_moe"], r["seqs"], -r["gpus"])) + if len(tries) > 1: + best["math"].append("cards tried, by budget: " + "; ".join( + "%d: %d x %d, experts of %d layers in DRAM" % (r["gpus"], r["ctx"], r["seqs"], r["n_cpu_moe"]) + if r["fits"] else "%d: no" % r["gpus"] for r in tries)) + yield best # CPU only: weights, KV and activations all in DRAM (vLLM: its CPU build, the KV # pool sized by VLLM_CPU_KVCACHE_SPACE, as EAT's vLLM chart sets it). if cpu_ok: @@ -1548,6 +1568,9 @@ def _llama_multi(m, kv, ladder, relaxed, want_seqs, use, ov, ovl, n_cpus, on_hos wl, kl = llama_layer_bytes(m, kv, c, n, n_cpu) cards_b = [int(b + ov) for b in llama_cards(ts, [a + b for a, b in zip(wl, kl)])] lines += _llama_card_lines(use, ts, wl, kl, ov) + if m["bytes"]["vision"]: + lines.append(" (%s holds the vision projector, %s: tools/mtmd/clip.cpp loads it on the first GPU)" + % (use[0]["card"], g1(m["bytes"]["vision"]))) kvused = per_seq(kv, c) * n lines.append("KV: %d seqs x %s at %d tokens = %s over the cards" % (n, _mb(per_seq(kv, c)), c, g1(kvused))) hb = int(HOST_RUNTIME["llama.cpp"] + on_host) @@ -1569,7 +1592,7 @@ def _llama_multi(m, kv, ladder, relaxed, want_seqs, use, ov, ovl, n_cpus, on_hos return _res("llama.cpp", place, False, "host side %s exceeds the host budget %s" % (g1(hb), g1(hfree)), lines, gpus=k) return _res("llama.cpp", place, True, "", lines, ctx=c, seqs=n, gpus=k, gpu_bytes=max(cards_b), card_bytes=cards_b, - host_bytes=hb, dram_tok_s=round(toks, 1) if toks else None, kv_bytes=int(kvused), + host_bytes=hb, dram_tok_s=round(toks, 1) if toks else None, kv_bytes=int(kvused), n_cpu_moe=n_cpu, flags=_flags("llama.cpp", place, c, n, k, max(cards_b), use, kvused, m, n_cpu, kv, ts)) diff --git a/docs/fit.md b/docs/fit.md index ff83426b..91d47ad4 100644 --- a/docs/fit.md +++ b/docs/fit.md @@ -71,7 +71,9 @@ in this order, and the first that fits wins: layer split, sized card by card (below); OVMS serves one GPU; 3. MoE experts in DRAM (llama.cpp `--n-cpu-moe N`): only when the GPU placements do not fit, the CPU has AMX or AVX-512, and the DRAM side's bandwidth bound per token stays - above a floor. The fit takes the fewest layers whose experts leave the GPU; + above a floor. The fit takes the fewest layers whose experts leave the GPU, trying + 1, 2, ... of the free cards (largest budget first) and keeping the plan with the most + context, then the fewest such layers, then the most sequences, then the fewest cards; 4. CPU only. Per placement: @@ -126,10 +128,11 @@ On several GPUs llama.cpp places whole layers, so the fit sizes each card from t layers it gets, the way llama.cpp b11749 assigns them with `-ngl 999` (`src/llama-model.cpp`, `load_tensors`). Layer `il` goes to the device `upper_bound(cumsum(ts) / sum(ts), il / (layers + 1))`, and the output layer -(`il = layers`, holding `lm_head`) goes the same way, which is the last card. Each -layer's weights are its share of the class totals, minus its experts when it is one of -the first `--n-cpu-moe` layers. Each layer's KV and recurrent state sit on its card. Every -card also carries its own overhead. +(`il = layers`, holding `lm_head` and a GGUF's MTP tensors) goes the same way, which is +the last card. Each layer's weights are its share of the class totals, minus its experts +when it is one of the first `--n-cpu-moe` layers. The vision projector counts on the first +card: `tools/mtmd/clip.cpp` loads it on the first GPU. Each layer's KV and recurrent state +sit on its card. Every card also carries its own overhead. The fit emits `-ts` as whole layer counts summing to layers + 1, so each card gets exactly the layers it was sized for. It keeps every card within the smallest budget of the cards used, because the device plugin, not the fit, decides which card is device 0 inside the diff --git a/tests/fit_test.py b/tests/fit_test.py index 3b4b9702..0cab9310 100755 --- a/tests/fit_test.py +++ b/tests/fit_test.py @@ -825,7 +825,9 @@ def llama_cards(d, cfg, w, kv_layer): count): layer il on device upper_bound(cumsum(ts) / sum(ts), il / (n_layer + 1)), the output layer (il = n_layer) the same way, the experts of the first --n-cpu-moe layers in host RAM, each layer's KV on its device, and each card's own overhead. Weights per layer - are the class totals over the layers. kv_layer(il, ctx) is one sequence's KV of layer il.""" + are the class totals over the layers; the output layer holds lm_head and the MTP tensors + (a GGUF's), and the vision projector is on the first GPU (tools/mtmd/clip.cpp). + kv_layer(il, ctx) is one sequence's KV of layer il.""" tc = cfg.get("text_config") or cfg n = tc["num_hidden_layers"] f = d["flags"] @@ -837,10 +839,11 @@ def llama_cards(d, cfg, w, kv_layer): c += t cum.append(c / sum(ts)) cards = [0] * len(ts) + cards[0] += cls.get("vision", 0) for il in range(n + 1): dev = bisect.bisect_right(cum, il / (n + 1)) if il == n: - cards[dev] += cls.get("lm_head", 0) + cls.get("mtp", 0) + cls.get("vision", 0) + cards[dev] += cls.get("lm_head", 0) + (cls.get("mtp", 0) if w["format"] == "gguf" else 0) else: cards[dev] += (cls.get("other", 0) / n + (cls.get("experts", 0) / n if il >= ncpu else 0) + kv_layer(il, d["ctx"]) * d["seqs"]) @@ -909,6 +912,51 @@ def test_gemma_layer_split_cards_within_budget(self): self.assertEqual((d["placement"], d["gpus"]), ("gpu-split", 2)) self.check(d, config(g), weights(g), gemma4_kv) + def test_layer_bytes_are_the_bytes_to_serve(self): + # Sized from safetensors for a GGUF conversion: the MTP head is not served + # (model_profile serve_bytes), and the vision projector loads on the first GPU + # (tools/mtmd/clip.cpp takes the first GPU backend), not on the output layer's card. + cfg = config("Qwen__Qwen3.8-Flash-Next") + m = fit.model_profile(cfg, weights("Qwen__Qwen3.8-Flash-Next")) + self.assertGreater(m["bytes"]["mtp"], GIB) + self.assertGreater(m["bytes"]["vision"], GIB / 2) + wl, _ = fit.llama_layer_bytes(m, fit.kv_profile(cfg, 2), 4096, 1, 0) + self.assertAlmostEqual(sum(wl), m["serve_bytes"] - m["lookup_bytes"], delta=1) + self.assertAlmostEqual(wl[-1], m["bytes"]["lm_head"], delta=1) + self.assertAlmostEqual(wl[0] - wl[1], m["bytes"]["vision"], delta=1) + + def test_moe_offload_takes_the_best_number_of_cards(self): + # Every card is held to the smallest budget of the cards used, so two smaller cards + # beside two large ones can leave less room than the two large ones alone. The fit + # tries 1, 2, ... of the free cards by budget and keeps the plan with the fewest + # CPU-expert layers, not every free card. + h = host("xeon6741p-4xb70") + for g, used in zip(h["gpus"], (0, 0, 12, 14)): + g["vram_used"], g["users"] = used * GIB, [] + model = mdir("Intel__Qwen3.8-Flash-Next-W4A16-AutoRound") + cfg = config("Intel__Qwen3.8-Flash-Next-W4A16-AutoRound") + n = (cfg.get("text_config") or cfg)["num_hidden_layers"] + + def ncpu(d): + f = d["flags"] + return int(f[f.index("--n-cpu-moe") + 1]) if "--n-cpu-moe" in f else n + + kw = dict(ctx=262144, seqs=6, device="gpu") + auto = fit.plan_with_variants(h, model, [], **kw) + self.assertTrue(auto["fits"], auto["summary"]) + self.assertEqual(auto["placement"], "moe-offload") + each = [fit.plan_with_variants(h, model, [], gpus=k, **kw) for k in range(1, 5)] + self.assertEqual(ncpu(auto), min(ncpu(d) for d in each if d["fits"])) + self.assertLess(auto["gpus"], 4) + budget = min(c["budget"] for c in fit.gpu_budget(h)[0][:auto["gpus"]]) + self.assertLessEqual(max(auto["card_bytes"]), budget) + + def test_moe_offload_takes_the_cards_asked_for(self): + # --gpus 3 on the Xeon, where two cards are free: refused, not two cards. + d = plan("xeon6741p-4xb70", FLASH_GGUF, gpus=3, device="gpu") + self.assertFalse(d["fits"]) + self.assertIn("3 GPUs asked, 2 not busy", d["reason"]) + def test_too_much_for_any_split_is_refused(self): d = plan("xeon6741p-4xb70", "google__gemma-4-26B-A4B-it-qat-q4_0-gguf", ctx=262144, seqs=12, device="gpu") self.assertFalse(d["fits"]) From 86d31c526892e2f3c613b4b455813b0be5544e55 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 02:20:06 -0700 Subject: [PATCH 06/13] fit: the per-card split counts llama.cpp's SWA cells and the context checkpoints With #214, #216 and #217 together, the multi-card llama.cpp path (_llama_multi, which now also runs every moe-offload plan) had two gaps. Its per-layer KV counted a sliding-window layer at the window (1024 tokens for Gemma 4) while llama.cpp b11749 allocates GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch), 256) = 1536 cells per slot, so card_bytes disagreed with kv_bytes. And it left the context checkpoints out of the host bytes and the pod limit, which #216 counts on one card: the Flash-Next IQ4_NL plan on the Xeon fixture reported ckpt_bytes 0 with -ctxcp 4 in its flags. kv_parts now takes the sliding-window tokens from swa_tokens, so per_seq, the per-layer KV and the checkpoints share one cell count, and _llama_multi adds slots x checkpoints to the host bytes. The Flash-Next plan's pod limit goes from 74Gi to 76Gi. Tests: the Gemma split's per-card bytes are worked out at 1536 SWA cells and the KV over the cards equals kv_bytes; a split plan counts its checkpoints in host bytes; the Flash-Next two-card shell check asserts the checkpoints are on the host. --- console/fit.py | 30 ++++++++++++++++-------------- docs/fit.md | 3 ++- tests/fit_test.py | 26 ++++++++++++++++++++++++-- tests/fit_test.sh | 5 ++++- 4 files changed, 46 insertions(+), 18 deletions(-) diff --git a/console/fit.py b/console/fit.py index b43ab411..026a5f4e 100644 --- a/console/fit.py +++ b/console/fit.py @@ -1053,14 +1053,14 @@ def swa_tokens(kv, ctx): return min(ctx, w) -def per_seq(kv, ctx, scale=1.0): - return int((kv["full_per_token"] * ctx + kv["swa_per_token"] * swa_tokens(kv, ctx)) * scale - + kv["state_per_seq"] * scale) - - def kv_parts(kv, ctx): """(full attention, sliding window, linear state) KV bytes of one sequence at context ctx.""" - return kv["full_per_token"] * ctx, kv["swa_per_token"] * min(ctx, kv["window"] or ctx), kv["state_per_seq"] + return kv["full_per_token"] * ctx, kv["swa_per_token"] * swa_tokens(kv, ctx), kv["state_per_seq"] + + +def per_seq(kv, ctx, scale=1.0): + f, w, st = kv_parts(kv, ctx) + return int((f + w) * scale + st * scale) # ---- llama.cpp over several GPUs ---------------------------------------------------------- @@ -1174,9 +1174,7 @@ def ckpt_seq(kv, ctx): """Host bytes of one llama-server slot's context checkpoints at context ctx: kv["ckpts"] of them, each the slot's SWA cache (GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch), 256) cells, src/llama-kv-cache-iswa.cpp) and recurrent state. A model with neither makes none.""" - w = kv["window"] or ctx - one = kv["swa_per_token"] * min(_pad(ctx, 256), _pad(w + LLAMA_UBATCH, 256)) + kv["state_per_seq"] - return kv.get("ckpts", 0) * one + return kv.get("ckpts", 0) * (kv["swa_per_token"] * swa_tokens(kv, ctx) + kv["state_per_seq"]) def ctx_ladder(max_ctx, want=None): @@ -1573,7 +1571,9 @@ def _llama_multi(m, kv, ladder, relaxed, want_seqs, use, ov, ovl, n_cpus, on_hos % (use[0]["card"], g1(m["bytes"]["vision"]))) kvused = per_seq(kv, c) * n lines.append("KV: %d seqs x %s at %d tokens = %s over the cards" % (n, _mb(per_seq(kv, c)), c, g1(kvused))) - hb = int(HOST_RUNTIME["llama.cpp"] + on_host) + # llama-server's context checkpoints stay in host RAM whichever card holds the layers. + ck = ckpt_seq(kv, c) * n + hb = int(HOST_RUNTIME["llama.cpp"] + on_host + ck) toks = None if n_cpu: exp_layer = m["bytes"]["experts"] / max(m["layers"], 1) @@ -1583,17 +1583,19 @@ def _llama_multi(m, kv, ladder, relaxed, want_seqs, use, ov, ovl, n_cpus, on_hos lines.append("DRAM decode bound: %g GB/s / (%s x %d/%d read per token) = %.0f tok/s (floor %g)" % (bw, g1(dram_w), m["experts_per_tok"], m["experts"], toks, MIN_DRAM_TOK_S)) hb += int(dram_w) - lines.append("host: experts %s + lookup tables %s + runtime %s = %s" - % (g1(dram_w), g1(on_host), g1(HOST_RUNTIME["llama.cpp"]), g1(hb))) + lines.append("host: experts %s + lookup tables %s + runtime %s + context checkpoints %d x %s = %s" + % (g1(dram_w), g1(on_host), g1(HOST_RUNTIME["llama.cpp"]), n, _mb(ckpt_seq(kv, c)), g1(hb))) if toks < MIN_DRAM_TOK_S: return _res("llama.cpp", place, False, "DRAM bandwidth gives %.1f tok/s, under the %g floor" % (toks, MIN_DRAM_TOK_S), lines, gpus=k) + elif ck: + lines.append("context checkpoints: %d seqs x %s = %s in host RAM" % (n, _mb(ckpt_seq(kv, c)), g1(ck))) if hb > hfree: return _res("llama.cpp", place, False, "host side %s exceeds the host budget %s" % (g1(hb), g1(hfree)), lines, gpus=k) return _res("llama.cpp", place, True, "", lines, ctx=c, seqs=n, gpus=k, gpu_bytes=max(cards_b), card_bytes=cards_b, - host_bytes=hb, dram_tok_s=round(toks, 1) if toks else None, kv_bytes=int(kvused), n_cpu_moe=n_cpu, - flags=_flags("llama.cpp", place, c, n, k, max(cards_b), use, kvused, m, n_cpu, kv, ts)) + host_bytes=hb, dram_tok_s=round(toks, 1) if toks else None, kv_bytes=int(kvused), ckpt_bytes=ck, + n_cpu_moe=n_cpu, flags=_flags("llama.cpp", place, c, n, k, max(cards_b), use, kvused, m, n_cpu, kv, ts)) def _flags(eng, place, ctx, seqs, k, gpu_use, cards, kvused, m, n_cpu_moe, kv=None, ts=None): diff --git a/docs/fit.md b/docs/fit.md index 91d47ad4..22ca5ff3 100644 --- a/docs/fit.md +++ b/docs/fit.md @@ -132,7 +132,8 @@ layers it gets, the way llama.cpp b11749 assigns them with `-ngl 999` the last card. Each layer's weights are its share of the class totals, minus its experts when it is one of the first `--n-cpu-moe` layers. The vision projector counts on the first card: `tools/mtmd/clip.cpp` loads it on the first GPU. Each layer's KV and recurrent state -sit on its card. Every card also carries its own overhead. +sit on its card, a sliding-window layer's at its padded cell count (above); the context +checkpoints stay in host RAM. Every card also carries its own overhead. The fit emits `-ts` as whole layer counts summing to layers + 1, so each card gets exactly the layers it was sized for. It keeps every card within the smallest budget of the cards used, because the device plugin, not the fit, decides which card is device 0 inside the diff --git a/tests/fit_test.py b/tests/fit_test.py index 0cab9310..01ab28f8 100755 --- a/tests/fit_test.py +++ b/tests/fit_test.py @@ -859,8 +859,12 @@ def flash_next_kv(il, ctx): def gemma4_kv(il, ctx): - # every 6th layer global: 2 x 2 kv heads x 512 x 2 B per token; the rest a 1024-token window - return 2 * 2 * 512 * 2 * ctx if il % 6 == 5 else 2 * 8 * 256 * 2 * min(ctx, 1024) + # every 6th layer global: 2 x 2 kv heads x 512 x 2 B per token; the rest a 1024-token window, + # whose llama.cpp b11749 cache is GGML_PAD(min(n_ctx_seq, 1024 + 512 ubatch), 256) cells per + # slot (src/llama-kv-cache-iswa.cpp), n_ctx_seq itself padded to 256 (src/llama-context.cpp) + if il % 6 == 5: + return 2 * 2 * 512 * 2 * ctx + return 2 * 8 * 256 * 2 * min((ctx + 255) // 256 * 256, 1536) class LlamaSplit(unittest.TestCase): @@ -897,6 +901,9 @@ def check(self, d, cfg, w, kv_layer): for got, want in zip(d["card_bytes"], cards): self.assertAlmostEqual(got / GIB, want / GIB, delta=0.01) self.assertEqual(d["gpu_bytes"], max(d["card_bytes"])) + # the KV over the cards is the plan's KV + self.assertAlmostEqual(sum(kv_layer(il, d["ctx"]) for il in range(n)) * d["seqs"] / GIB, d["kv_bytes"] / GIB, + delta=0.01) def test_flash_next_iq4nl_cards_within_budget(self): d = plan("xeon6741p-4xb70", FLASH_GGUF) @@ -912,6 +919,21 @@ def test_gemma_layer_split_cards_within_budget(self): self.assertEqual((d["placement"], d["gpus"]), ("gpu-split", 2)) self.check(d, config(g), weights(g), gemma4_kv) + def test_split_counts_the_context_checkpoints_on_the_host(self): + # llama-server keeps -ctxcp checkpoints of each slot's SWA cache in host RAM whatever + # card holds the layers: a split plan counts them in its host bytes, as one card does. + g = "google__gemma-4-26B-A4B-it-qat-q4_0-gguf" + d = plan("xeon6741p-4xb70", g, ctx=262144, seqs=6) + self.assertEqual(d["placement"], "gpu-split") + f = d["flags"] + self.assertEqual(f[f.index("--cache-ram") + 1], "0") + k = int(f[f.index("-ctxcp") + 1]) + ckpt = 25 * 2 * 8 * 256 * 2 * 1536 # 25 SWA layers, K and V at 2 B, over 1536 cells + self.assertEqual(d["ckpt_bytes"], d["seqs"] * k * ckpt) + m = fit.model_profile(config(g), weights(g)) + self.assertEqual(d["host_bytes"], int(fit.HOST_RUNTIME["llama.cpp"] + m["lookup_bytes"] + d["ckpt_bytes"])) + self.assertGreaterEqual(int(d["k8s"]["memory_limit"][:-2]) * GIB, d["host_bytes"]) + def test_layer_bytes_are_the_bytes_to_serve(self): # Sized from safetensors for a GGUF conversion: the MTP head is not served # (model_profile serve_bytes), and the vision projector loads on the first GPU diff --git a/tests/fit_test.sh b/tests/fit_test.sh index 85570b2d..e8afdf01 100755 --- a/tests/fit_test.sh +++ b/tests/fit_test.sh @@ -124,9 +124,12 @@ budget = min(c["budget"] for c in fit.gpu_budget(json.load(open(sys.argv[3])))[0 assert d["placement"] == "moe-offload" and len(ts) == d["gpus"] == 2 and sum(ts) == 49, (ts, d["gpus"]) assert ts != [30, 30] and "--n-cpu-moe" in f, f assert len(d["card_bytes"]) == 2 and max(d["card_bytes"]) <= budget, (d["card_bytes"], budget) +# context checkpoints (here the recurrent state) stay in host RAM on a split too +k = int(f[f.index("-ctxcp") + 1]) +assert f[f.index("--cache-ram") + 1] == "0" and 0 < d["seqs"] * k <= d["ckpt_bytes"] < d["host_bytes"], (k, d["ckpt_bytes"]) print(f[f.index("-ts") + 1]) ' "${ROOT}/console" "${out}" "${XEON}"); then - ok "fit: Flash-Next IQ4_NL on two cards, -ts ${ts} layers, every card within its budget" + ok "fit: Flash-Next IQ4_NL on two cards, -ts ${ts} layers, every card within its budget, checkpoints on the host" else fail "fit: Flash-Next IQ4_NL per-card split (rc ${rc})" "$(grep -E '"-ts"|card_bytes' -A3 <<< "${out}")"; fi out=$(bash "${K3D_GPU}" fit --host "${XEON}" --model "${FLASH}" --file Qwen3.8-Flash-Next-Q8_0-00002-of-00002.gguf --explain 2>&1); rc=$? if [ "${rc}" -eq 0 ] && grep -q ':Q8_0 on' <<< "${out}" && grep -q -- "-hf ${FLASH}:Q8_0" <<< "${out}"; then From 5dbc4df16bd6472f56df472e29fc303b8a9d2b40 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 02:25:25 -0700 Subject: [PATCH 07/13] fit: llama.cpp's output layer holds the copy of tied embeddings A model that ties its embeddings (Gemma 4) has no output.weight. llama.cpp b11749 then loads token_embd a second time as the output tensor (TENSOR_DUPLICATED), and src/llama-model-loader.cpp gives that copy the output layer's buffer type: the last card of a split, the one card otherwise. The input copy stays on the CPU, which the fit already counts as a host lookup table, but the output copy was counted nowhere, so the Gemma 4 QAT plans were 0.56 GiB short on the card holding the output layer. llama_output_copy names those bytes; llama_layer_bytes puts them on the output layer and the one-card llama.cpp placement adds them to its weights. --explain says so on both paths. On the CPU the copy maps the same file pages, so the CPU placement is unchanged. The fixture plans keep their context and sequence counts. Tests, failing before the change: the Gemma two-card split's per-card bytes with the copy on the output card, the one-card Gemma plan's GPU bytes with the copy, and a shell check that --explain names it. --- console/fit.py | 27 ++++++++++++++++++++------- docs/fit.md | 5 ++++- tests/fit_test.py | 23 ++++++++++++++++++++--- tests/fit_test.sh | 8 ++++++++ 4 files changed, 52 insertions(+), 11 deletions(-) diff --git a/console/fit.py b/console/fit.py index 026a5f4e..50cfd872 100644 --- a/console/fit.py +++ b/console/fit.py @@ -1076,13 +1076,22 @@ def llama_layer_devices(ts, n_layer): return [bisect.bisect_right(splits, il / (n_layer + 1)) for il in range(n_layer + 1)] +def llama_output_copy(m): + """Bytes of the token embedding llama.cpp loads a second time as the output tensor when + the model ties its embeddings (no output.weight / lm_head): TENSOR_DUPLICATED, placed on + the output layer's device (src/llama-model-loader.cpp). On the CPU it maps the same file + pages; on a GPU it is a copy.""" + return m["bytes"]["embed"] if not m["bytes"]["lm_head"] else 0 + + def llama_layer_bytes(m, kv, ctx, seqs, n_cpu_moe): """([weights], [KV]) on the GPU per repeating layer, the output layer last: the class totals spread evenly over the layers, the experts of the first n_cpu_moe layers in host RAM (--n-cpu-moe), and each layer's own KV (its device holds it) for seqs sequences. The weights add up to the bytes to serve less the lookup tables (MTP only when the GGUF - holds it). The output layer holds lm_head and the MTP tensors; the vision projector - counts with layer 0, on the first card: tools/mtmd/clip.cpp loads it on the first GPU.""" + holds it), plus the output copy of tied embeddings. The output layer holds lm_head (or + that copy) and the MTP tensors; the vision projector counts with layer 0, on the first + card: tools/mtmd/clip.cpp loads it on the first GPU.""" n = max(m["layers"], 1) b = m["bytes"] exp, other = b["experts"] / n, b["other"] / n @@ -1093,7 +1102,7 @@ def llama_layer_bytes(m, kv, ctx, seqs, n_cpu_moe): wl = [other + (exp if il >= n_cpu_moe else 0) for il in range(n)] wl[0] += b["vision"] kl = [per[kinds[il]] * seqs if il < len(kinds) else 0 for il in range(n)] - out = m["serve_bytes"] - m["lookup_bytes"] - b["experts"] - b["other"] - b["vision"] + out = m["serve_bytes"] - m["lookup_bytes"] - b["experts"] - b["other"] - b["vision"] + llama_output_copy(m) return wl + [out], kl + [0] @@ -1158,7 +1167,7 @@ def split(n): return None -def _llama_card_lines(use, ts, wl, kl, ov): +def _llama_card_lines(use, ts, wl, kl, ov, m): """One explain line per card of a llama.cpp split.""" lines, first = [], 0 for c, t, w, kvb in zip(use, ts, llama_cards(ts, wl), llama_cards(ts, kl)): @@ -1166,7 +1175,8 @@ def _llama_card_lines(use, ts, wl, kl, ov): % (c["card"], t, "%d-%d" % (first, first + t - 1) if t else "none", g1(w), g1(kvb), g1(ov), g1(w + kvb + ov), g1(c["budget"]))) first += t - lines.append(" (layer %d is the output layer: lm_head)" % (len(wl) - 1)) + lines.append(" (layer %d is the output layer: %s)" % (len(wl) - 1, "lm_head" if not llama_output_copy(m) else + "a %s copy of the tied token embedding" % g1(llama_output_copy(m)))) return lines @@ -1449,11 +1459,14 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, else: # One card (llama.cpp on several is sized card by card in _llama_multi). budget = sum(c["budget"] for c in use) - w = gpu_w_all + w = gpu_w_all + (llama_output_copy(m) if eng == "llama.cpp" else 0) left = budget - w - ov * k scale = 1.0 lines = ["%s: budget %s - weights %s - overhead %d x %s = %s for KV" % ("+".join(c["card"] for c in use), g1(budget), g1(w), k, ovl, g1(max(left, 0)))] + if eng == "llama.cpp" and llama_output_copy(m): + lines.append("weights include the output tensor: a %s copy of the tied token embedding" + % g1(llama_output_copy(m))) if on_host: lines.append("lookup tables %s stay in host RAM" % g1(on_host)) if left <= 0: @@ -1565,7 +1578,7 @@ def _llama_multi(m, kv, ladder, relaxed, want_seqs, use, ov, ovl, n_cpus, on_hos c, n, n_cpu, ts = found wl, kl = llama_layer_bytes(m, kv, c, n, n_cpu) cards_b = [int(b + ov) for b in llama_cards(ts, [a + b for a, b in zip(wl, kl)])] - lines += _llama_card_lines(use, ts, wl, kl, ov) + lines += _llama_card_lines(use, ts, wl, kl, ov, m) if m["bytes"]["vision"]: lines.append(" (%s holds the vision projector, %s: tools/mtmd/clip.cpp loads it on the first GPU)" % (use[0]["card"], g1(m["bytes"]["vision"]))) diff --git a/docs/fit.md b/docs/fit.md index 22ca5ff3..d62c2298 100644 --- a/docs/fit.md +++ b/docs/fit.md @@ -129,7 +129,10 @@ layers it gets, the way llama.cpp b11749 assigns them with `-ngl 999` (`src/llama-model.cpp`, `load_tensors`). Layer `il` goes to the device `upper_bound(cumsum(ts) / sum(ts), il / (layers + 1))`, and the output layer (`il = layers`, holding `lm_head` and a GGUF's MTP tensors) goes the same way, which is -the last card. Each layer's weights are its share of the class totals, minus its experts +the last card. A model with tied embeddings (Gemma 4) has no `lm_head`: llama.cpp loads +`token_embd` a second time as the output tensor on that device +(`src/llama-model-loader.cpp`, `TENSOR_DUPLICATED`), so the output layer counts that copy, +on one card as on several. Each layer's weights are its share of the class totals, minus its experts when it is one of the first `--n-cpu-moe` layers. The vision projector counts on the first card: `tools/mtmd/clip.cpp` loads it on the first GPU. Each layer's KV and recurrent state sit on its card, a sliding-window layer's at its padded cell count (above); the context diff --git a/tests/fit_test.py b/tests/fit_test.py index 01ab28f8..36b842f2 100755 --- a/tests/fit_test.py +++ b/tests/fit_test.py @@ -825,8 +825,9 @@ def llama_cards(d, cfg, w, kv_layer): count): layer il on device upper_bound(cumsum(ts) / sum(ts), il / (n_layer + 1)), the output layer (il = n_layer) the same way, the experts of the first --n-cpu-moe layers in host RAM, each layer's KV on its device, and each card's own overhead. Weights per layer - are the class totals over the layers; the output layer holds lm_head and the MTP tensors - (a GGUF's), and the vision projector is on the first GPU (tools/mtmd/clip.cpp). + are the class totals over the layers; the output layer holds lm_head (with tied embeddings + a copy of token_embd) and the MTP tensors (a GGUF's), and the vision projector is on the + first GPU (tools/mtmd/clip.cpp). kv_layer(il, ctx) is one sequence's KV of layer il.""" tc = cfg.get("text_config") or cfg n = tc["num_hidden_layers"] @@ -843,7 +844,9 @@ def llama_cards(d, cfg, w, kv_layer): for il in range(n + 1): dev = bisect.bisect_right(cum, il / (n + 1)) if il == n: - cards[dev] += cls.get("lm_head", 0) + (cls.get("mtp", 0) if w["format"] == "gguf" else 0) + # the output weight: lm_head, or with tied embeddings (no output.weight) a copy of + # token_embd that llama.cpp loads as the output tensor (src/llama-model-loader.cpp) + cards[dev] += (cls.get("lm_head", 0) or cls.get("embed", 0)) + (cls.get("mtp", 0) if w["format"] == "gguf" else 0) else: cards[dev] += (cls.get("other", 0) / n + (cls.get("experts", 0) / n if il >= ncpu else 0) + kv_layer(il, d["ctx"]) * d["seqs"]) @@ -919,6 +922,20 @@ def test_gemma_layer_split_cards_within_budget(self): self.assertEqual((d["placement"], d["gpus"]), ("gpu-split", 2)) self.check(d, config(g), weights(g), gemma4_kv) + def test_tied_output_copy_on_one_card(self): + # Gemma 4 ties its embeddings: no output.weight, so llama.cpp loads token_embd a second + # time as the output tensor, on the output layer's device (src/llama-model-loader.cpp, + # TENSOR_DUPLICATED). On one card that copy is GPU bytes; the input copy stays on the host. + g = "google__gemma-4-26B-A4B-it-qat-q4_0-gguf" + d = plan("xeon6741p-4xb70", g) + self.assertEqual((d["engine"], d["placement"], d["gpus"]), ("llama.cpp", "gpu", 1)) + m = fit.model_profile(config(g), weights(g)) + self.assertEqual(m["bytes"]["lm_head"], 0) + ov = fit._overhead("llama.cpp", m, fit.TARGET_SEQS, None)[0] + want = m["serve_bytes"] - m["lookup_bytes"] + m["bytes"]["embed"] + ov + d["kv_bytes"] + self.assertAlmostEqual(d["gpu_bytes"] / GIB, want / GIB, delta=0.01) + self.assertLessEqual(d["gpu_bytes"], max(c["budget"] for c in fit.gpu_budget(host("xeon6741p-4xb70"))[0])) + def test_split_counts_the_context_checkpoints_on_the_host(self): # llama-server keeps -ctxcp checkpoints of each slot's SWA cache in host RAM whatever # card holds the layers: a split plan counts them in its host bytes, as one card does. diff --git a/tests/fit_test.sh b/tests/fit_test.sh index e8afdf01..d06a216e 100755 --- a/tests/fit_test.sh +++ b/tests/fit_test.sh @@ -86,6 +86,14 @@ assert int(d["k8s"]["memory_limit"][:-2]) * 2 ** 30 >= d["host_bytes"] >= d["kv_ ok "fit (llama.cpp, Gemma QAT on the desktop CPU): --cache-ram 0, -ctxcp K, checkpoints in the pod limit" else fail "fit (llama.cpp host caches) on the desktop (rc ${rc})" "${out}"; fi +# Gemma 4 ties its embeddings: llama.cpp loads token_embd again as the output tensor, on the +# output layer's device, so one card holds that copy beside the layers. +out=$(bash "${K3D_GPU}" fit --host "${XEON}" --model google/gemma-4-26B-A4B-it-qat-q4_0-gguf --explain 2>&1); rc=$? +if [ "${rc}" -eq 0 ] && grep -q 'llama.cpp/gpu: FITS' <<< "${out}" \ + && grep -q 'weights include the output tensor: a 0.6 GiB copy of the tied token embedding' <<< "${out}"; then + ok "fit (llama.cpp, Gemma 4 on one card): the tied embedding's output copy counts as GPU bytes" +else fail "fit (llama.cpp, tied output copy) on the Xeon (rc ${rc})" "$(grep -E 'llama.cpp/|weights' <<< "${out}")"; fi + out=$(bash "${K3D_GPU}" fit --host "${XEON}" --model not/in-cache-offline 2>&1 Date: Thu, 29 Oct 2026 03:18:18 -0700 Subject: [PATCH 08/13] fit: a GGUF's MTP blocks take llama.cpp layer slots on a split llama.cpp b11749 assigns devices over n_layer_all, the GGUF's block count, which counts the MTP (nextn) blocks a GGUF keeps after its trunk (src/llama-model.cpp load_tensors, get_layer_buft_list). Intel/Qwen3.8-27B-q4km-AutoRound's GGUF holds one: qwen35.block_count 65 and qwen35.nextn_predict_layers 1 for a 64-layer trunk, with blk.64.nextn.* in its header. The per-card split counted only the trunk, so its -ts summed to 65 where llama.cpp spreads 66 slots, and the first card got one trunk layer more than it was sized for (-ts 34,31 put layers 0-34 there, not 0-33). model_profile now records the MTP blocks a GGUF holds (the config's mtp_num_hidden_layers or num_nextn_predict_layers, when the GGUF has MTP tensors), and llama_layer_bytes puts them after the trunk with their MTP tensors and no KV: the cache covers il < n_layer(), the trunk only. The output layer holds lm_head (or the tied embedding's copy) alone. The same plan is now -ts 34,32, and each card's bytes match llama.cpp's assignment over 66 slots. --explain names the MTP block. Plans of GGUFs without MTP tensors (Gemma 4, the Flash-Next GGUFs) are unchanged. Test written first and failing before the change: the Qwen3.8 27B GGUF split on two cards, its -ts summing to 66 and its per-card bytes worked out the way llama.cpp assigns them. --- console/fit.py | 31 +++++++++++++++++--------- docs/fit.md | 9 +++++--- tests/fit_test.py | 55 +++++++++++++++++++++++++++++++++++++---------- 3 files changed, 71 insertions(+), 24 deletions(-) diff --git a/console/fit.py b/console/fit.py index 50cfd872..9a434b59 100644 --- a/console/fit.py +++ b/console/fit.py @@ -903,6 +903,10 @@ def model_profile(cfg, weights, tokenizer_cfg=None): "intermediate": max(tc.get("intermediate_size") or 0, (tc.get("moe_intermediate_size") or 0) * max(k, 1)), "max_ctx": cap, "ctx_why": capwhy, "experts": ne, "experts_per_tok": k, "moe": bool(ne and b["experts"]), + # MTP (nextn) blocks a GGUF holds after its trunk (llama.cpp's converter keeps them all + # or none): llama.cpp counts them in its layers (block_count = trunk + nextn_predict_layers). + "nextn": (tc.get("mtp_num_hidden_layers") or tc.get("num_nextn_predict_layers") or 1) + if weights["format"] == "gguf" and b["mtp"] else 0, "bytes": b, "serve_bytes": serve, "lookup_bytes": b["embed"] + b["ngram"], "format": weights["format"], "dtype_bytes": 2 if dtype in ("bfloat16", "float16") else 4, # compressed-tensors kv_cache_scheme: vLLM switches its KV cache to that type (fp8) on its own. @@ -1086,23 +1090,26 @@ def llama_output_copy(m): def llama_layer_bytes(m, kv, ctx, seqs, n_cpu_moe): """([weights], [KV]) on the GPU per repeating layer, the output layer last: the class - totals spread evenly over the layers, the experts of the first n_cpu_moe layers in host - RAM (--n-cpu-moe), and each layer's own KV (its device holds it) for seqs sequences. - The weights add up to the bytes to serve less the lookup tables (MTP only when the GGUF - holds it), plus the output copy of tied embeddings. The output layer holds lm_head (or - that copy) and the MTP tensors; the vision projector counts with layer 0, on the first - card: tools/mtmd/clip.cpp loads it on the first GPU.""" - n = max(m["layers"], 1) + totals spread evenly over the trunk's layers, the experts of the first n_cpu_moe layers in + host RAM (--n-cpu-moe), and each layer's own KV (its device holds it) for seqs sequences. + A GGUF's MTP (nextn) blocks follow the trunk as repeating layers (llama.cpp b11749 counts + them in n_layer_all, src/llama-model.cpp) with their share of the MTP tensors and no KV: + the cache covers il < n_layer(), the trunk. The weights add up to the bytes to serve less + the lookup tables, plus the output copy of tied embeddings. The output layer holds lm_head + (or that copy); the vision projector counts with layer 0, on the first card: + tools/mtmd/clip.cpp loads it on the first GPU.""" + n, nx = max(m["layers"], 1), m["nextn"] b = m["bytes"] exp, other = b["experts"] / n, b["other"] / n f, w, st = kv_parts(kv, ctx) per = {"full": f / max(kv["full_layers"], 1), "swa": w / max(kv["swa_layers"], 1), "linear": st / max(kv["linear_layers"], 1)} kinds = kv.get("kinds") or [] - wl = [other + (exp if il >= n_cpu_moe else 0) for il in range(n)] + wl = [other + (exp if il >= n_cpu_moe else 0) for il in range(n)] + [b["mtp"] / max(nx, 1)] * nx wl[0] += b["vision"] - kl = [per[kinds[il]] * seqs if il < len(kinds) else 0 for il in range(n)] - out = m["serve_bytes"] - m["lookup_bytes"] - b["experts"] - b["other"] - b["vision"] + llama_output_copy(m) + kl = [per[kinds[il]] * seqs if il < len(kinds) else 0 for il in range(n)] + [0] * nx + out = (m["serve_bytes"] - m["lookup_bytes"] - b["experts"] - b["other"] - b["vision"] - (b["mtp"] if nx else 0) + + llama_output_copy(m)) return wl + [out], kl + [0] @@ -1175,6 +1182,10 @@ def _llama_card_lines(use, ts, wl, kl, ov, m): % (c["card"], t, "%d-%d" % (first, first + t - 1) if t else "none", g1(w), g1(kvb), g1(ov), g1(w + kvb + ov), g1(c["budget"]))) first += t + if m["nextn"]: + a, z = m["layers"], m["layers"] + m["nextn"] - 1 + lines.append(" (%s the GGUF's MTP %s: nextn tensors, no KV)" + % ("layer %d is" % a if a == z else "layers %d-%d are" % (a, z), "block" if a == z else "blocks")) lines.append(" (layer %d is the output layer: %s)" % (len(wl) - 1, "lm_head" if not llama_output_copy(m) else "a %s copy of the tied token embedding" % g1(llama_output_copy(m)))) return lines diff --git a/docs/fit.md b/docs/fit.md index d62c2298..dffb2e38 100644 --- a/docs/fit.md +++ b/docs/fit.md @@ -127,9 +127,12 @@ linear-attention layers makes none. On several GPUs llama.cpp places whole layers, so the fit sizes each card from the layers it gets, the way llama.cpp b11749 assigns them with `-ngl 999` (`src/llama-model.cpp`, `load_tensors`). Layer `il` goes to the device -`upper_bound(cumsum(ts) / sum(ts), il / (layers + 1))`, and the output layer -(`il = layers`, holding `lm_head` and a GGUF's MTP tensors) goes the same way, which is -the last card. A model with tied embeddings (Gemma 4) has no `lm_head`: llama.cpp loads +`upper_bound(cumsum(ts) / sum(ts), il / (layers + 1))`, where layers is the GGUF's block +count (`n_layer_all`): the trunk, then the MTP (nextn) blocks a GGUF holds. Qwen3.8 27B's +GGUF keeps one (`qwen35.block_count` 65 for 64 trunk layers); an MTP block counts its MTP +tensors and no KV, since the cache covers the trunk only (`il < n_layer()`). The output +layer (`il = layers`, holding `lm_head`) goes the same way, which is the last card. A +model with tied embeddings (Gemma 4) has no `lm_head`: llama.cpp loads `token_embd` a second time as the output tensor on that device (`src/llama-model-loader.cpp`, `TENSOR_DUPLICATED`), so the output layer counts that copy, on one card as on several. Each layer's weights are its share of the class totals, minus its experts diff --git a/tests/fit_test.py b/tests/fit_test.py index 36b842f2..5600834c 100755 --- a/tests/fit_test.py +++ b/tests/fit_test.py @@ -819,18 +819,29 @@ def test_falls_to_fewer_bits_when_int4_does_not_fit(self): self.assertTrue(any("IQ4_NL" in s for s in d["explain"]) and any("Q2_K" in s for s in d["explain"])) +def gguf_nextn(cfg, w): + """MTP (nextn) blocks a GGUF holds after its trunk: the config's count when the GGUF has + MTP tensors (llama.cpp's converter keeps or drops them all).""" + tc = cfg.get("text_config") or cfg + if w["format"] != "gguf" or not w["classes"].get("mtp", {}).get("bytes"): + return 0 + return tc.get("mtp_num_hidden_layers") or tc.get("num_nextn_predict_layers") or 1 + + def llama_cards(d, cfg, w, kv_layer): """Each card's GPU bytes for a llama.cpp plan, worked out here from its flags the way llama.cpp b11749 assigns them (src/llama-model.cpp load_tensors, -ngl past the layer - count): layer il on device upper_bound(cumsum(ts) / sum(ts), il / (n_layer + 1)), the - output layer (il = n_layer) the same way, the experts of the first --n-cpu-moe layers in - host RAM, each layer's KV on its device, and each card's own overhead. Weights per layer - are the class totals over the layers; the output layer holds lm_head (with tied embeddings - a copy of token_embd) and the MTP tensors (a GGUF's), and the vision projector is on the - first GPU (tools/mtmd/clip.cpp). + count): layer il on device upper_bound(cumsum(ts) / sum(ts), il / (n_layer_all + 1)), the + output layer (il = n_layer_all) the same way, the experts of the first --n-cpu-moe layers + in host RAM, each layer's KV on its device, and each card's own overhead. n_layer_all is + the GGUF's block count: the trunk, then the MTP blocks a GGUF holds (gguf_nextn). Weights + per trunk layer are the class totals over the trunk; an MTP block holds its share of the + MTP tensors and no KV; the output layer holds lm_head (with tied embeddings a copy of + token_embd), and the vision projector is on the first GPU (tools/mtmd/clip.cpp). kv_layer(il, ctx) is one sequence's KV of layer il.""" tc = cfg.get("text_config") or cfg n = tc["num_hidden_layers"] + nx = gguf_nextn(cfg, w) f = d["flags"] ts = [int(x) for x in f[f.index("-ts") + 1].split(",")] ncpu = int(f[f.index("--n-cpu-moe") + 1]) if "--n-cpu-moe" in f else n if "--cpu-moe" in f else 0 @@ -841,12 +852,14 @@ def llama_cards(d, cfg, w, kv_layer): cum.append(c / sum(ts)) cards = [0] * len(ts) cards[0] += cls.get("vision", 0) - for il in range(n + 1): - dev = bisect.bisect_right(cum, il / (n + 1)) - if il == n: + for il in range(n + nx + 1): + dev = bisect.bisect_right(cum, il / (n + nx + 1)) + if il == n + nx: # the output weight: lm_head, or with tied embeddings (no output.weight) a copy of # token_embd that llama.cpp loads as the output tensor (src/llama-model-loader.cpp) - cards[dev] += (cls.get("lm_head", 0) or cls.get("embed", 0)) + (cls.get("mtp", 0) if w["format"] == "gguf" else 0) + cards[dev] += cls.get("lm_head", 0) or cls.get("embed", 0) + elif il >= n: + cards[dev] += cls["mtp"] / nx else: cards[dev] += (cls.get("other", 0) / n + (cls.get("experts", 0) / n if il >= ncpu else 0) + kv_layer(il, d["ctx"]) * d["seqs"]) @@ -861,6 +874,12 @@ def flash_next_kv(il, ctx): return (2 * 2 * 256 * 2 + 128 * 2) * ctx if il % 4 == 3 else 48 * 128 * 128 * 4 + 3 * 10240 * 2 +def qwen38_27b_kv(il, ctx): + # every 4th layer full attention: 2 x 4 kv heads x 256 x 2 B per token; the rest gated + # DeltaNet: 48 x 128 x 128 x 4 B state + 3 x (2 x 16 x 128 + 48 x 128) conv ch x 2 B per sequence + return 2 * 4 * 256 * 2 * ctx if il % 4 == 3 else 48 * 128 * 128 * 4 + 3 * (2 * 16 * 128 + 48 * 128) * 2 + + def gemma4_kv(il, ctx): # every 6th layer global: 2 x 2 kv heads x 512 x 2 B per token; the rest a 1024-token window, # whose llama.cpp b11749 cache is GGML_PAD(min(n_ctx_seq, 1024 + 512 ubatch), 256) cells per @@ -894,7 +913,7 @@ def test_old_split_overflows_the_second_card(self): def check(self, d, cfg, w, kv_layer): n = (cfg.get("text_config") or cfg)["num_hidden_layers"] ts, cards = llama_cards(d, cfg, w, kv_layer) - self.assertEqual(sum(ts), n + 1, ts) + self.assertEqual(sum(ts), n + gguf_nextn(cfg, w) + 1, ts) self.assertEqual(len(ts), d["gpus"]) budget = self.budget(d["gpus"]) for i, b in enumerate(cards): @@ -922,6 +941,20 @@ def test_gemma_layer_split_cards_within_budget(self): self.assertEqual((d["placement"], d["gpus"]), ("gpu-split", 2)) self.check(d, config(g), weights(g), gemma4_kv) + def test_mtp_blocks_take_layer_slots(self): + # Intel/Qwen3.8-27B-q4km-AutoRound's GGUF holds Qwen3.8's MTP block after its 64-layer + # trunk (its header at 4b30b30e: qwen35.block_count 65, qwen35.nextn_predict_layers 1, + # blk.64.nextn.*). llama.cpp b11749 assigns devices over all 65 blocks plus the output + # layer (src/llama-model.cpp, n_layer_all) and gives the MTP block no KV (the cache + # covers il < n_layer()), so -ts counts 66 layers: counts summing to 65 put one more + # trunk layer on the first card than the fit sized it for. + g = "Intel__Qwen3.8-27B-q4km-AutoRound" + d = plan("xeon6741p-4xb70", g, gpus=2, device="gpu") + self.assertTrue(d["fits"], d["summary"]) + self.assertEqual((d["engine"], d["placement"], d["gpus"]), ("llama.cpp", "gpu-split", 2)) + self.check(d, config(g), weights(g), qwen38_27b_kv) + self.assertTrue(any("layer 64" in s and "MTP" in s for s in d["explain"]), d["explain"]) + def test_tied_output_copy_on_one_card(self): # Gemma 4 ties its embeddings: no output.weight, so llama.cpp loads token_embd a second # time as the output tensor, on the output layer's device (src/llama-model-loader.cpp, From 303d7d520e99aec1123c08193e92d2fbdb4bcafc Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 12:53:51 -0700 Subject: [PATCH 09/13] fit: a GGUF conversion keeps the MTP blocks, so llama.cpp's split counts them The agent converts safetensors that only llama.cpp runs here (Qwen3.8-Flash-Next) with convert_hf_to_gguf.py and no --no-mtp. llama.cpp v0.8.0's converter then keeps the MTP head after the trunk (conversion/qwen.py _QwenMtpMixin adds mtp_num_hidden_layers to block_count), so the GGUF it writes has 49 blocks for Flash-Next's 48 trunk layers, and llama-server assigns devices over all 49 plus the output layer. The fit sized the conversion as safetensors, without the MTP block, so its -ts summed to 49 and put one more trunk layer on the first card than it sized (-ts 39,10). llama_profile is the model llama.cpp serves: a GGUF as it is, other weights as the GGUF the conversion writes, with the MTP blocks and their bytes, the way a GGUF repo that holds an MTP head is already sized. vLLM and OVMS keep the profile without them. Flash-Next on the Xeon fixture is now -ts 39,11 --n-cpu-moe 45, every card within its budget, and --explain names the conversion's MTP block. --- console/fit.py | 24 ++++++++++++++++++++---- docs/fit.md | 7 +++++-- tests/fit_test.py | 33 +++++++++++++++++++++++++++++---- 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/console/fit.py b/console/fit.py index 9a434b59..7d13be9a 100644 --- a/console/fit.py +++ b/console/fit.py @@ -889,6 +889,7 @@ def model_profile(cfg, weights, tokenizer_cfg=None): b = {k: cls.get(k, {}).get("bytes", 0) for k in ("experts", "embed", "ngram", "lm_head", "vision", "mtp", "other")} # The MTP head loads only for speculative decoding; a GGUF loads every tensor it holds. serve = sum(v for k, v in b.items() if k != "mtp" or weights["format"] == "gguf") + mtp_blocks = (tc.get("mtp_num_hidden_layers") or tc.get("num_nextn_predict_layers") or 1) if b["mtp"] else 0 ne = tc.get("num_experts") or tc.get("n_routed_experts") or tc.get("num_local_experts") or 0 k = tc.get("num_experts_per_tok") or tc.get("top_k_experts") or tc.get("num_experts_per_token") or 0 dtype = str(tc.get("dtype") or tc.get("torch_dtype") or cfg.get("torch_dtype") or "bfloat16") @@ -905,8 +906,8 @@ def model_profile(cfg, weights, tokenizer_cfg=None): "experts": ne, "experts_per_tok": k, "moe": bool(ne and b["experts"]), # MTP (nextn) blocks a GGUF holds after its trunk (llama.cpp's converter keeps them all # or none): llama.cpp counts them in its layers (block_count = trunk + nextn_predict_layers). - "nextn": (tc.get("mtp_num_hidden_layers") or tc.get("num_nextn_predict_layers") or 1) - if weights["format"] == "gguf" and b["mtp"] else 0, + # mtp_blocks: the MTP blocks the weights have, which a GGUF conversion keeps (llama_profile). + "nextn": mtp_blocks if weights["format"] == "gguf" else 0, "mtp_blocks": mtp_blocks, "bytes": b, "serve_bytes": serve, "lookup_bytes": b["embed"] + b["ngram"], "format": weights["format"], "dtype_bytes": 2 if dtype in ("bfloat16", "float16") else 4, # compressed-tensors kv_cache_scheme: vLLM switches its KV cache to that type (fp8) on its own. @@ -914,6 +915,16 @@ def model_profile(cfg, weights, tokenizer_cfg=None): } +def llama_profile(m): + """The model llama.cpp serves: a GGUF as it is; other weights as the GGUF the agent's + conversion writes from them. convert_hf_to_gguf.py runs without --no-mtp, so that GGUF + keeps the MTP blocks after the trunk (llama.cpp v0.8.0 conversion/qwen.py _QwenMtpMixin + adds mtp_num_hidden_layers to block_count) and is served like a GGUF that holds them.""" + if m["format"] == "gguf" or not m["mtp_blocks"]: + return m + return dict(m, nextn=m["mtp_blocks"], serve_bytes=m["serve_bytes"] + m["bytes"]["mtp"]) + + # ---- engines --------------------------------------------------------------------------- # Which engine serves which architecture on Intel GPUs or the CPU, from each engine's # own release notes and recipes. An architecture not listed is not refused (the engine @@ -1282,11 +1293,16 @@ def fit(host, cfg, weights, engine=None, ctx=None, seqs=None, gpus=None, kv_dtyp "summary": "fit: %s does not fit this host: %s cannot serve these %s weights (%s)" % (name, engine, m["format"], why)} cands = asked - kvs = {} + kvs, ms = {}, {} for eng, note in cands: ex.append("engine %s: %s" % (eng or FORMAT_ENGINE.get(m["format"], "?"), note)) if not eng: continue + ms[eng] = llama_profile(m) if eng == "llama.cpp" else m + if ms[eng] is not m: + ex.append(" the GGUF conversion keeps the MTP head (convert_hf_to_gguf.py without --no-mtp): " + "%d block(s) after the trunk, %s more to serve, %s" + % (ms[eng]["nextn"], g1(m["bytes"]["mtp"]), g1(ms[eng]["serve_bytes"]))) if kv_dtype != "auto" and kv_dtype not in KV_DTYPES_OF.get(eng, ()): raise ValueError("--kv-dtype %s is not a KV cache type of %s (%s)" % (kv_dtype, eng, ", ".join(KV_DTYPES_OF.get(eng, ())) or "auto only")) @@ -1310,7 +1326,7 @@ def fit(host, cfg, weights, engine=None, ctx=None, seqs=None, gpus=None, kv_dtyp # vLLM's CPU build is a placement only when asked for (--device cpu: the EAT # overlay); the agent's vLLM images are the XPU and Gaudi builds. cpu_ok = device == "cpu" if eng == "vllm" else device != "gpu" - for res in _placements(eng, m, kvs[eng], ladder, relaxed, seqs, gpus, cards, hfree, host, overhead, + for res in _placements(eng, ms[eng], kvs[eng], ladder, relaxed, seqs, gpus, cards, hfree, host, overhead, cpu_ok): res["note"] = note tried.append(res) diff --git a/docs/fit.md b/docs/fit.md index dffb2e38..ac7f9051 100644 --- a/docs/fit.md +++ b/docs/fit.md @@ -79,7 +79,8 @@ in this order, and the first that fits wins: Per placement: ``` -weights = bytes of each tensor class to serve (MTP left out unless the GGUF holds it) +weights = bytes of each tensor class to serve (MTP left out unless the GGUF holds it, + as llama.cpp's conversion of safetensors does) KV per token = full layers x 2 x kv heads x head dim x bytes + indexer keys (sparse attention) + sliding layers x 2 x kv heads x head dim x bytes, for: @@ -129,7 +130,9 @@ layers it gets, the way llama.cpp b11749 assigns them with `-ngl 999` (`src/llama-model.cpp`, `load_tensors`). Layer `il` goes to the device `upper_bound(cumsum(ts) / sum(ts), il / (layers + 1))`, where layers is the GGUF's block count (`n_layer_all`): the trunk, then the MTP (nextn) blocks a GGUF holds. Qwen3.8 27B's -GGUF keeps one (`qwen35.block_count` 65 for 64 trunk layers); an MTP block counts its MTP +GGUF keeps one (`qwen35.block_count` 65 for 64 trunk layers), and so does the GGUF the agent +converts from safetensors: `convert_hf_to_gguf.py` runs without `--no-mtp`, so +Qwen3.8-Flash-Next's has 49 blocks for 48 trunk layers. An MTP block counts its MTP tensors and no KV, since the cache covers the trunk only (`il < n_layer()`). The output layer (`il = layers`, holding `lm_head`) goes the same way, which is the last card. A model with tied embeddings (Gemma 4) has no `lm_head`: llama.cpp loads diff --git a/tests/fit_test.py b/tests/fit_test.py index 5600834c..550d9ba3 100755 --- a/tests/fit_test.py +++ b/tests/fit_test.py @@ -985,17 +985,42 @@ def test_split_counts_the_context_checkpoints_on_the_host(self): self.assertGreaterEqual(int(d["k8s"]["memory_limit"][:-2]) * GIB, d["host_bytes"]) def test_layer_bytes_are_the_bytes_to_serve(self): - # Sized from safetensors for a GGUF conversion: the MTP head is not served - # (model_profile serve_bytes), and the vision projector loads on the first GPU + # Sized from safetensors for a GGUF conversion: vLLM and OVMS leave the MTP head out + # (model_profile serve_bytes), but the GGUF the agent converts keeps it + # (convert_hf_to_gguf.py without --no-mtp), so llama.cpp's profile serves it as one + # more block after the trunk. The vision projector loads on the first GPU # (tools/mtmd/clip.cpp takes the first GPU backend), not on the output layer's card. cfg = config("Qwen__Qwen3.8-Flash-Next") m = fit.model_profile(cfg, weights("Qwen__Qwen3.8-Flash-Next")) self.assertGreater(m["bytes"]["mtp"], GIB) self.assertGreater(m["bytes"]["vision"], GIB / 2) - wl, _ = fit.llama_layer_bytes(m, fit.kv_profile(cfg, 2), 4096, 1, 0) - self.assertAlmostEqual(sum(wl), m["serve_bytes"] - m["lookup_bytes"], delta=1) + lm = fit.llama_profile(m) + self.assertEqual((m["nextn"], lm["nextn"]), (0, 1)) + self.assertEqual(lm["serve_bytes"], m["serve_bytes"] + m["bytes"]["mtp"]) + wl, kl = fit.llama_layer_bytes(lm, fit.kv_profile(cfg, 2), 4096, 1, 0) + self.assertEqual(len(wl), 48 + 1 + 1) + self.assertAlmostEqual(sum(wl), lm["serve_bytes"] - lm["lookup_bytes"], delta=1) + self.assertAlmostEqual(wl[48], m["bytes"]["mtp"], delta=1) + self.assertEqual(kl[48], 0) self.assertAlmostEqual(wl[-1], m["bytes"]["lm_head"], delta=1) self.assertAlmostEqual(wl[0] - wl[1], m["bytes"]["vision"], delta=1) + # a GGUF is served as it is + g = fit.model_profile(config(FLASH_GGUF), weights(FLASH_GGUF)["groups"]["IQ4_NL"]) + self.assertIs(fit.llama_profile(g), g) + + def test_conversion_keeps_the_mtp_blocks(self): + # The agent converts safetensors only llama.cpp runs here (Qwen3.8-Flash-Next) with + # convert_hf_to_gguf.py and no --no-mtp, so the GGUF it serves holds the MTP block: + # llama.cpp v0.8.0 conversion/qwen.py (_QwenMtpMixin) extends block_count by + # mtp_num_hidden_layers. llama.cpp assigns devices over that block too, so -ts counts + # it as for a GGUF repo that holds one, and the card it lands on holds its tensors. + g = "Qwen__Qwen3.8-Flash-Next" + d = plan("xeon6741p-4xb70", g) + self.assertTrue(d["fits"], d["summary"]) + self.assertEqual((d["engine"], d["placement"]), ("llama.cpp", "moe-offload")) + self.assertEqual(d["convert"]["args"], ["--outtype", "bf16"]) + self.check(d, config(g), dict(weights(g), format="gguf"), flash_next_kv) # the converted GGUF + self.assertTrue(any("layer 48 is the GGUF's MTP block" in s for s in d["explain"]), d["explain"]) def test_moe_offload_takes_the_best_number_of_cards(self): # Every card is held to the smallest budget of the cards used, so two smaller cards From 9874581dff48357803991ca1a67b86c4cc3fe450 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 12:55:17 -0700 Subject: [PATCH 10/13] llamacpp_test: the per-card fit's flags reach llama-server The fit now hands llama-server --cache-ram 0 and -ctxcp 4 (the host caches it counts) and -ts as layer counts per card. Both options are in llama.cpp v0.8.0's parser (common/arg.cpp: -ctxcp/--ctx-checkpoints :1709, -cram/--cache-ram :1727), so the args check takes them, and the rendered servers are checked for them. The QAT GGUF on one free B70 is 131072 x 5 (-c 655360 -np 5): the card also holds the tied embedding's output copy. Flash-Next on two B70s is -ts 39,11 --n-cpu-moe 45 -np 4, the split sized card by card with the conversion's MTP block. --- tests/llamacpp_test.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/llamacpp_test.sh b/tests/llamacpp_test.sh index 0b864630..7c0f3801 100755 --- a/tests/llamacpp_test.sh +++ b/tests/llamacpp_test.sh @@ -92,15 +92,16 @@ pair() { server_args "$1" | awk -v o="$2" -v v="$3" 'p && $0 == v { f = 1 } { p # llama-server's parser (llama.cpp v0.8.0 common/arg.cpp parse_cli_args, :827-865) takes an # option's value from the next argument, never from "--opt=value", and refuses a name it does # not define. llama_argv_ok : the server args parse so, with the names and arities the -# renderer uses (common/arg.cpp v0.8.0: -t :1528, -tb :1538, -c :1650, -ctk :2449, -ctv :2462, -# -np :2556, --load-mode :2703, --cpu-moe :2772, --n-cpu-moe :2779, -ngl :2811, -ts :2853, -# --alias :3033, --model-url :3066, --host :3343, --port :3359, --jinja :3727, -# --reasoning :3746, --spec-draft-model :4322, --spec-type :4330, --model :3057). +# renderer uses (common/arg.cpp v0.8.0: -t :1528, -tb :1538, -c :1650, -ctxcp :1709, +# --cache-ram :1727, -ctk :2449, -ctv :2462, -np :2556, --load-mode :2703, --cpu-moe :2772, +# --n-cpu-moe :2779, -ngl :2811, -ts :2853, --alias :3033, --model-url :3066, --host :3343, +# --port :3359, --jinja :3727, --reasoning :3746, --spec-draft-model :4322, --spec-type :4330, +# --model :3057). llama_argv_ok() { server_args "$1" | python3 -c 'import sys arity = {"--model": 1, "--model-url": 1, "--alias": 1, "--host": 1, "--port": 1, "--jinja": 0, "--reasoning": 1, "-c": 1, "-np": 1, "-ngl": 1, "-ts": 1, "--n-cpu-moe": 1, "--cpu-moe": 0, - "-ctk": 1, "-ctv": 1, "--threads": 1, "--threads-batch": 1, "--load-mode": 1, + "-ctk": 1, "-ctv": 1, "--threads": 1, "--threads-batch": 1, "--load-mode": 1, "--cache-ram": 1, "-ctxcp": 1, "--spec-type": 1, "--spec-draft-model": 1} argv = sys.stdin.read().split("\n")[:-1] i = 0 @@ -133,7 +134,7 @@ if render gguf "${XEON_ENV[@]}" -- --model "${GEMMA_GGUF}" \ && pair gguf --model "/models/gguf/google__gemma-4-26B-A4B-it-qat-q4_0-gguf@${GREV}/gemma-4-26B_q4_0-it.gguf" \ && pair gguf --model-url "https://huggingface.co/${GEMMA_GGUF}/resolve/${GREV}/gemma-4-26B_q4_0-it.gguf" \ && pair gguf --alias 'gemma-4-26B-A4B-it-qat-q4_0-gguf' && arg gguf '--jinja' && pair gguf --reasoning 'off' \ - && arg gguf '-c' && arg gguf '786432' && arg gguf '-np' && arg gguf '6' && arg gguf '-ngl' && arg gguf '999' \ + && pair gguf -c 655360 && pair gguf -np 5 && pair gguf -ngl 999 && pair gguf --cache-ram 0 && pair gguf -ctxcp 4 \ && pair gguf --port '8000' && server_args gguf | grep -A1 -x -- '--threads' | tail -n1 | grep -qE '^[0-9]+$' \ && ! arg gguf '-ts' && ! has gguf 'spec-type' && ! has gguf 'kind: Job' && ! has gguf 'load-mode' && ! arg gguf '-hf' \ && grep -A3 '^ limits:$' "${WORK}/gguf.yaml" | grep -q '^ gpu.intel.com/xe: 1$' \ @@ -189,7 +190,7 @@ FN=Qwen/Qwen3.8-Flash-Next FNREV=de4b8e4d43b917e7706784d8bb445c9af86a3540 if render fn "${XEON_ENV[@]}" -- --model "${FN}" \ && pair fn --model "/models/gguf/Qwen__Qwen3.8-Flash-Next@${FNREV}/model.gguf" && ! has fn 'model-url' \ - && arg fn '--n-cpu-moe' && arg fn '44' && arg fn '-ts' && arg fn '30,30' && arg fn '-np' && arg fn '4' \ + && pair fn --n-cpu-moe 45 && pair fn -ts 39,11 && pair fn -np 4 && pair fn --cache-ram 0 && pair fn -ctxcp 4 \ && grep -A3 '^ limits:$' "${WORK}/fn.yaml" | grep -q '^ gpu.intel.com/xe: 2$' \ && has fn 'kind: Job' && has fn "image: ghcr.io/ggml-org/llama.cpp:full-b11960@${FULL}" \ && has fn '^ - "Qwen/Qwen3.8-Flash-Next"$' && has fn "^ - \"${FNREV}\"\$" \ From 6e05e9e2084dba75bce8a5edce7d7856a961c12a Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 12:58:50 -0700 Subject: [PATCH 11/13] agent: K3D_GPU_AGENT_GPUS reaches the fit for llama.cpp as for vLLM The OVMS fix passed --gpus to the fit only when the agent app was vLLM, so an OVMS pod, which holds one gpu.intel.com/xe, is no longer sized for two cards. The llama.cpp backend splits a model's layers over several cards and takes its card count from the fit, so with that rule K3D_GPU_AGENT_BACKEND=llama.cpp K3D_GPU_AGENT_GPUS=2 lost the 2 and served the QAT GGUF on one card. --gpus now reaches the fit for every app but OVMS: the QAT GGUF is split over the two free B70s with -ts in layers, and the OVMS pod stays on one card. The help and docs/fit.md say which backends read it. --- docs/fit.md | 6 +++--- scripts/k3d-gpu | 13 ++++++++----- tests/llamacpp_test.sh | 8 ++++++++ 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/docs/fit.md b/docs/fit.md index ac7f9051..7dc57689 100644 --- a/docs/fit.md +++ b/docs/fit.md @@ -163,9 +163,9 @@ The overhead is an estimate from the formula, not a measurement; `agent up` applies them, sets the pod's memory request and limit to the fit, and annotates the pod `k3d-gpu.io/agent-fit`. `K3D_GPU_AGENT_MAX_LEN`, `K3D_GPU_AGENT_MAX_NUM_SEQS` -and `K3D_GPU_AGENT_GPUS` (vLLM only: an OVMS pod holds one GPU, and `fit --gpus 2` with -OVMS is refused as "OVMS serves one GPU") are inputs the fit checks, not values that -replace it. With +and `K3D_GPU_AGENT_GPUS` are inputs the fit checks, not values that replace it. +`K3D_GPU_AGENT_GPUS` reaches the fit for the vLLM and llama.cpp backends only: an OVMS pod +holds one GPU, and `fit --gpus 2` with OVMS is refused as "OVMS serves one GPU". With `K3D_GPU_AGENT_BACKEND` unset, the engine the fit picks serves the model: OVMS, vLLM or llama-server. With `--model`, the model optimizer first picks the variant for the default engine (OVMS: Gemma 4's OpenVINO IR, for example). When that engine has nothing to serve, diff --git a/scripts/k3d-gpu b/scripts/k3d-gpu index 5ef513bc..ce470276 100755 --- a/scripts/k3d-gpu +++ b/scripts/k3d-gpu @@ -423,7 +423,8 @@ Environment: K3D_GPU_AUTOQUANT auto | off: on OVMS/CPU with no K3D_GPU_AGENT_MODEL, serve the precision measured for this host fingerprint (default: auto) K3D_GPU_AUTOQUANT_CACHE measured-pick cache (default: \$XDG_STATE_HOME/k3d-gpu/autoquant.json) - K3D_GPU_AGENT_GPUS vllm/llm-scaler: GPUs (or Gaudis) to shard the model across (default: 1) + K3D_GPU_AGENT_GPUS vllm/llm-scaler/llama.cpp: GPUs (or Gaudis) to shard or split the model + across (default: 1; OVMS serves one GPU) K3D_GPU_AGENT_MAX_LEN vllm/llm-scaler: max context tokens (default: 8192) K3D_GPU_FIT auto | on | off: size the agent model with 'fit' before agent up renders it (default: auto = a model you chose; on also sizes the @@ -7634,8 +7635,9 @@ agent_config # OVMS only: DEBUG also prints the compiled model's EXECUTION_DEVICES (e.g. GPU.0: # Intel(R) UHD Graphics 770), and every HTTP request, readiness probes included. AGENT_LOG_LEVEL="${K3D_GPU_AGENT_LOG_LEVEL:-INFO}" -# vLLM only: GPUs (or Gaudis) the model is sharded across (--tensor-parallel-size), -# and the longest context, sized so the default model's KV cache fits a 12 GB card. +# vLLM: GPUs (or Gaudis) the model is sharded across (--tensor-parallel-size); llama.cpp: +# the cards its layers are split over. And the longest context, sized so the default +# model's KV cache fits a 12 GB card. AGENT_GPUS="${K3D_GPU_AGENT_GPUS:-1}" AGENT_MAX_LEN="${K3D_GPU_AGENT_MAX_LEN:-8192}" @@ -11200,8 +11202,9 @@ agent_fit() { || { fit_soft "${mode}" "fit: no metadata for ${AGENT_MODEL} (offline, private or gated: set HF_TOKEN)"; return 0; } [ -z "${K3D_GPU_AGENT_MAX_LEN:-}" ] || args+=(--ctx "${K3D_GPU_AGENT_MAX_LEN}") [ -z "${K3D_GPU_AGENT_MAX_NUM_SEQS:-}" ] || args+=(--seqs "${K3D_GPU_AGENT_MAX_NUM_SEQS}") - # K3D_GPU_AGENT_GPUS shards a vLLM model; an OVMS pod holds one GPU whatever it says. - [ -z "${K3D_GPU_AGENT_GPUS:-}" ] || [ "${AGENT_APP}" != vllm ] || args+=(--gpus "${K3D_GPU_AGENT_GPUS}") + # K3D_GPU_AGENT_GPUS shards a vLLM model or splits a llama.cpp one; an OVMS pod holds + # one GPU whatever it says. + [ -z "${K3D_GPU_AGENT_GPUS:-}" ] || [ "${AGENT_APP}" = ovms ] || args+=(--gpus "${K3D_GPU_AGENT_GPUS}") # agent up replaces the running server (the Deployment is Recreate, and a backend # switch deletes the other one): it is stopped before the probe, so the memory it holds # is measured as free, not as someone else's. A model that then does not fit gets the diff --git a/tests/llamacpp_test.sh b/tests/llamacpp_test.sh index 7c0f3801..982a759f 100755 --- a/tests/llamacpp_test.sh +++ b/tests/llamacpp_test.sh @@ -379,6 +379,14 @@ else fail "an explicit backend is kept" "$(show set)"; fi if render setl "${XEON_ENV[@]}" K3D_GPU_AGENT_BACKEND=llama.cpp -- --model "${GEMMA_GGUF}" && has setl '^ name: llamacpp$'; then ok "K3D_GPU_AGENT_BACKEND=llama.cpp serves the GGUF" else fail "K3D_GPU_AGENT_BACKEND=llama.cpp" "$(show setl)"; fi +# K3D_GPU_AGENT_GPUS is the fit's --gpus for llama.cpp as for vLLM (an OVMS pod holds one +# GPU): the QAT GGUF, one card's model, is split layer by layer over the two free B70s asked. +if render setl2 "${XEON_ENV[@]}" K3D_GPU_AGENT_BACKEND=llama.cpp K3D_GPU_AGENT_GPUS=2 -- --model "${GEMMA_GGUF}" \ + && grep -q "fit: ${GEMMA_GGUF}:Q4_0 on 2 GPUs, layer split via llama.cpp" "${WORK}/setl2.err" \ + && server_args setl2 | grep -A1 -x -- '-ts' | tail -n1 | grep -qE '^[0-9]+,[0-9]+$' \ + && grep -A3 '^ limits:$' "${WORK}/setl2.yaml" | grep -q '^ gpu.intel.com/xe: 2$'; then + ok "K3D_GPU_AGENT_BACKEND=llama.cpp K3D_GPU_AGENT_GPUS=2: the fit splits the GGUF over two B70s, -ts in layers" +else fail "K3D_GPU_AGENT_GPUS=2 reaches the llama.cpp fit" "$(show setl2)"; fi # --- MTP on request --------------------------------------------------------------------- if render mtp "${XEON_ENV[@]}" K3D_GPU_AGENT_DRAFT=mtp -- --model "${GEMMA_GGUF}" \ From 46d55b35cf71a7bd85b935999b8bcf2cb9d279bb Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 12:59:32 -0700 Subject: [PATCH 12/13] fit: cite llama.cpp v0.8.0, the llama-server the agent runs The planner fixes read llama.cpp b11749; the agent's llama-server is now v0.8.0 (b11960). Every rule they size by holds there unchanged: the SWA cache of GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch), 256) cells (src/llama-kv-cache-iswa.cpp:73, n_ctx_seq padded to 256 in src/llama-context.cpp:393), at most -ctxcp checkpoints per slot made at the last user message and 4 + n_ubatch and 4 tokens before the prompt's end (common/common.h:641, tools/server/server-context.cpp:2746 and :4502-4518), the 8192 MiB --cache-ram default (common/common.h:644), the upper_bound layer split over n_layer_all + 1 with the output layer last (src/llama-model.cpp:1685-1710), and the tied embedding's output copy (src/llama-model-loader.cpp:1176, :1381). The comments, the tests' comments and docs/fit.md cite v0.8.0, and _flags cites the parser the flags were checked against. --- console/fit.py | 10 +++++----- docs/fit.md | 6 +++--- tests/fit_test.py | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/console/fit.py b/console/fit.py index 7d13be9a..112bca65 100644 --- a/console/fit.py +++ b/console/fit.py @@ -95,7 +95,7 @@ # the agent stack sets it, llama.cpp's default -ub 512). VLLM_BATCHED_TOKENS = 2048 LLAMA_UBATCH = 512 -# llama-server context checkpoints per slot (-ctxcp; b11749 default 32, common/common.h). Each +# llama-server context checkpoints per slot (-ctxcp; v0.8.0 default 32, common/common.h). Each # is a host-RAM copy of the slot's SWA cache and recurrent state (tools/server/server-context.cpp # create_checkpoint, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY). A request makes them at the last user # message and 4 + n_ubatch and 4 tokens before its prompt's end; an appended turn restores from @@ -127,7 +127,7 @@ # llama.cpp -ctk/-ctv; OVMS's KV_CACHE_PRECISION is not set by the fit). KV_DTYPES_OF = {"vllm": ("fp8", "fp16", "bf16"), "llama.cpp": ("f16", "fp16", "bf16", "q8_0", "q4_0")} # Tokens a sliding-window layer's cache holds per sequence (swa_tokens). vLLM: the window. -# llama.cpp b11749: GGML_PAD(n_swa + n_ubatch, 256) cells per stream (src/llama-kv-cache-iswa.cpp). +# llama.cpp v0.8.0: GGML_PAD(n_swa + n_ubatch, 256) cells per stream (src/llama-kv-cache-iswa.cpp). # OVMS: OpenVINO GenAI 2026.4 has only KV_CACHE and LINEAR_ATTENTION_CACHE types # (src/cpp/src/continuous_batching/cache/cache_type.hpp), and a KV block spans every # attention layer (kv_cache_manager.hpp), so sliding-window layers hold the whole context. @@ -1081,7 +1081,7 @@ def per_seq(kv, ctx, scale=1.0): # ---- llama.cpp over several GPUs ---------------------------------------------------------- def llama_layer_devices(ts, n_layer): """Device of each repeating layer and of the output layer (the last entry), as llama.cpp - b11749 assigns them with -ngl past the layer count (src/llama-model.cpp load_tensors): + v0.8.0 assigns them with -ngl past the layer count (src/llama-model.cpp load_tensors): upper_bound(cumsum(ts) / sum(ts), il / (n_layer + 1)). Integer layer counts summing to n_layer + 1 give each device exactly its count, the output layer on the last one.""" total, cum, splits = float(sum(ts)), 0.0, [] @@ -1103,7 +1103,7 @@ def llama_layer_bytes(m, kv, ctx, seqs, n_cpu_moe): """([weights], [KV]) on the GPU per repeating layer, the output layer last: the class totals spread evenly over the trunk's layers, the experts of the first n_cpu_moe layers in host RAM (--n-cpu-moe), and each layer's own KV (its device holds it) for seqs sequences. - A GGUF's MTP (nextn) blocks follow the trunk as repeating layers (llama.cpp b11749 counts + A GGUF's MTP (nextn) blocks follow the trunk as repeating layers (llama.cpp v0.8.0 counts them in n_layer_all, src/llama-model.cpp) with their share of the MTP tensors and no KV: the cache covers il < n_layer(), the trunk. The weights add up to the bytes to serve less the lookup tables, plus the output copy of tied embeddings. The output layer holds lm_head @@ -1641,7 +1641,7 @@ def _llama_multi(m, kv, ladder, relaxed, want_seqs, use, ov, ovl, n_cpus, on_hos def _flags(eng, place, ctx, seqs, k, gpu_use, cards, kvused, m, n_cpu_moe, kv=None, ts=None): """Engine memory flags from the fit. Spellings from each engine's own argument parser: vLLM v0.32.0 vllm/engine/arg_utils.py, OVMS v2026.4.1 src/graph_export/graph_cli_parser.cpp, - llama.cpp 1242be6 common/arg.cpp.""" + llama.cpp v0.8.0 (b11960) common/arg.cpp.""" kv = kv or {} if eng == "vllm" and place == "cpu": return ["--max-model-len=%d" % ctx, "--max-num-seqs=%d" % seqs, diff --git a/docs/fit.md b/docs/fit.md index 7dc57689..c3ac4c32 100644 --- a/docs/fit.md +++ b/docs/fit.md @@ -108,7 +108,7 @@ llama.cpp keeps the token embeddings and the n-gram table on the CPU by design bytes on every llama.cpp placement. OpenVINO's GPU plugin compiles from a host copy, so an OVMS GPU placement counts the weights as host bytes too. -A sliding-window layer's cache is engine-specific. llama.cpp b11749 gives each slot +A sliding-window layer's cache is engine-specific. llama.cpp v0.8.0 gives each slot `GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch), 256)` cells (`src/llama-kv-cache-iswa.cpp`), so Gemma 4's 1024-token window holds 1536 cells. OpenVINO GenAI 2026.4 has only a full-attention KV cache and a linear-attention state cache @@ -116,7 +116,7 @@ full-attention KV cache and a linear-attention state cache attention layer, so OVMS holds every token in the sliding-window layers too: Gemma 4 costs 225280 B per token there (5 full and 25 sliding layers, K and V at 2 B). -llama-server b11749 keeps two host caches beside the KV cache: a prompt cache of up to +llama-server v0.8.0 keeps two host caches beside the KV cache: a prompt cache of up to 8192 MiB (`--cache-ram`) and up to 32 context checkpoints per slot (`-ctxcp`), each a host copy of the slot's SWA cache and recurrent state (`tools/server/server-context.cpp`, `create_checkpoint`). The fit turns the prompt cache off (`--cache-ram 0`), keeps 4 @@ -126,7 +126,7 @@ checkpoint in the host bytes and the pod limit. A model with no sliding-window a linear-attention layers makes none. On several GPUs llama.cpp places whole layers, so the fit sizes each card from the -layers it gets, the way llama.cpp b11749 assigns them with `-ngl 999` +layers it gets, the way llama.cpp v0.8.0 assigns them with `-ngl 999` (`src/llama-model.cpp`, `load_tensors`). Layer `il` goes to the device `upper_bound(cumsum(ts) / sum(ts), il / (layers + 1))`, where layers is the GGUF's block count (`n_layer_all`): the trunk, then the MTP (nextn) blocks a GGUF holds. Qwen3.8 27B's diff --git a/tests/fit_test.py b/tests/fit_test.py index 550d9ba3..3590fe0a 100755 --- a/tests/fit_test.py +++ b/tests/fit_test.py @@ -270,7 +270,7 @@ def test_swa_cache_per_engine(self): self.assertEqual(full + swa, 225280) # vLLM: the window self.assertEqual(fit.per_seq(fit.kv_profile(cfg, 2, "vllm"), 131072), full * 131072 + swa * 1024) - # llama.cpp b11749: GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch 512), 256) cells per stream + # llama.cpp v0.8.0: GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch 512), 256) cells per stream kv = fit.kv_profile(cfg, 2, "llama.cpp") self.assertEqual(fit.per_seq(kv, 131072), full * 131072 + swa * 1536) self.assertEqual(fit.per_seq(kv, 1024), (full + swa) * 1024) @@ -477,7 +477,7 @@ def test_no_moe_offload_without_amx_or_avx512(self): class LlamaHostCaches(unittest.TestCase): - """llama-server b11749 keeps an 8192 MiB host prompt cache (--cache-ram) and up to 32 + """llama-server v0.8.0 keeps an 8192 MiB host prompt cache (--cache-ram) and up to 32 context checkpoints per slot (-ctxcp), each a slot's SWA cache or recurrent state, in host RAM. The fit turns the prompt cache off, bounds the checkpoints, and counts them.""" @@ -830,7 +830,7 @@ def gguf_nextn(cfg, w): def llama_cards(d, cfg, w, kv_layer): """Each card's GPU bytes for a llama.cpp plan, worked out here from its flags the way - llama.cpp b11749 assigns them (src/llama-model.cpp load_tensors, -ngl past the layer + llama.cpp v0.8.0 assigns them (src/llama-model.cpp load_tensors, -ngl past the layer count): layer il on device upper_bound(cumsum(ts) / sum(ts), il / (n_layer_all + 1)), the output layer (il = n_layer_all) the same way, the experts of the first --n-cpu-moe layers in host RAM, each layer's KV on its device, and each card's own overhead. n_layer_all is @@ -882,7 +882,7 @@ def qwen38_27b_kv(il, ctx): def gemma4_kv(il, ctx): # every 6th layer global: 2 x 2 kv heads x 512 x 2 B per token; the rest a 1024-token window, - # whose llama.cpp b11749 cache is GGML_PAD(min(n_ctx_seq, 1024 + 512 ubatch), 256) cells per + # whose llama.cpp v0.8.0 cache is GGML_PAD(min(n_ctx_seq, 1024 + 512 ubatch), 256) cells per # slot (src/llama-kv-cache-iswa.cpp), n_ctx_seq itself padded to 256 (src/llama-context.cpp) if il % 6 == 5: return 2 * 2 * 512 * 2 * ctx @@ -944,7 +944,7 @@ def test_gemma_layer_split_cards_within_budget(self): def test_mtp_blocks_take_layer_slots(self): # Intel/Qwen3.8-27B-q4km-AutoRound's GGUF holds Qwen3.8's MTP block after its 64-layer # trunk (its header at 4b30b30e: qwen35.block_count 65, qwen35.nextn_predict_layers 1, - # blk.64.nextn.*). llama.cpp b11749 assigns devices over all 65 blocks plus the output + # blk.64.nextn.*). llama.cpp v0.8.0 assigns devices over all 65 blocks plus the output # layer (src/llama-model.cpp, n_layer_all) and gives the MTP block no KV (the cache # covers il < n_layer()), so -ts counts 66 layers: counts summing to 65 put one more # trunk layer on the first card than the fit sized it for. From e49d0f8ec548244f16952bc6d23d506da6cea951 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 17:37:19 -0700 Subject: [PATCH 13/13] agent: K3D_GPU_AGENT_GPUS reaches the fit when the backend is unset and OVMS does not serve the model With K3D_GPU_AGENT_BACKEND unset, AGENT_APP is OVMS's until the fit picks the engine, so agent_fit dropped --gpus for every model the optimizer did not first move to another backend, not just the ones OVMS serves. A model the fit then sent to vLLM or llama.cpp lost the cards asked for: K3D_GPU_AGENT_GPUS=2 with K3D_GPU_AGENT_MODEL set to the QAT GGUF, or with K3D_GPU_MODEL_OPT=off and RedHatAI/Qwen3.8-27B-INT4, rendered one card where main renders two. With the backend set, --gpus is still left out for OVMS alone. With it unset, it is left out for an OpenVINO IR, the only weights the fit gives to OVMS (FORMAT_ENGINE); any other model gets it. The OVMS test from the one-GPU fix still renders one card. New tests (each failed before the change): the INT4 checkpoint with the backend unset on two cards in tests/fit_test.sh, and the QAT GGUF from K3D_GPU_AGENT_MODEL with the backend unset split over two B70s in tests/llamacpp_test.sh. --- docs/fit.md | 5 +++-- scripts/k3d-gpu | 19 ++++++++++++++++--- tests/fit_test.sh | 10 ++++++++++ tests/llamacpp_test.sh | 8 ++++++++ 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/docs/fit.md b/docs/fit.md index c3ac4c32..67419485 100644 --- a/docs/fit.md +++ b/docs/fit.md @@ -164,8 +164,9 @@ The overhead is an estimate from the formula, not a measurement; `agent up` applies them, sets the pod's memory request and limit to the fit, and annotates the pod `k3d-gpu.io/agent-fit`. `K3D_GPU_AGENT_MAX_LEN`, `K3D_GPU_AGENT_MAX_NUM_SEQS` and `K3D_GPU_AGENT_GPUS` are inputs the fit checks, not values that replace it. -`K3D_GPU_AGENT_GPUS` reaches the fit for the vLLM and llama.cpp backends only: an OVMS pod -holds one GPU, and `fit --gpus 2` with OVMS is refused as "OVMS serves one GPU". With +`K3D_GPU_AGENT_GPUS` reaches the fit unless OVMS serves the model (the backend set to OVMS +or, with it unset, an OpenVINO IR): an OVMS pod holds one GPU, and `fit --gpus 2` with OVMS +is refused as "OVMS serves one GPU". With `K3D_GPU_AGENT_BACKEND` unset, the engine the fit picks serves the model: OVMS, vLLM or llama-server. With `--model`, the model optimizer first picks the variant for the default engine (OVMS: Gemma 4's OpenVINO IR, for example). When that engine has nothing to serve, diff --git a/scripts/k3d-gpu b/scripts/k3d-gpu index ce470276..6a15390a 100755 --- a/scripts/k3d-gpu +++ b/scripts/k3d-gpu @@ -11203,8 +11203,16 @@ agent_fit() { [ -z "${K3D_GPU_AGENT_MAX_LEN:-}" ] || args+=(--ctx "${K3D_GPU_AGENT_MAX_LEN}") [ -z "${K3D_GPU_AGENT_MAX_NUM_SEQS:-}" ] || args+=(--seqs "${K3D_GPU_AGENT_MAX_NUM_SEQS}") # K3D_GPU_AGENT_GPUS shards a vLLM model or splits a llama.cpp one; an OVMS pod holds - # one GPU whatever it says. - [ -z "${K3D_GPU_AGENT_GPUS:-}" ] || [ "${AGENT_APP}" = ovms ] || args+=(--gpus "${K3D_GPU_AGENT_GPUS}") + # one GPU whatever it says. OVMS serves the model when the backend is set to it or, with + # the backend unset (the fit picks the engine), when it is an OpenVINO IR: only OVMS + # serves one, and the fit picks vLLM or llama.cpp for any other weights. + if [ -n "${K3D_GPU_AGENT_GPUS:-}" ]; then + if [ -n "${AGENT_BACKEND_SET}" ]; then + [ "${AGENT_APP}" = ovms ] || args+=(--gpus "${K3D_GPU_AGENT_GPUS}") + else + [ "$(fit_dir_format "${dir}")" = openvino ] || args+=(--gpus "${K3D_GPU_AGENT_GPUS}") + fi + fi # agent up replaces the running server (the Deployment is Recreate, and a backend # switch deletes the other one): it is stopped before the probe, so the memory it holds # is measured as free, not as someone else's. A model that then does not fit gets the @@ -11614,7 +11622,12 @@ agent_model_format() { local d command -v python3 >/dev/null 2>&1 || return 0 d=$(fit_model_dir "$1" "${K3D_GPU_AGENT_REVISION:-main}" 2>/dev/null) || return 0 - python3 -c 'import json, sys; print(json.load(open(sys.argv[1])).get("format") or "")' "${d}/weights.json" 2>/dev/null || true + fit_dir_format "${d}" +} + +# fit_dir_format : the weight format a fit metadata directory names, or nothing. +fit_dir_format() { + python3 -c 'import json, sys; print(json.load(open(sys.argv[1])).get("format") or "")' "$1/weights.json" 2>/dev/null || true } # Applies the optimizer's choices for AGENT_OPT_MODEL to the agent settings. diff --git a/tests/fit_test.sh b/tests/fit_test.sh index d06a216e..0c5f58ef 100755 --- a/tests/fit_test.sh +++ b/tests/fit_test.sh @@ -214,6 +214,16 @@ if render ovms2 STUB_ALLOC=$'cpu 64\ngpu.intel.com/xe 4' STUB_NODE_LABELS=gpu.in ok "agent manifest (ovms, K3D_GPU_AGENT_GPUS=2): --cache_size=${cache} fits one card's ${one_card} GiB budget" else fail "agent manifest (ovms, K3D_GPU_AGENT_GPUS=2) sized for one card (cache ${cache:-?}, one card ${one_card})" \ "$(cat "${WORK}/ovms2.err"; grep -E 'cache_size|agent-fit|gpu.intel.com/xe' "${WORK}/ovms2.yaml")"; fi +# With K3D_GPU_AGENT_BACKEND unset the fit picks the engine, and K3D_GPU_AGENT_GPUS still +# reaches it for a model OVMS does not serve: RedHatAI's INT4 checkpoint fits one B70, so +# only the two GPUs asked put it on two (vLLM tensor parallel), as with the backend set. +if render vllm2 STUB_ALLOC=$'cpu 64\ngpu.intel.com/xe 4' STUB_NODE_LABELS=gpu.intel.com/xmx=true K3D_GPU_FIT_HOST="${XEON}" \ + K3D_GPU_MODEL_OPT=off K3D_GPU_AGENT_GPUS=2 -- --model RedHatAI/Qwen3.8-27B-INT4 \ + && grep -q 'fit: RedHatAI/Qwen3.8-27B-INT4 on 2 GPUs, tensor parallel via vllm' "${WORK}/vllm2.err" \ + && has vllm2 '--tensor-parallel-size=2$' && has vllm2 '^ gpu.intel.com/xe: 2$'; then + ok "agent manifest (backend unset, K3D_GPU_AGENT_GPUS=2): the fit's vLLM pick shards over the two GPUs asked" +else fail "agent manifest (backend unset, K3D_GPU_AGENT_GPUS=2) reaches the fit" \ + "$(cat "${WORK}/vllm2.err"; grep -E 'tensor-parallel|agent-fit|gpu.intel.com/xe' "${WORK}/vllm2.yaml")"; fi # OVMS has no sliding-window cache: Gemma 4's 25 SWA layers hold the whole context, so the # static pool holds seqs x context x 225280 B (5 full + 25 SWA layers' K and V at 2 B). diff --git a/tests/llamacpp_test.sh b/tests/llamacpp_test.sh index 982a759f..21fc7efe 100755 --- a/tests/llamacpp_test.sh +++ b/tests/llamacpp_test.sh @@ -387,6 +387,14 @@ if render setl2 "${XEON_ENV[@]}" K3D_GPU_AGENT_BACKEND=llama.cpp K3D_GPU_AGENT_G && grep -A3 '^ limits:$' "${WORK}/setl2.yaml" | grep -q '^ gpu.intel.com/xe: 2$'; then ok "K3D_GPU_AGENT_BACKEND=llama.cpp K3D_GPU_AGENT_GPUS=2: the fit splits the GGUF over two B70s, -ts in layers" else fail "K3D_GPU_AGENT_GPUS=2 reaches the llama.cpp fit" "$(show setl2)"; fi +# With the backend unset and no optimizer (K3D_GPU_AGENT_MODEL, not --model), the backend is +# OVMS's until the fit picks llama.cpp for the GGUF, and K3D_GPU_AGENT_GPUS still reaches the +# fit: only OVMS, which serves an OpenVINO IR alone, holds one GPU whatever it says. +if render unset2 "${XEON_ENV[@]}" K3D_GPU_AGENT_GPUS=2 K3D_GPU_AGENT_MODEL="${GEMMA_GGUF}" -- && has unset2 '^ name: llamacpp$' \ + && grep -q "fit: ${GEMMA_GGUF}:Q4_0 on 2 GPUs, layer split via llama.cpp" "${WORK}/unset2.err" \ + && grep -A3 '^ limits:$' "${WORK}/unset2.yaml" | grep -q '^ gpu.intel.com/xe: 2$'; then + ok "K3D_GPU_AGENT_BACKEND unset, K3D_GPU_AGENT_GPUS=2: the fit's llama.cpp pick splits the GGUF over two B70s" +else fail "K3D_GPU_AGENT_GPUS=2 reaches the fit with the backend unset" "$(show unset2)"; fi # --- MTP on request --------------------------------------------------------------------- if render mtp "${XEON_ENV[@]}" K3D_GPU_AGENT_DRAFT=mtp -- --model "${GEMMA_GGUF}" \