diff --git a/console/fit.py b/console/fit.py index 41211e77..1db6d4de 100644 --- a/console/fit.py +++ b/console/fit.py @@ -117,6 +117,12 @@ # KV cache types each engine takes and the fit sets a flag for (vLLM --kv-cache-dtype, # llama.cpp -ctk/-ctv; OVMS's KV_CACHE_PRECISION is not set by the fit). KV_DTYPES_OF = {"vllm": ("fp8", "fp16", "bf16"), "llama.cpp": ("f16", "fp16", "bf16", "q8_0", "q4_0")} +# Tokens a sliding-window layer's cache holds per sequence (swa_tokens). vLLM: the window. +# llama.cpp b11749: GGML_PAD(n_swa + n_ubatch, 256) cells per stream (src/llama-kv-cache-iswa.cpp). +# OVMS: OpenVINO GenAI 2026.4 has only KV_CACHE and LINEAR_ATTENTION_CACHE types +# (src/cpp/src/continuous_batching/cache/cache_type.hpp), and a KV block spans every +# attention layer (kv_cache_manager.hpp), so sliding-window layers hold the whole context. +SWA_CACHE = {"vllm": "window", "llama.cpp": "llama.cpp", "ovms": "full"} # Tensor classes by name: the parts a placement moves separately. CLASSES = ( @@ -784,9 +790,10 @@ def model_type(cfg): return str(text_config(cfg).get("model_type") or cfg.get("model_type") or "").replace("_text", "") -def kv_profile(cfg, kv_bytes): +def kv_profile(cfg, kv_bytes, engine=None): """Per-token KV bytes for full-attention layers, per-token-within-window bytes for - sliding-window layers, and per-sequence state bytes for linear-attention layers.""" + sliding-window layers, and per-sequence state bytes for linear-attention layers. + engine sets how many tokens a sliding-window layer holds (swa_tokens).""" tc = text_config(cfg) n = tc.get("num_hidden_layers") types = tc.get("layer_types") or ["full_attention"] * n @@ -812,9 +819,16 @@ def kv_profile(cfg, kv_bytes): % (full, tc["indexer_kv_heads"], tc["indexer_head_dim"], _num(kv_bytes), full * idx)) window = tc.get("sliding_window") or 0 per_swa = 2 * kvh * hd * kv_bytes + swa_cache = SWA_CACHE.get(engine, "window") if swa: - lines.append("sliding window: %d layers x 2 x %d kv heads x %d head dim x %s B = %d B/token, at most %d tokens" - % (swa, kvh, hd, _num(kv_bytes), swa * per_swa, window)) + lines.append("sliding window: %d layers x 2 x %d kv heads x %d head dim x %s B = %d B/token, %s" + % (swa, kvh, hd, _num(kv_bytes), swa * per_swa, { + "full": "for the whole context: OpenVINO GenAI 2026.4 has no sliding-window cache type" + " (continuous_batching/cache/cache_type.hpp), every attention layer gets every token", + "llama.cpp": "for GGML_PAD(%d window + %d ubatch, 256) = %d tokens per sequence at most" + " (src/llama-kv-cache-iswa.cpp)" % (window, LLAMA_UBATCH, + _pad(window + LLAMA_UBATCH, 256)), + "window": "at most %d tokens" % window}[swa_cache])) state = 0 if lin: vh, kd = tc.get("linear_num_value_heads", 0), tc.get("linear_key_head_dim", 0) @@ -828,10 +842,15 @@ def kv_profile(cfg, kv_bytes): lines.append("linear attention state: %d layers x (%d v heads x %d x %d x %d B + conv (%d-1) x %d ch x 2 B)" " = %s per sequence, any length" % (lin, vh, kd, vd, ssm_b, ck, ch, _mb(state))) return {"full_layers": full, "full_per_token": full * (per_full + idx), "swa_layers": swa, - "swa_per_token": swa * per_swa, "window": window, "linear_layers": lin, + "swa_per_token": swa * per_swa, "window": window, "swa_cache": swa_cache, "linear_layers": lin, "state_per_seq": state, "kv_heads": gkvh, "lines": lines} +def _pad(n, to): + """GGML_PAD: n rounded up to a multiple of to.""" + return (n + to - 1) // to * to + + def _num(x): return ("%g" % x) if isinstance(x, float) else str(x) @@ -966,8 +985,21 @@ def act_bytes(m, tokens, seqs): return tokens * width * 2 * 2 + seqs * m["vocab"] * 4 +def swa_tokens(kv, ctx): + """Tokens one sequence's sliding-window layers hold at context ctx. vLLM: the window. + llama.cpp: its SWA cache of GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch), 256) cells per + stream (one stream per slot with -np set). OVMS: the whole context.""" + w = kv["window"] or ctx + mode = kv.get("swa_cache", "window") + if mode == "full": + return ctx + if mode == "llama.cpp": + return min(_pad(ctx, 256), _pad(w + LLAMA_UBATCH, 256)) + return min(ctx, w) + + def per_seq(kv, ctx, scale=1.0): - return int((kv["full_per_token"] * ctx + kv["swa_per_token"] * min(ctx, kv["window"] or ctx)) * scale + return int((kv["full_per_token"] * ctx + kv["swa_per_token"] * swa_tokens(kv, ctx)) * scale + kv["state_per_seq"] * scale) @@ -1063,7 +1095,7 @@ def fit(host, cfg, weights, engine=None, ctx=None, seqs=None, gpus=None, kv_dtyp raise ValueError("--kv-dtype %s is not a KV cache type of %s (%s)" % (kv_dtype, eng, ", ".join(KV_DTYPES_OF.get(eng, ())) or "auto only")) kvb = KV_DTYPE_BYTES.get(kv_dtype) or (1 if eng == "vllm" and m["kv_scheme_bits"] == 8 else m["dtype_bytes"]) - kvs[eng] = kv_profile(cfg, kvb) + kvs[eng] = kv_profile(cfg, kvb, eng) kvs[eng]["dtype"] = kv_dtype if kv_dtype != "auto" else ("fp8" if kvb == 1 else "auto") ex.append(" KV cache (%s B per element%s):" % (_num(kvb), ", the checkpoint's kv_cache_scheme" if kvb == 1 and kv_dtype == "auto" else "")) @@ -1102,7 +1134,7 @@ def fit(host, cfg, weights, engine=None, ctx=None, seqs=None, gpus=None, kv_dtyp rung_kv, rung_limit = [], [] for rs, rc in rungs: kb = per_seq(kvp, rc, picked["kv_scale"]) * rs - if picked["engine"] == "vllm" and picked["placement"] == "cpu": + if _whole_gib_pool(picked["engine"], picked["placement"]): kb = int(math.ceil(kb / GIB)) * GIB hb = picked["host_bytes"] - picked["kv_bytes"] + kb if picked["placement"] == "cpu" else picked["host_bytes"] rung_kv.append(int(kb)) @@ -1151,6 +1183,22 @@ def _res(eng, place, fits, why="", math_lines=(), **kw): return r +def _whole_gib_pool(eng, place): + """KV pools set in whole GiB: OVMS --cache_size, vLLM's CPU VLLM_CPU_KVCACHE_SPACE.""" + return eng == "ovms" or (eng == "vllm" and place == "cpu") + + +def _kv_room(eng, place, left): + """The KV room sequences are sized in: whole GiB for a whole-GiB pool, so the pool + rounded up to hold them stays inside left.""" + return left // GIB * GIB if _whole_gib_pool(eng, place) else left + + +def _kv_pool(eng, place, kvbytes): + """The KV bytes the engine sets aside for kvbytes of sequences.""" + return int(math.ceil(kvbytes / GIB)) * GIB if _whole_gib_pool(eng, place) else kvbytes + + def _dram_rate(bw, nbytes): return bw * 1e9 / nbytes if bw and nbytes > 0 else None @@ -1211,12 +1259,12 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, yield _res(eng, place, False, "weights + overhead %s exceed the GPU budget %s" % (g1(w + ov * (1 if eng == "vllm" else k)), g1(budget)), lines, gpus=k, tp=k if eng == "vllm" else 1) continue - c, n = size_kv(kv, left, ladder, want_seqs, scale, relaxed) + c, n = size_kv(kv, _kv_room(eng, place, left), ladder, want_seqs, scale, relaxed) if not c: yield _res(eng, place, False, "KV room %s holds no %d-token sequence" % (g1(left), ladder[-1]), lines, gpus=k) continue - kvused = per_seq(kv, c, scale) * n + kvused = _kv_pool(eng, place, per_seq(kv, c, scale) * n) lines.append("KV: %d seqs x %s at %d tokens = %s%s" % (n, _mb(per_seq(kv, c, scale)), c, g1(kvused), " per card" if eng == "vllm" else "")) # vLLM runs one worker process per tensor-parallel rank, each with its own runtime. @@ -1307,13 +1355,11 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, yield _res(eng, "cpu", False, "DRAM bandwidth gives %.1f tok/s, under the %g floor" % (toks, MIN_DRAM_TOK_S), lines) return - c, n = size_kv(kv, left, ladder, want_seqs, 1.0, relaxed) + c, n = size_kv(kv, _kv_room(eng, "cpu", left), ladder, want_seqs, 1.0, relaxed) if not c: yield _res(eng, "cpu", False, "KV room %s holds no %d-token sequence" % (g1(left), ladder[-1]), lines) return - kvused = per_seq(kv, c) * n - if eng == "vllm": - kvused = int(math.ceil(kvused / GIB)) * GIB # VLLM_CPU_KVCACHE_SPACE is whole GiB + kvused = _kv_pool(eng, "cpu", per_seq(kv, c) * n) lines.append("KV: %d seqs x %s at %d tokens = %s" % (n, _mb(per_seq(kv, c)), c, g1(kvused))) hb = int(m["serve_bytes"] + act + HOST_RUNTIME[eng] + kvused) yield _res(eng, "cpu", True, "", lines, ctx=c, seqs=n, host_bytes=hb, kv_bytes=int(kvused), diff --git a/docs/fit.md b/docs/fit.md index ab06e5f1..8dbc6b57 100644 --- a/docs/fit.md +++ b/docs/fit.md @@ -69,7 +69,10 @@ Per placement: weights = bytes of each tensor class to serve (MTP left out unless the GGUF holds it) KV per token = full layers x 2 x kv heads x head dim x bytes + indexer keys (sparse attention) - + sliding layers x 2 x kv heads x head dim x bytes, for at most the window + + sliding layers x 2 x kv heads x head dim x bytes, for: + vLLM at most the window + llama.cpp at most GGML_PAD(window + 512 ubatch, 256) tokens per slot + OVMS the whole context (OpenVINO GenAI has no sliding-window cache) state per seq = linear layers x (v heads x k dim x v dim x 4 B + conv state) overhead (GPU) = runtime + activations (batch x widest matmul x 2 x 2 B) + logits (seqs x vocab x 4 B) host reserve = max(10% of MemTotal, 4 GiB); host budget = min(MemAvailable, cgroup room) - reserve @@ -87,6 +90,14 @@ llama.cpp keeps the token embeddings and the n-gram table on the CPU by design bytes on every llama.cpp placement. OpenVINO's GPU plugin compiles from a host copy, so an OVMS GPU placement counts the weights as host bytes too. +A sliding-window layer's cache is engine-specific. llama.cpp b11749 gives each slot +`GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch), 256)` cells (`src/llama-kv-cache-iswa.cpp`), +so Gemma 4's 1024-token window holds 1536 cells. OpenVINO GenAI 2026.4 has only a +full-attention KV cache and a linear-attention state cache +(`src/cpp/src/continuous_batching/cache/cache_type.hpp`), and one KV block spans every +attention layer, so OVMS holds every token in the sliding-window layers too: Gemma 4 +costs 225280 B per token there (5 full and 25 sliding layers, K and V at 2 B). + The overhead is an estimate from the formula, not a measurement; `K3D_GPU_FIT_OVERHEAD_GIB` replaces it with a measured value. @@ -95,7 +106,7 @@ The overhead is an estimate from the formula, not a measurement; | Engine | Flags (spellings from each engine's own argument parser) | | --- | --- | | vLLM v0.32.0 | `--tensor-parallel-size`, `--max-model-len`, `--max-num-seqs`, `--gpu-memory-utilization` (vLLM still profiles activations inside that share), `--kv-cache-dtype fp8` when the checkpoint's `kv_cache_scheme` made the fit count one byte. CPU build (EAT): `VLLM_CPU_KVCACHE_SPACE` | -| OVMS 2026.4.1 | `--max_num_seqs`, `--cache_size` (a static pool: the dynamic one grows past the fit) | +| OVMS 2026.4.1 | `--max_num_seqs`, `--cache_size` (a static pool in whole GiB, rounded up to hold every sequence: the dynamic one grows past the fit) | | llama.cpp | `-c` (context x slots), `-np`, `-ngl`, `-ts`, `--n-cpu-moe` or `--cpu-moe` | `agent up` applies them, sets the pod's memory request and limit to the fit, and diff --git a/tests/fit_test.py b/tests/fit_test.py index 56bf7440..c0016165 100755 --- a/tests/fit_test.py +++ b/tests/fit_test.py @@ -258,6 +258,19 @@ def test_gemma4_hybrid_swa(self): self.assertEqual(kv["state_per_seq"], 0) self.assertEqual(fit.per_seq(kv, 131072), 5 * 2 * 2 * 512 * 2 * 131072 + 25 * 2 * 8 * 256 * 2 * 1024) + def test_swa_cache_per_engine(self): + cfg = config("google__gemma-4-26B-A4B-it") + full, swa = 5 * 2 * 2 * 512 * 2, 25 * 2 * 8 * 256 * 2 + self.assertEqual(full + swa, 225280) + # vLLM: the window + self.assertEqual(fit.per_seq(fit.kv_profile(cfg, 2, "vllm"), 131072), full * 131072 + swa * 1024) + # llama.cpp b11749: GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch 512), 256) cells per stream + kv = fit.kv_profile(cfg, 2, "llama.cpp") + self.assertEqual(fit.per_seq(kv, 131072), full * 131072 + swa * 1536) + self.assertEqual(fit.per_seq(kv, 1024), (full + swa) * 1024) + # OVMS (OpenVINO GenAI 2026.4): no sliding-window cache type, every layer at full context + self.assertEqual(fit.per_seq(fit.kv_profile(cfg, 2, "ovms"), 131072), 225280 * 131072) + def test_qwen38_gated_deltanet(self): kv = fit.kv_profile(config("Qwen__Qwen3.8-27B"), 2) self.assertEqual(kv["full_per_token"], 16 * 2 * 4 * 256 * 2) @@ -316,10 +329,28 @@ def test_gemma_qat_gguf(self): d = plan("xeon6741p-4xb70", "google__gemma-4-26B-A4B-it-qat-q4_0-gguf") self.assertTrue(d["fits"], d["summary"]) self.assertEqual((d["engine"], d["placement"], d["gpus"]), ("llama.cpp", "gpu", 1)) - self.assertEqual((d["ctx"], d["seqs"]), (131072, 6)) + # each slot's SWA cache is GGML_PAD(1024 + 512, 256) = 1536 cells, not the 1024 window + self.assertEqual((d["ctx"], d["seqs"]), (131072, 5)) self.assertFalse(d["degraded"]) self.assertIn("-ngl", d["flags"]) + def test_gemma_ovms_cache_holds_every_token(self): + # OVMS gives Gemma 4's 25 sliding-window layers the whole context: 225280 B per token, + # and --cache_size (whole GiB) holds context x sequences at that, on every guard rung. + one = max(c["budget"] for c in fit.gpu_budget(host("xeon6741p-4xb70"))[0]) + for hname, dev in (("xeon6741p-4xb70", "gpu"), ("xeon6741p-4xb70", "cpu"), ("desktop-i9-nogpu", None)): + d = plan(hname, "OpenVINO__gemma-4-26b-a4b-it-int4-ov", device=dev) + self.assertTrue(d["fits"], (hname, d["summary"])) + self.assertEqual(d["engine"], "ovms") + cache = [int(f.split("=")[1]) for f in d["flags"] if f.startswith("--cache_size=")][0] + self.assertGreaterEqual(cache * GIB, d["ctx"] * d["seqs"] * 225280, (hname, d["flags"])) + self.assertEqual(cache * GIB, d["kv_bytes"], hname) + if d["placement"] == "gpu": + self.assertLessEqual(d["gpu_bytes"], one) + for r, (s, c) in enumerate(d["guard"]["rungs"]): + k = dict(line.split("=", 1) for line in fit.knobs(d, r).splitlines()) + self.assertGreaterEqual(int(k["cache_gib"]) * GIB, s * c * 225280, (hname, r)) + def test_vllm_host_side_per_rank(self): d = plan("xeon6741p-4xb70", "Qwen__Qwen3.8-27B") # one runtime per tensor-parallel rank, plus the /dev/shm emptyDir diff --git a/tests/fit_test.sh b/tests/fit_test.sh index b655baf5..f36b469c 100755 --- a/tests/fit_test.sh +++ b/tests/fit_test.sh @@ -156,6 +156,17 @@ if render ovms STUB_ALLOC='cpu 32' K3D_GPU_FIT_HOST="${DESK}" K3D_GPU_AGENT_MODE ok "agent manifest (ovms, CPU): static KV pool, seqs and pod memory limit from the fit" else fail "agent manifest (ovms, CPU) from the fit" "$(cat "${WORK}/ovms.err" "${WORK}/ovms.yaml" 2>/dev/null | head -60)"; fi +# OVMS has no sliding-window cache: Gemma 4's 25 SWA layers hold the whole context, so the +# static pool holds seqs x context x 225280 B (5 full + 25 SWA layers' K and V at 2 B). +if render ovms-swa STUB_ALLOC=$'cpu 64\ngpu.intel.com/xe 4' STUB_NODE_LABELS=gpu.intel.com/xmx=true K3D_GPU_FIT_HOST="${XEON}" \ + K3D_GPU_AGENT_MODEL=OpenVINO/gemma-4-26b-a4b-it-int4-ov -- \ + && cache=$(sed -n 's/^ *- --cache_size=\([0-9]*\)$/\1/p' "${WORK}/ovms-swa.yaml") \ + && sc=$(sed -n 's/^ *k3d-gpu.io\/agent-fit: "gpu \([0-9]*x[0-9]*\)"$/\1/p' "${WORK}/ovms-swa.yaml") \ + && [ -n "${cache}" ] && [ -n "${sc}" ] && [ $(( cache * 1073741824 )) -ge $(( ${sc%x*} * ${sc#*x} * 225280 )) ]; then + ok "agent manifest (ovms, Gemma 4 on a GPU): --cache_size=${cache} holds ${sc} tokens at 225280 B each" +else fail "agent manifest (ovms, Gemma 4): --cache_size holds seqs x context at 225280 B/token (cache ${cache:-?}, ${sc:-?})" \ + "$(cat "${WORK}/ovms-swa.err"; grep -E 'cache_size|agent-fit' "${WORK}/ovms-swa.yaml")"; fi + if render tier STUB_ALLOC='cpu 32' K3D_GPU_FIT_HOST="${LAPTOP}" K3D_GPU_AGENT_MODEL=OpenVINO/Qwen3.8-27B-int8-ov \ K3D_GPU_AGENT_MODEL_TIER=1 -- && ! has tier 'agent-fit'; then ok "a quickstart tier pick (K3D_GPU_AGENT_MODEL_TIER=1) is not sized: it is no user choice"