fit: size sliding-window KV per engine

console/fit.py

@@ -117,6 +117,12 @@
 # KV cache types each engine takes and the fit sets a flag for (vLLM --kv-cache-dtype,
 # llama.cpp -ctk/-ctv; OVMS's KV_CACHE_PRECISION is not set by the fit).
 KV_DTYPES_OF = {"vllm": ("fp8", "fp16", "bf16"), "llama.cpp": ("f16", "fp16", "bf16", "q8_0", "q4_0")}
+# Tokens a sliding-window layer's cache holds per sequence (swa_tokens). vLLM: the window.
+# llama.cpp b11749: GGML_PAD(n_swa + n_ubatch, 256) cells per stream (src/llama-kv-cache-iswa.cpp).
+# OVMS: OpenVINO GenAI 2026.4 has only KV_CACHE and LINEAR_ATTENTION_CACHE types
+# (src/cpp/src/continuous_batching/cache/cache_type.hpp), and a KV block spans every
+# attention layer (kv_cache_manager.hpp), so sliding-window layers hold the whole context.
+SWA_CACHE = {"vllm": "window", "llama.cpp": "llama.cpp", "ovms": "full"}
 
 # Tensor classes by name: the parts a placement moves separately.
 CLASSES = (
@@ -784,9 +790,10 @@ def model_type(cfg):
     return str(text_config(cfg).get("model_type") or cfg.get("model_type") or "").replace("_text", "")
 
 
-def kv_profile(cfg, kv_bytes):
+def kv_profile(cfg, kv_bytes, engine=None):
     """Per-token KV bytes for full-attention layers, per-token-within-window bytes for
-    sliding-window layers, and per-sequence state bytes for linear-attention layers."""
+    sliding-window layers, and per-sequence state bytes for linear-attention layers.
+    engine sets how many tokens a sliding-window layer holds (swa_tokens)."""
     tc = text_config(cfg)
     n = tc.get("num_hidden_layers")
     types = tc.get("layer_types") or ["full_attention"] * n
@@ -812,9 +819,16 @@ def kv_profile(cfg, kv_bytes):
                      % (full, tc["indexer_kv_heads"], tc["indexer_head_dim"], _num(kv_bytes), full * idx))
     window = tc.get("sliding_window") or 0
     per_swa = 2 * kvh * hd * kv_bytes
+    swa_cache = SWA_CACHE.get(engine, "window")
     if swa:
-        lines.append("sliding window: %d layers x 2 x %d kv heads x %d head dim x %s B = %d B/token, at most %d tokens"
-                     % (swa, kvh, hd, _num(kv_bytes), swa * per_swa, window))
+        lines.append("sliding window: %d layers x 2 x %d kv heads x %d head dim x %s B = %d B/token, %s"
+                     % (swa, kvh, hd, _num(kv_bytes), swa * per_swa, {
+                         "full": "for the whole context: OpenVINO GenAI 2026.4 has no sliding-window cache type"
+                                 " (continuous_batching/cache/cache_type.hpp), every attention layer gets every token",
+                         "llama.cpp": "for GGML_PAD(%d window + %d ubatch, 256) = %d tokens per sequence at most"
+                                      " (src/llama-kv-cache-iswa.cpp)" % (window, LLAMA_UBATCH,
+                                                                          _pad(window + LLAMA_UBATCH, 256)),
+                         "window": "at most %d tokens" % window}[swa_cache]))
     state = 0
     if lin:
         vh, kd = tc.get("linear_num_value_heads", 0), tc.get("linear_key_head_dim", 0)
@@ -828,10 +842,15 @@ def kv_profile(cfg, kv_bytes):
         lines.append("linear attention state: %d layers x (%d v heads x %d x %d x %d B + conv (%d-1) x %d ch x 2 B)"
                      " = %s per sequence, any length" % (lin, vh, kd, vd, ssm_b, ck, ch, _mb(state)))
     return {"full_layers": full, "full_per_token": full * (per_full + idx), "swa_layers": swa,
-            "swa_per_token": swa * per_swa, "window": window, "linear_layers": lin,
+            "swa_per_token": swa * per_swa, "window": window, "swa_cache": swa_cache, "linear_layers": lin,
             "state_per_seq": state, "kv_heads": gkvh, "lines": lines}
 
 
+def _pad(n, to):
+    """GGML_PAD: n rounded up to a multiple of to."""
+    return (n + to - 1) // to * to
+
+
 def _num(x):
     return ("%g" % x) if isinstance(x, float) else str(x)
 
@@ -966,8 +985,21 @@ def act_bytes(m, tokens, seqs):
     return tokens * width * 2 * 2 + seqs * m["vocab"] * 4
 
 
+def swa_tokens(kv, ctx):
+    """Tokens one sequence's sliding-window layers hold at context ctx. vLLM: the window.
+    llama.cpp: its SWA cache of GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch), 256) cells per
+    stream (one stream per slot with -np set). OVMS: the whole context."""
+    w = kv["window"] or ctx
+    mode = kv.get("swa_cache", "window")
+    if mode == "full":
+        return ctx
+    if mode == "llama.cpp":
+        return min(_pad(ctx, 256), _pad(w + LLAMA_UBATCH, 256))
+    return min(ctx, w)
+
+
 def per_seq(kv, ctx, scale=1.0):
-    return int((kv["full_per_token"] * ctx + kv["swa_per_token"] * min(ctx, kv["window"] or ctx)) * scale
+    return int((kv["full_per_token"] * ctx + kv["swa_per_token"] * swa_tokens(kv, ctx)) * scale
                + kv["state_per_seq"] * scale)
 
 
@@ -1063,7 +1095,7 @@ def fit(host, cfg, weights, engine=None, ctx=None, seqs=None, gpus=None, kv_dtyp
             raise ValueError("--kv-dtype %s is not a KV cache type of %s (%s)"
                              % (kv_dtype, eng, ", ".join(KV_DTYPES_OF.get(eng, ())) or "auto only"))
         kvb = KV_DTYPE_BYTES.get(kv_dtype) or (1 if eng == "vllm" and m["kv_scheme_bits"] == 8 else m["dtype_bytes"])
-        kvs[eng] = kv_profile(cfg, kvb)
+        kvs[eng] = kv_profile(cfg, kvb, eng)
         kvs[eng]["dtype"] = kv_dtype if kv_dtype != "auto" else ("fp8" if kvb == 1 else "auto")
         ex.append("  KV cache (%s B per element%s):" % (_num(kvb), ", the checkpoint's kv_cache_scheme"
                                                        if kvb == 1 and kv_dtype == "auto" else ""))
@@ -1102,7 +1134,7 @@ def fit(host, cfg, weights, engine=None, ctx=None, seqs=None, gpus=None, kv_dtyp
     rung_kv, rung_limit = [], []
     for rs, rc in rungs:
         kb = per_seq(kvp, rc, picked["kv_scale"]) * rs
-        if picked["engine"] == "vllm" and picked["placement"] == "cpu":
+        if _whole_gib_pool(picked["engine"], picked["placement"]):
             kb = int(math.ceil(kb / GIB)) * GIB
         hb = picked["host_bytes"] - picked["kv_bytes"] + kb if picked["placement"] == "cpu" else picked["host_bytes"]
         rung_kv.append(int(kb))
@@ -1151,6 +1183,22 @@ def _res(eng, place, fits, why="", math_lines=(), **kw):
     return r
 
 
+def _whole_gib_pool(eng, place):
+    """KV pools set in whole GiB: OVMS --cache_size, vLLM's CPU VLLM_CPU_KVCACHE_SPACE."""
+    return eng == "ovms" or (eng == "vllm" and place == "cpu")
+
+
+def _kv_room(eng, place, left):
+    """The KV room sequences are sized in: whole GiB for a whole-GiB pool, so the pool
+    rounded up to hold them stays inside left."""
+    return left // GIB * GIB if _whole_gib_pool(eng, place) else left
+
+
+def _kv_pool(eng, place, kvbytes):
+    """The KV bytes the engine sets aside for kvbytes of sequences."""
+    return int(math.ceil(kvbytes / GIB)) * GIB if _whole_gib_pool(eng, place) else kvbytes
+
+
 def _dram_rate(bw, nbytes):
     return bw * 1e9 / nbytes if bw and nbytes > 0 else None
 
@@ -1211,12 +1259,12 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree,
                 yield _res(eng, place, False, "weights + overhead %s exceed the GPU budget %s"
                            % (g1(w + ov * (1 if eng == "vllm" else k)), g1(budget)), lines, gpus=k, tp=k if eng == "vllm" else 1)
                 continue
-            c, n = size_kv(kv, left, ladder, want_seqs, scale, relaxed)
+            c, n = size_kv(kv, _kv_room(eng, place, left), ladder, want_seqs, scale, relaxed)
             if not c:
                 yield _res(eng, place, False, "KV room %s holds no %d-token sequence" % (g1(left), ladder[-1]),
                            lines, gpus=k)
                 continue
-            kvused = per_seq(kv, c, scale) * n
+            kvused = _kv_pool(eng, place, per_seq(kv, c, scale) * n)
             lines.append("KV: %d seqs x %s at %d tokens = %s%s" % (n, _mb(per_seq(kv, c, scale)), c, g1(kvused),
                                                                  " per card" if eng == "vllm" else ""))
             # vLLM runs one worker process per tensor-parallel rank, each with its own runtime.
@@ -1307,13 +1355,11 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree,
             yield _res(eng, "cpu", False, "DRAM bandwidth gives %.1f tok/s, under the %g floor" % (toks, MIN_DRAM_TOK_S),
                        lines)
             return
-        c, n = size_kv(kv, left, ladder, want_seqs, 1.0, relaxed)
+        c, n = size_kv(kv, _kv_room(eng, "cpu", left), ladder, want_seqs, 1.0, relaxed)
         if not c:
             yield _res(eng, "cpu", False, "KV room %s holds no %d-token sequence" % (g1(left), ladder[-1]), lines)
             return
-        kvused = per_seq(kv, c) * n
-        if eng == "vllm":
-            kvused = int(math.ceil(kvused / GIB)) * GIB  # VLLM_CPU_KVCACHE_SPACE is whole GiB
+        kvused = _kv_pool(eng, "cpu", per_seq(kv, c) * n)
         lines.append("KV: %d seqs x %s at %d tokens = %s" % (n, _mb(per_seq(kv, c)), c, g1(kvused)))
         hb = int(m["serve_bytes"] + act + HOST_RUNTIME[eng] + kvused)
         yield _res(eng, "cpu", True, "", lines, ctx=c, seqs=n, host_bytes=hb, kv_bytes=int(kvused),

docs/fit.md

@@ -69,7 +69,10 @@ Per placement:
 weights        = bytes of each tensor class to serve (MTP left out unless the GGUF holds it)
 KV per token   = full layers x 2 x kv heads x head dim x bytes
                + indexer keys (sparse attention)
-               + sliding layers x 2 x kv heads x head dim x bytes, for at most the window
+               + sliding layers x 2 x kv heads x head dim x bytes, for:
+                   vLLM       at most the window
+                   llama.cpp  at most GGML_PAD(window + 512 ubatch, 256) tokens per slot
+                   OVMS       the whole context (OpenVINO GenAI has no sliding-window cache)
 state per seq  = linear layers x (v heads x k dim x v dim x 4 B + conv state)
 overhead (GPU) = runtime + activations (batch x widest matmul x 2 x 2 B) + logits (seqs x vocab x 4 B)
 host reserve   = max(10% of MemTotal, 4 GiB);   host budget = min(MemAvailable, cgroup room) - reserve
@@ -87,6 +90,14 @@ llama.cpp keeps the token embeddings and the n-gram table on the CPU by design
 bytes on every llama.cpp placement. OpenVINO's GPU plugin compiles from a host copy, so
 an OVMS GPU placement counts the weights as host bytes too.
 
+A sliding-window layer's cache is engine-specific. llama.cpp b11749 gives each slot
+`GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch), 256)` cells (`src/llama-kv-cache-iswa.cpp`),
+so Gemma 4's 1024-token window holds 1536 cells. OpenVINO GenAI 2026.4 has only a
+full-attention KV cache and a linear-attention state cache
+(`src/cpp/src/continuous_batching/cache/cache_type.hpp`), and one KV block spans every
+attention layer, so OVMS holds every token in the sliding-window layers too: Gemma 4
+costs 225280 B per token there (5 full and 25 sliding layers, K and V at 2 B).
+
 The overhead is an estimate from the formula, not a measurement;
 `K3D_GPU_FIT_OVERHEAD_GIB` replaces it with a measured value.
 
@@ -95,7 +106,7 @@ The overhead is an estimate from the formula, not a measurement;
 | Engine | Flags (spellings from each engine's own argument parser) |
 | --- | --- |
 | vLLM v0.32.0 | `--tensor-parallel-size`, `--max-model-len`, `--max-num-seqs`, `--gpu-memory-utilization` (vLLM still profiles activations inside that share), `--kv-cache-dtype fp8` when the checkpoint's `kv_cache_scheme` made the fit count one byte. CPU build (EAT): `VLLM_CPU_KVCACHE_SPACE` |
-| OVMS 2026.4.1 | `--max_num_seqs`, `--cache_size` (a static pool: the dynamic one grows past the fit) |
+| OVMS 2026.4.1 | `--max_num_seqs`, `--cache_size` (a static pool in whole GiB, rounded up to hold every sequence: the dynamic one grows past the fit) |
 | llama.cpp | `-c` (context x slots), `-np`, `-ngl`, `-ts`, `--n-cpu-moe` or `--cpu-moe` |
 
 `agent up` applies them, sets the pod's memory request and limit to the fit, and

tests/fit_test.py

@@ -258,6 +258,19 @@ def test_gemma4_hybrid_swa(self):
         self.assertEqual(kv["state_per_seq"], 0)
         self.assertEqual(fit.per_seq(kv, 131072), 5 * 2 * 2 * 512 * 2 * 131072 + 25 * 2 * 8 * 256 * 2 * 1024)
 
+    def test_swa_cache_per_engine(self):
+        cfg = config("google__gemma-4-26B-A4B-it")
+        full, swa = 5 * 2 * 2 * 512 * 2, 25 * 2 * 8 * 256 * 2
+        self.assertEqual(full + swa, 225280)
+        # vLLM: the window
+        self.assertEqual(fit.per_seq(fit.kv_profile(cfg, 2, "vllm"), 131072), full * 131072 + swa * 1024)
+        # llama.cpp b11749: GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch 512), 256) cells per stream
+        kv = fit.kv_profile(cfg, 2, "llama.cpp")
+        self.assertEqual(fit.per_seq(kv, 131072), full * 131072 + swa * 1536)
+        self.assertEqual(fit.per_seq(kv, 1024), (full + swa) * 1024)
+        # OVMS (OpenVINO GenAI 2026.4): no sliding-window cache type, every layer at full context
+        self.assertEqual(fit.per_seq(fit.kv_profile(cfg, 2, "ovms"), 131072), 225280 * 131072)
+
     def test_qwen38_gated_deltanet(self):
         kv = fit.kv_profile(config("Qwen__Qwen3.8-27B"), 2)
         self.assertEqual(kv["full_per_token"], 16 * 2 * 4 * 256 * 2)
@@ -316,10 +329,28 @@ def test_gemma_qat_gguf(self):
         d = plan("xeon6741p-4xb70", "google__gemma-4-26B-A4B-it-qat-q4_0-gguf")
         self.assertTrue(d["fits"], d["summary"])
         self.assertEqual((d["engine"], d["placement"], d["gpus"]), ("llama.cpp", "gpu", 1))
-        self.assertEqual((d["ctx"], d["seqs"]), (131072, 6))
+        # each slot's SWA cache is GGML_PAD(1024 + 512, 256) = 1536 cells, not the 1024 window
+        self.assertEqual((d["ctx"], d["seqs"]), (131072, 5))
         self.assertFalse(d["degraded"])
         self.assertIn("-ngl", d["flags"])
 
+    def test_gemma_ovms_cache_holds_every_token(self):
+        # OVMS gives Gemma 4's 25 sliding-window layers the whole context: 225280 B per token,
+        # and --cache_size (whole GiB) holds context x sequences at that, on every guard rung.
+        one = max(c["budget"] for c in fit.gpu_budget(host("xeon6741p-4xb70"))[0])
+        for hname, dev in (("xeon6741p-4xb70", "gpu"), ("xeon6741p-4xb70", "cpu"), ("desktop-i9-nogpu", None)):
+            d = plan(hname, "OpenVINO__gemma-4-26b-a4b-it-int4-ov", device=dev)
+            self.assertTrue(d["fits"], (hname, d["summary"]))
+            self.assertEqual(d["engine"], "ovms")
+            cache = [int(f.split("=")[1]) for f in d["flags"] if f.startswith("--cache_size=")][0]
+            self.assertGreaterEqual(cache * GIB, d["ctx"] * d["seqs"] * 225280, (hname, d["flags"]))
+            self.assertEqual(cache * GIB, d["kv_bytes"], hname)
+            if d["placement"] == "gpu":
+                self.assertLessEqual(d["gpu_bytes"], one)
+            for r, (s, c) in enumerate(d["guard"]["rungs"]):
+                k = dict(line.split("=", 1) for line in fit.knobs(d, r).splitlines())
+                self.assertGreaterEqual(int(k["cache_gib"]) * GIB, s * c * 225280, (hname, r))
+
     def test_vllm_host_side_per_rank(self):
         d = plan("xeon6741p-4xb70", "Qwen__Qwen3.8-27B")
         # one runtime per tensor-parallel rank, plus the /dev/shm emptyDir

tests/fit_test.sh

@@ -156,6 +156,17 @@ if render ovms STUB_ALLOC='cpu 32' K3D_GPU_FIT_HOST="${DESK}" K3D_GPU_AGENT_MODE
     ok "agent manifest (ovms, CPU): static KV pool, seqs and pod memory limit from the fit"
 else fail "agent manifest (ovms, CPU) from the fit" "$(cat "${WORK}/ovms.err" "${WORK}/ovms.yaml" 2>/dev/null | head -60)"; fi
 
+# OVMS has no sliding-window cache: Gemma 4's 25 SWA layers hold the whole context, so the
+# static pool holds seqs x context x 225280 B (5 full + 25 SWA layers' K and V at 2 B).
+if render ovms-swa STUB_ALLOC=$'cpu 64\ngpu.intel.com/xe 4' STUB_NODE_LABELS=gpu.intel.com/xmx=true K3D_GPU_FIT_HOST="${XEON}" \
+        K3D_GPU_AGENT_MODEL=OpenVINO/gemma-4-26b-a4b-it-int4-ov -- \
+    && cache=$(sed -n 's/^ *- --cache_size=\([0-9]*\)$/\1/p' "${WORK}/ovms-swa.yaml") \
+    && sc=$(sed -n 's/^ *k3d-gpu.io\/agent-fit: "gpu \([0-9]*x[0-9]*\)"$/\1/p' "${WORK}/ovms-swa.yaml") \
+    && [ -n "${cache}" ] && [ -n "${sc}" ] && [ $(( cache * 1073741824 )) -ge $(( ${sc%x*} * ${sc#*x} * 225280 )) ]; then
+    ok "agent manifest (ovms, Gemma 4 on a GPU): --cache_size=${cache} holds ${sc} tokens at 225280 B each"
+else fail "agent manifest (ovms, Gemma 4): --cache_size holds seqs x context at 225280 B/token (cache ${cache:-?}, ${sc:-?})" \
+    "$(cat "${WORK}/ovms-swa.err"; grep -E 'cache_size|agent-fit' "${WORK}/ovms-swa.yaml")"; fi
+
 if render tier STUB_ALLOC='cpu 32' K3D_GPU_FIT_HOST="${LAPTOP}" K3D_GPU_AGENT_MODEL=OpenVINO/Qwen3.8-27B-int8-ov \
         K3D_GPU_AGENT_MODEL_TIER=1 -- && ! has tier 'agent-fit'; then
     ok "a quickstart tier pick (K3D_GPU_AGENT_MODEL_TIER=1) is not sized: it is no user choice"