fit: count llama-server's host caches

console/fit.py

@@ -93,6 +93,13 @@
 # the agent stack sets it, llama.cpp's default -ub 512).
 VLLM_BATCHED_TOKENS = 2048
 LLAMA_UBATCH = 512
+# llama-server context checkpoints per slot (-ctxcp; b11749 default 32, common/common.h). Each
+# is a host-RAM copy of the slot's SWA cache and recurrent state (tools/server/server-context.cpp
+# create_checkpoint, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY). A request makes them at the last user
+# message and 4 + n_ubatch and 4 tokens before its prompt's end; an appended turn restores from
+# those, so four keep the latest request's set. The host prompt cache (--cache-ram, default
+# 8192 MiB) is turned off with --cache-ram 0: a slot's KV stays in its own cache.
+LLAMA_CTX_CHECKPOINTS = 4
 # Fixed runtime per GPU process (device context, allocator pools, collective
 # buffers) and per pod on the host (Python, torch or the server binary). Estimates.
 GPU_RUNTIME = 1 * GIB
@@ -832,6 +839,11 @@ def kv_profile(cfg, kv_bytes):
             "state_per_seq": state, "kv_heads": gkvh, "lines": lines}
 
 
+def _pad(n, to):
+    """GGML_PAD: n rounded up to a multiple of to."""
+    return (n + to - 1) // to * to
+
+
 def _num(x):
     return ("%g" % x) if isinstance(x, float) else str(x)
 
@@ -971,6 +983,15 @@ def per_seq(kv, ctx, scale=1.0):
                + kv["state_per_seq"] * scale)
 
 
+def ckpt_seq(kv, ctx):
+    """Host bytes of one llama-server slot's context checkpoints at context ctx: kv["ckpts"]
+    of them, each the slot's SWA cache (GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch), 256)
+    cells, src/llama-kv-cache-iswa.cpp) and recurrent state. A model with neither makes none."""
+    w = kv["window"] or ctx
+    one = kv["swa_per_token"] * min(_pad(ctx, 256), _pad(w + LLAMA_UBATCH, 256)) + kv["state_per_seq"]
+    return kv.get("ckpts", 0) * one
+
+
 def ctx_ladder(max_ctx, want=None):
     if want:
         return [min(want, max_ctx)]
@@ -981,13 +1002,14 @@ def ctx_ladder(max_ctx, want=None):
     return out or [max_ctx]
 
 
-def size_kv(kv, budget, ladder, want_seqs, scale=1.0, relaxed=False):
+def size_kv(kv, budget, ladder, want_seqs, scale=1.0, relaxed=False, host_ckpt=False):
     """(ctx, seqs): the most context at which TARGET_SEQS (or want_seqs) whole sequences fit
     in budget, then as many sequences as fit at that context (capped at MAX_SEQS).
-    relaxed: at the ladder's last rung one sequence is enough."""
+    relaxed: at the ladder's last rung one sequence is enough. host_ckpt: the budget also
+    holds each sequence's context checkpoints (KV in host RAM)."""
     need = want_seqs or TARGET_SEQS
     for ctx in ladder:
-        s = per_seq(kv, ctx, scale)
+        s = per_seq(kv, ctx, scale) + (ckpt_seq(kv, ctx) if host_ckpt else 0)
         n = int(budget // s) if s else MAX_SEQS
         if n >= need or (relaxed and ctx == ladder[-1] and n >= 1 and not want_seqs):
             return ctx, min(want_seqs or n, MAX_SEQS)
@@ -1065,9 +1087,14 @@ def fit(host, cfg, weights, engine=None, ctx=None, seqs=None, gpus=None, kv_dtyp
         kvb = KV_DTYPE_BYTES.get(kv_dtype) or (1 if eng == "vllm" and m["kv_scheme_bits"] == 8 else m["dtype_bytes"])
         kvs[eng] = kv_profile(cfg, kvb)
         kvs[eng]["dtype"] = kv_dtype if kv_dtype != "auto" else ("fp8" if kvb == 1 else "auto")
+        kvs[eng]["ckpts"] = LLAMA_CTX_CHECKPOINTS if eng == "llama.cpp" else 0
         ex.append("  KV cache (%s B per element%s):" % (_num(kvb), ", the checkpoint's kv_cache_scheme"
                                                        if kvb == 1 and kv_dtype == "auto" else ""))
         ex += ["    " + s for s in kvs[eng]["lines"]]
+        if eng == "llama.cpp":
+            ex.append("    host prompt cache off (--cache-ram 0); context checkpoints (-ctxcp %d) in host RAM: "
+                      "%d x the slot's SWA cache and recurrent state = %s per slot at %d tokens"
+                      % (LLAMA_CTX_CHECKPOINTS, LLAMA_CTX_CHECKPOINTS, _mb(ckpt_seq(kvs[eng], full[0])), full[0]))
     tried, picked, degraded = [], None, False
     for i, (label, ladder, relaxed) in enumerate(passes):
         ex.append("placements, %s:" % label)
@@ -1104,11 +1131,12 @@ def fit(host, cfg, weights, engine=None, ctx=None, seqs=None, gpus=None, kv_dtyp
         kb = per_seq(kvp, rc, picked["kv_scale"]) * rs
         if picked["engine"] == "vllm" and picked["placement"] == "cpu":
             kb = int(math.ceil(kb / GIB)) * GIB
-        hb = picked["host_bytes"] - picked["kv_bytes"] + kb if picked["placement"] == "cpu" else picked["host_bytes"]
+        hb = (picked["host_bytes"] - picked["kv_bytes"] - picked["ckpt_bytes"] + kb + ckpt_seq(kvp, rc) * rs
+              if picked["placement"] == "cpu" else picked["host_bytes"])
         rung_kv.append(int(kb))
         rung_limit.append("%dGi" % int(math.ceil(hb * (1 + LIMIT_SLACK) / GIB)))
     doc.update({k: picked[k] for k in ("engine", "placement", "ctx", "seqs", "tp", "gpus", "gpu_bytes", "host_bytes",
-                                       "flags", "dram_tok_s", "note", "kv_bytes")})
+                                       "flags", "dram_tok_s", "note", "kv_bytes", "ckpt_bytes")})
     lim = int(math.ceil(picked["host_bytes"] * (1 + LIMIT_SLACK) / GIB))
     doc["k8s"] = {"memory_request": "%dGi" % lim, "memory_limit": "%dGi" % lim}
     doc["guard"] = {"rungs": rungs, "placement": picked["placement"], "some_avg10": GUARD_SOME_AVG10,
@@ -1146,7 +1174,7 @@ def _overhead(eng, m, seqs, override):
 def _res(eng, place, fits, why="", math_lines=(), **kw):
     r = {"engine": eng, "placement": place, "fits": fits, "why": why, "math": list(math_lines),
          "ctx": None, "seqs": 0, "tp": 1, "gpus": 0, "gpu_bytes": 0, "host_bytes": 0, "flags": [], "dram_tok_s": None,
-         "kv_bytes": 0, "kv_scale": 1.0}
+         "kv_bytes": 0, "kv_scale": 1.0, "ckpt_bytes": 0}
     r.update(kw)
     return r
 
@@ -1220,13 +1248,16 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree,
             lines.append("KV: %d seqs x %s at %d tokens = %s%s" % (n, _mb(per_seq(kv, c, scale)), c, g1(kvused),
                                                                  " per card" if eng == "vllm" else ""))
             # vLLM runs one worker process per tensor-parallel rank, each with its own runtime.
-            hb = int(hostrt + HOST_RUNTIME[eng] * (k - 1) * (eng == "vllm") + on_host + compile_host)
+            ck = ckpt_seq(kv, c) * n
+            if ck:
+                lines.append("context checkpoints: %d seqs x %s = %s in host RAM" % (n, _mb(ckpt_seq(kv, c)), g1(ck)))
+            hb = int(hostrt + HOST_RUNTIME[eng] * (k - 1) * (eng == "vllm") + on_host + compile_host + ck)
             if hb > hfree:
                 yield _res(eng, place, False, "host side %s exceeds the host budget %s" % (g1(hb), g1(hfree)), lines, gpus=k)
                 continue
             gpu_use = int(w + ov + kvused) if eng == "vllm" else int((w + ov * k + kvused) / k)
             yield _res(eng, place, True, "", lines, ctx=c, seqs=n, tp=k if eng == "vllm" else 1, gpus=k,
-                       gpu_bytes=gpu_use, host_bytes=hb, kv_bytes=int(kvused), kv_scale=scale,
+                       gpu_bytes=gpu_use, host_bytes=hb, kv_bytes=int(kvused), kv_scale=scale, ckpt_bytes=ck,
                        flags=_flags(eng, place, c, n, k, gpu_use, use, kvused, m, 0, kv))
     # MoE experts to DRAM (llama.cpp --n-cpu-moe): every free card, fewest CPU layers.
     if eng == "llama.cpp" and cards and m["moe"]:
@@ -1272,9 +1303,10 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree,
                              % (bw, g1(dram_w), m["experts_per_tok"], m["experts"], toks, MIN_DRAM_TOK_S))
                 kvused = per_seq(kv, c) * n
                 lines.append("KV: %d seqs x %s at %d tokens = %s on GPU" % (n, _mb(per_seq(kv, c)), c, g1(kvused)))
-                hb = int(dram_w + on_host + HOST_RUNTIME[eng])
-                lines.append("host: experts %s + lookup tables %s + runtime %s = %s"
-                             % (g1(dram_w), g1(on_host), g1(HOST_RUNTIME[eng]), g1(hb)))
+                ck = ckpt_seq(kv, c) * n
+                hb = int(dram_w + on_host + HOST_RUNTIME[eng] + ck)
+                lines.append("host: experts %s + lookup tables %s + runtime %s + context checkpoints %d x %s = %s"
+                             % (g1(dram_w), g1(on_host), g1(HOST_RUNTIME[eng]), n, _mb(ckpt_seq(kv, c)), g1(hb)))
                 if toks < MIN_DRAM_TOK_S:
                     yield _res(eng, "moe-offload", False, "DRAM bandwidth gives %.1f tok/s, under the %g floor"
                                % (toks, MIN_DRAM_TOK_S), lines, gpus=k)
@@ -1284,7 +1316,7 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree,
                 else:
                     gpu_use = int((gw + ov * k + kvused) / k)
                     yield _res(eng, "moe-offload", True, "", lines, ctx=c, seqs=n, gpus=k, gpu_bytes=gpu_use,
-                               host_bytes=hb, dram_tok_s=round(toks, 1), kv_bytes=int(kvused),
+                               host_bytes=hb, dram_tok_s=round(toks, 1), kv_bytes=int(kvused), ckpt_bytes=ck,
                                flags=_flags(eng, "moe-offload", c, n, k, gpu_use, use, kvused, m, n_cpu, kv))
     # CPU only: weights, KV and activations all in DRAM (vLLM: its CPU build, the KV
     # pool sized by VLLM_CPU_KVCACHE_SPACE, as EAT's vLLM chart sets it).
@@ -1295,8 +1327,9 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree,
         if m["moe"]:
             active -= m["bytes"]["experts"] * (1 - m["experts_per_tok"] / m["experts"])
         toks = _dram_rate(bw, active)
-        lines = ["host budget / %.2f slack %s - weights %s - activations %s - runtime %s = %s for KV"
-                 % (1 + LIMIT_SLACK, g1(hfree), g1(m["serve_bytes"]), g1(act), g1(HOST_RUNTIME[eng]), g1(max(left, 0)))]
+        lines = ["host budget / %.2f slack %s - weights %s - activations %s - runtime %s = %s for KV%s"
+                 % (1 + LIMIT_SLACK, g1(hfree), g1(m["serve_bytes"]), g1(act), g1(HOST_RUNTIME[eng]), g1(max(left, 0)),
+                    " and context checkpoints" if kv.get("ckpts") else "")]
         if toks:
             lines.append("DRAM decode bound: %g GB/s / %s read per token = %.1f tok/s" % (bw, g1(active), toks))
         if left <= 0:
@@ -1307,16 +1340,19 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree,
             yield _res(eng, "cpu", False, "DRAM bandwidth gives %.1f tok/s, under the %g floor" % (toks, MIN_DRAM_TOK_S),
                        lines)
             return
-        c, n = size_kv(kv, left, ladder, want_seqs, 1.0, relaxed)
+        c, n = size_kv(kv, left, ladder, want_seqs, 1.0, relaxed, host_ckpt=True)
         if not c:
             yield _res(eng, "cpu", False, "KV room %s holds no %d-token sequence" % (g1(left), ladder[-1]), lines)
             return
         kvused = per_seq(kv, c) * n
         if eng == "vllm":
             kvused = int(math.ceil(kvused / GIB)) * GIB  # VLLM_CPU_KVCACHE_SPACE is whole GiB
         lines.append("KV: %d seqs x %s at %d tokens = %s" % (n, _mb(per_seq(kv, c)), c, g1(kvused)))
-        hb = int(m["serve_bytes"] + act + HOST_RUNTIME[eng] + kvused)
-        yield _res(eng, "cpu", True, "", lines, ctx=c, seqs=n, host_bytes=hb, kv_bytes=int(kvused),
+        ck = ckpt_seq(kv, c) * n
+        if ck:
+            lines.append("context checkpoints: %d seqs x %s = %s" % (n, _mb(ckpt_seq(kv, c)), g1(ck)))
+        hb = int(m["serve_bytes"] + act + HOST_RUNTIME[eng] + kvused + ck)
+        yield _res(eng, "cpu", True, "", lines, ctx=c, seqs=n, host_bytes=hb, kv_bytes=int(kvused), ckpt_bytes=ck,
                    dram_tok_s=round(toks, 1) if toks else None,
                    flags=_flags(eng, "cpu", c, n, 1, 0, [], kvused, m, 0, kv))
 
@@ -1347,6 +1383,8 @@ def _flags(eng, place, ctx, seqs, k, gpu_use, cards, kvused, m, n_cpu_moe, kv=No
         out += ["-ts", ",".join(str(max(int(c["budget"] // GIB), 1)) for c in cards)]
     if place == "moe-offload":
         out += ["--cpu-moe"] if n_cpu_moe >= m["layers"] else ["--n-cpu-moe", str(n_cpu_moe)]
+    # -cram/--cache-ram, -ctxcp/--ctx-checkpoints: the host caches the fit counts (LLAMA_CTX_CHECKPOINTS)
+    out += ["--cache-ram", "0", "-ctxcp", str(LLAMA_CTX_CHECKPOINTS)]
     t = {"fp16": "f16"}.get(kv.get("dtype"), kv.get("dtype"))
     if t in ("f16", "bf16", "q8_0", "q4_0"):
         out += ["-ctk", t, "-ctv", t]  # -ctk/--cache-type-k, -ctv/--cache-type-v

docs/fit.md

@@ -71,6 +71,7 @@ KV per token   = full layers x 2 x kv heads x head dim x bytes
                + indexer keys (sparse attention)
                + sliding layers x 2 x kv heads x head dim x bytes, for at most the window
 state per seq  = linear layers x (v heads x k dim x v dim x 4 B + conv state)
+checkpoints    = llama.cpp: 4 per slot in host RAM, each the slot's SWA cache and state
 overhead (GPU) = runtime + activations (batch x widest matmul x 2 x 2 B) + logits (seqs x vocab x 4 B)
 host reserve   = max(10% of MemTotal, 4 GiB);   host budget = min(MemAvailable, cgroup room) - reserve
 GPU reserve    = max(5% of VRAM, 1 GiB);        GPU budget  = VRAM - in use by others - reserve
@@ -87,6 +88,15 @@ llama.cpp keeps the token embeddings and the n-gram table on the CPU by design
 bytes on every llama.cpp placement. OpenVINO's GPU plugin compiles from a host copy, so
 an OVMS GPU placement counts the weights as host bytes too.
 
+llama-server b11749 keeps two host caches beside the KV cache: a prompt cache of up to
+8192 MiB (`--cache-ram`) and up to 32 context checkpoints per slot (`-ctxcp`), each a
+host copy of the slot's SWA cache and recurrent state (`tools/server/server-context.cpp`,
+`create_checkpoint`). The fit turns the prompt cache off (`--cache-ram 0`), keeps 4
+checkpoints per slot (a request makes them at the last user message and just before its
+prompt's end, which is what the next turn restores from), and counts slots x 4 x one
+checkpoint in the host bytes and the pod limit. A model with no sliding-window and no
+linear-attention layers makes none.
+
 The overhead is an estimate from the formula, not a measurement;
 `K3D_GPU_FIT_OVERHEAD_GIB` replaces it with a measured value.
 
@@ -96,7 +106,7 @@ The overhead is an estimate from the formula, not a measurement;
 | --- | --- |
 | vLLM v0.32.0 | `--tensor-parallel-size`, `--max-model-len`, `--max-num-seqs`, `--gpu-memory-utilization` (vLLM still profiles activations inside that share), `--kv-cache-dtype fp8` when the checkpoint's `kv_cache_scheme` made the fit count one byte. CPU build (EAT): `VLLM_CPU_KVCACHE_SPACE` |
 | OVMS 2026.4.1 | `--max_num_seqs`, `--cache_size` (a static pool: the dynamic one grows past the fit) |
-| llama.cpp | `-c` (context x slots), `-np`, `-ngl`, `-ts`, `--n-cpu-moe` or `--cpu-moe` |
+| llama.cpp | `-c` (context x slots), `-np`, `-ngl`, `-ts`, `--n-cpu-moe` or `--cpu-moe`, `--cache-ram 0`, `-ctxcp 4` |
 
 `agent up` applies them, sets the pod's memory request and limit to the fit, and
 annotates the pod `k3d-gpu.io/agent-fit`. `K3D_GPU_AGENT_MAX_LEN`, `K3D_GPU_AGENT_MAX_NUM_SEQS`

tests/fit_test.py

@@ -377,6 +377,58 @@ def test_no_moe_offload_without_amx_or_avx512(self):
         self.assertIn("exceed the host budget", d["reason"])
 
 
+class LlamaHostCaches(unittest.TestCase):
+    """llama-server b11749 keeps an 8192 MiB host prompt cache (--cache-ram) and up to 32
+    context checkpoints per slot (-ctxcp), each a slot's SWA cache or recurrent state, in
+    host RAM. The fit turns the prompt cache off, bounds the checkpoints, and counts them."""
+
+    def flag(self, d, name):
+        i = d["flags"].index(name)
+        return int(d["flags"][i + 1])
+
+    def test_gemma_qat_cpu_on_the_desktop(self):
+        d = plan("desktop-i9-nogpu", "google__gemma-4-26B-A4B-it-qat-q4_0-gguf")
+        self.assertTrue(d["fits"], d["summary"])
+        self.assertEqual((d["engine"], d["placement"]), ("llama.cpp", "cpu"))
+        self.assertEqual(self.flag(d, "--cache-ram"), 0)
+        k = self.flag(d, "-ctxcp")
+        self.assertGreaterEqual(k, 1)
+        # one checkpoint: 25 SWA layers x 2 x 8 kv heads x 256 x 2 B over the slot's SWA
+        # cache, GGML_PAD(1024 window + 512 ubatch, 256) = 1536 cells
+        ckpt = 25 * 2 * 8 * 256 * 2 * 1536
+        self.assertGreaterEqual(d["ckpt_bytes"], d["seqs"] * k * ckpt)
+        cfg = config("google__gemma-4-26B-A4B-it-qat-q4_0-gguf")
+        w = json.load(open(os.path.join(mdir("google__gemma-4-26B-A4B-it-qat-q4_0-gguf"), "weights.json")))
+        m = fit.model_profile(cfg, w)
+        self.assertGreaterEqual(d["host_bytes"], m["serve_bytes"] + fit.HOST_RUNTIME["llama.cpp"] + d["kv_bytes"]
+                                + d["seqs"] * k * ckpt)
+        # the pod limit holds the checkpoints and stays inside the host budget
+        budget, _, _ = fit.host_budget(host("desktop-i9-nogpu"))
+        self.assertGreaterEqual(int(d["k8s"]["memory_limit"][:-2]) * GIB, d["host_bytes"])
+        self.assertLessEqual(int(d["k8s"]["memory_limit"][:-2]) * GIB, budget)
+        # and on every guard rung
+        for r, (rs, rc) in enumerate(d["guard"]["rungs"]):
+            self.assertGreaterEqual(int(d["guard"]["rung_memory_limit"][r][:-2]) * GIB,
+                                    d["host_bytes"] - d["kv_bytes"] - d["ckpt_bytes"]
+                                    + d["guard"]["rung_kv_bytes"][r] + rs * k * ckpt, r)
+
+    def test_recurrent_state_checkpoints_on_the_host(self):
+        # Flash-Next: 36 gated-DeltaNet layers, each checkpoint the per-sequence state
+        d = plan("xeon6741p-4xb70", "ggml-org__Qwen3.8-Flash-Next-GGUF")
+        self.assertTrue(d["fits"], d["summary"])
+        k = self.flag(d, "-ctxcp")
+        state = 36 * (48 * 128 * 128 * 4 + 3 * (2 * 16 * 128 + 48 * 128) * 2)
+        self.assertEqual(d["ckpt_bytes"], d["seqs"] * k * state)
+        self.assertIn("--cache-ram", d["flags"])
+
+    def test_full_attention_makes_no_checkpoints(self):
+        cfg = config("Qwen__Qwen3.8-27B")
+        cfg = dict(cfg, text_config=dict(cfg.get("text_config") or cfg, layer_types=None))
+        kv = fit.kv_profile(cfg, 2)
+        kv["ckpts"] = 4
+        self.assertEqual(fit.ckpt_seq(kv, 65536), 0)
+
+
 class FitLaptop(unittest.TestCase):
     """Core Ultra 7 258V, 16 GB, 9 GB available, swap in use."""
 

tests/fit_test.sh

@@ -69,6 +69,23 @@ if [ "${rc}" -eq 0 ] && python3 -c 'import json, sys; d = json.loads(sys.argv[1]
     ok "fit --json: a fit/v1 document, from a local model directory"
 else fail "fit --json: a fit/v1 document" "${out}"; fi
 
+# llama-server keeps a host prompt cache (8192 MiB by default) and context checkpoints per
+# slot (32 by default, each the slot's SWA cache): the fit turns the first off, bounds the
+# second, and counts the checkpoints in the host bytes and the pod limit.
+out=$(bash "${K3D_GPU}" fit --host "${DESK}" --model google/gemma-4-26B-A4B-it-qat-q4_0-gguf --json 2>&1); rc=$?
+if [ "${rc}" -eq 0 ] && python3 -c '
+import json, sys
+d = json.loads(sys.argv[1])
+f = d["flags"]
+assert d["placement"] == "cpu" and f[f.index("--cache-ram") + 1] == "0", f
+k = int(f[f.index("-ctxcp") + 1])
+ckpt = 25 * 2 * 8 * 256 * 2 * 1536  # 25 SWA layers K and V at 2 B over 1536 cells
+assert d["ckpt_bytes"] >= d["seqs"] * k * ckpt > 0, (d["ckpt_bytes"], d["seqs"], k)
+assert int(d["k8s"]["memory_limit"][:-2]) * 2 ** 30 >= d["host_bytes"] >= d["kv_bytes"] + d["ckpt_bytes"]
+' "${out}"; then
+    ok "fit (llama.cpp, Gemma QAT on the desktop CPU): --cache-ram 0, -ctxcp K, checkpoints in the pod limit"
+else fail "fit (llama.cpp host caches) on the desktop (rc ${rc})" "${out}"; fi
+
 out=$(bash "${K3D_GPU}" fit --host "${XEON}" --model not/in-cache-offline 2>&1 </dev/null); rc=$?
 if [ "${rc}" -ne 0 ] && grep -q 'no metadata for not/in-cache-offline' <<< "${out}"; then
     ok "fit: a model with no metadata is an error, not a guess"