diff --git a/console/fit.py b/console/fit.py index 41211e77..9cacae39 100644 --- a/console/fit.py +++ b/console/fit.py @@ -93,6 +93,13 @@ # the agent stack sets it, llama.cpp's default -ub 512). VLLM_BATCHED_TOKENS = 2048 LLAMA_UBATCH = 512 +# llama-server context checkpoints per slot (-ctxcp; b11749 default 32, common/common.h). Each +# is a host-RAM copy of the slot's SWA cache and recurrent state (tools/server/server-context.cpp +# create_checkpoint, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY). A request makes them at the last user +# message and 4 + n_ubatch and 4 tokens before its prompt's end; an appended turn restores from +# those, so four keep the latest request's set. The host prompt cache (--cache-ram, default +# 8192 MiB) is turned off with --cache-ram 0: a slot's KV stays in its own cache. +LLAMA_CTX_CHECKPOINTS = 4 # Fixed runtime per GPU process (device context, allocator pools, collective # buffers) and per pod on the host (Python, torch or the server binary). Estimates. GPU_RUNTIME = 1 * GIB @@ -832,6 +839,11 @@ def kv_profile(cfg, kv_bytes): "state_per_seq": state, "kv_heads": gkvh, "lines": lines} +def _pad(n, to): + """GGML_PAD: n rounded up to a multiple of to.""" + return (n + to - 1) // to * to + + def _num(x): return ("%g" % x) if isinstance(x, float) else str(x) @@ -971,6 +983,15 @@ def per_seq(kv, ctx, scale=1.0): + kv["state_per_seq"] * scale) +def ckpt_seq(kv, ctx): + """Host bytes of one llama-server slot's context checkpoints at context ctx: kv["ckpts"] + of them, each the slot's SWA cache (GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch), 256) + cells, src/llama-kv-cache-iswa.cpp) and recurrent state. A model with neither makes none.""" + w = kv["window"] or ctx + one = kv["swa_per_token"] * min(_pad(ctx, 256), _pad(w + LLAMA_UBATCH, 256)) + kv["state_per_seq"] + return kv.get("ckpts", 0) * one + + def ctx_ladder(max_ctx, want=None): if want: return [min(want, max_ctx)] @@ -981,13 +1002,14 @@ def ctx_ladder(max_ctx, want=None): return out or [max_ctx] -def size_kv(kv, budget, ladder, want_seqs, scale=1.0, relaxed=False): +def size_kv(kv, budget, ladder, want_seqs, scale=1.0, relaxed=False, host_ckpt=False): """(ctx, seqs): the most context at which TARGET_SEQS (or want_seqs) whole sequences fit in budget, then as many sequences as fit at that context (capped at MAX_SEQS). - relaxed: at the ladder's last rung one sequence is enough.""" + relaxed: at the ladder's last rung one sequence is enough. host_ckpt: the budget also + holds each sequence's context checkpoints (KV in host RAM).""" need = want_seqs or TARGET_SEQS for ctx in ladder: - s = per_seq(kv, ctx, scale) + s = per_seq(kv, ctx, scale) + (ckpt_seq(kv, ctx) if host_ckpt else 0) n = int(budget // s) if s else MAX_SEQS if n >= need or (relaxed and ctx == ladder[-1] and n >= 1 and not want_seqs): return ctx, min(want_seqs or n, MAX_SEQS) @@ -1065,9 +1087,14 @@ def fit(host, cfg, weights, engine=None, ctx=None, seqs=None, gpus=None, kv_dtyp kvb = KV_DTYPE_BYTES.get(kv_dtype) or (1 if eng == "vllm" and m["kv_scheme_bits"] == 8 else m["dtype_bytes"]) kvs[eng] = kv_profile(cfg, kvb) kvs[eng]["dtype"] = kv_dtype if kv_dtype != "auto" else ("fp8" if kvb == 1 else "auto") + kvs[eng]["ckpts"] = LLAMA_CTX_CHECKPOINTS if eng == "llama.cpp" else 0 ex.append(" KV cache (%s B per element%s):" % (_num(kvb), ", the checkpoint's kv_cache_scheme" if kvb == 1 and kv_dtype == "auto" else "")) ex += [" " + s for s in kvs[eng]["lines"]] + if eng == "llama.cpp": + ex.append(" host prompt cache off (--cache-ram 0); context checkpoints (-ctxcp %d) in host RAM: " + "%d x the slot's SWA cache and recurrent state = %s per slot at %d tokens" + % (LLAMA_CTX_CHECKPOINTS, LLAMA_CTX_CHECKPOINTS, _mb(ckpt_seq(kvs[eng], full[0])), full[0])) tried, picked, degraded = [], None, False for i, (label, ladder, relaxed) in enumerate(passes): ex.append("placements, %s:" % label) @@ -1104,11 +1131,12 @@ def fit(host, cfg, weights, engine=None, ctx=None, seqs=None, gpus=None, kv_dtyp kb = per_seq(kvp, rc, picked["kv_scale"]) * rs if picked["engine"] == "vllm" and picked["placement"] == "cpu": kb = int(math.ceil(kb / GIB)) * GIB - hb = picked["host_bytes"] - picked["kv_bytes"] + kb if picked["placement"] == "cpu" else picked["host_bytes"] + hb = (picked["host_bytes"] - picked["kv_bytes"] - picked["ckpt_bytes"] + kb + ckpt_seq(kvp, rc) * rs + if picked["placement"] == "cpu" else picked["host_bytes"]) rung_kv.append(int(kb)) rung_limit.append("%dGi" % int(math.ceil(hb * (1 + LIMIT_SLACK) / GIB))) doc.update({k: picked[k] for k in ("engine", "placement", "ctx", "seqs", "tp", "gpus", "gpu_bytes", "host_bytes", - "flags", "dram_tok_s", "note", "kv_bytes")}) + "flags", "dram_tok_s", "note", "kv_bytes", "ckpt_bytes")}) lim = int(math.ceil(picked["host_bytes"] * (1 + LIMIT_SLACK) / GIB)) doc["k8s"] = {"memory_request": "%dGi" % lim, "memory_limit": "%dGi" % lim} doc["guard"] = {"rungs": rungs, "placement": picked["placement"], "some_avg10": GUARD_SOME_AVG10, @@ -1146,7 +1174,7 @@ def _overhead(eng, m, seqs, override): def _res(eng, place, fits, why="", math_lines=(), **kw): r = {"engine": eng, "placement": place, "fits": fits, "why": why, "math": list(math_lines), "ctx": None, "seqs": 0, "tp": 1, "gpus": 0, "gpu_bytes": 0, "host_bytes": 0, "flags": [], "dram_tok_s": None, - "kv_bytes": 0, "kv_scale": 1.0} + "kv_bytes": 0, "kv_scale": 1.0, "ckpt_bytes": 0} r.update(kw) return r @@ -1220,13 +1248,16 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, lines.append("KV: %d seqs x %s at %d tokens = %s%s" % (n, _mb(per_seq(kv, c, scale)), c, g1(kvused), " per card" if eng == "vllm" else "")) # vLLM runs one worker process per tensor-parallel rank, each with its own runtime. - hb = int(hostrt + HOST_RUNTIME[eng] * (k - 1) * (eng == "vllm") + on_host + compile_host) + ck = ckpt_seq(kv, c) * n + if ck: + lines.append("context checkpoints: %d seqs x %s = %s in host RAM" % (n, _mb(ckpt_seq(kv, c)), g1(ck))) + hb = int(hostrt + HOST_RUNTIME[eng] * (k - 1) * (eng == "vllm") + on_host + compile_host + ck) if hb > hfree: yield _res(eng, place, False, "host side %s exceeds the host budget %s" % (g1(hb), g1(hfree)), lines, gpus=k) continue gpu_use = int(w + ov + kvused) if eng == "vllm" else int((w + ov * k + kvused) / k) yield _res(eng, place, True, "", lines, ctx=c, seqs=n, tp=k if eng == "vllm" else 1, gpus=k, - gpu_bytes=gpu_use, host_bytes=hb, kv_bytes=int(kvused), kv_scale=scale, + gpu_bytes=gpu_use, host_bytes=hb, kv_bytes=int(kvused), kv_scale=scale, ckpt_bytes=ck, flags=_flags(eng, place, c, n, k, gpu_use, use, kvused, m, 0, kv)) # MoE experts to DRAM (llama.cpp --n-cpu-moe): every free card, fewest CPU layers. if eng == "llama.cpp" and cards and m["moe"]: @@ -1272,9 +1303,10 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, % (bw, g1(dram_w), m["experts_per_tok"], m["experts"], toks, MIN_DRAM_TOK_S)) kvused = per_seq(kv, c) * n lines.append("KV: %d seqs x %s at %d tokens = %s on GPU" % (n, _mb(per_seq(kv, c)), c, g1(kvused))) - hb = int(dram_w + on_host + HOST_RUNTIME[eng]) - lines.append("host: experts %s + lookup tables %s + runtime %s = %s" - % (g1(dram_w), g1(on_host), g1(HOST_RUNTIME[eng]), g1(hb))) + ck = ckpt_seq(kv, c) * n + hb = int(dram_w + on_host + HOST_RUNTIME[eng] + ck) + lines.append("host: experts %s + lookup tables %s + runtime %s + context checkpoints %d x %s = %s" + % (g1(dram_w), g1(on_host), g1(HOST_RUNTIME[eng]), n, _mb(ckpt_seq(kv, c)), g1(hb))) if toks < MIN_DRAM_TOK_S: yield _res(eng, "moe-offload", False, "DRAM bandwidth gives %.1f tok/s, under the %g floor" % (toks, MIN_DRAM_TOK_S), lines, gpus=k) @@ -1284,7 +1316,7 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, else: gpu_use = int((gw + ov * k + kvused) / k) yield _res(eng, "moe-offload", True, "", lines, ctx=c, seqs=n, gpus=k, gpu_bytes=gpu_use, - host_bytes=hb, dram_tok_s=round(toks, 1), kv_bytes=int(kvused), + host_bytes=hb, dram_tok_s=round(toks, 1), kv_bytes=int(kvused), ckpt_bytes=ck, flags=_flags(eng, "moe-offload", c, n, k, gpu_use, use, kvused, m, n_cpu, kv)) # CPU only: weights, KV and activations all in DRAM (vLLM: its CPU build, the KV # pool sized by VLLM_CPU_KVCACHE_SPACE, as EAT's vLLM chart sets it). @@ -1295,8 +1327,9 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, if m["moe"]: active -= m["bytes"]["experts"] * (1 - m["experts_per_tok"] / m["experts"]) toks = _dram_rate(bw, active) - lines = ["host budget / %.2f slack %s - weights %s - activations %s - runtime %s = %s for KV" - % (1 + LIMIT_SLACK, g1(hfree), g1(m["serve_bytes"]), g1(act), g1(HOST_RUNTIME[eng]), g1(max(left, 0)))] + lines = ["host budget / %.2f slack %s - weights %s - activations %s - runtime %s = %s for KV%s" + % (1 + LIMIT_SLACK, g1(hfree), g1(m["serve_bytes"]), g1(act), g1(HOST_RUNTIME[eng]), g1(max(left, 0)), + " and context checkpoints" if kv.get("ckpts") else "")] if toks: lines.append("DRAM decode bound: %g GB/s / %s read per token = %.1f tok/s" % (bw, g1(active), toks)) if left <= 0: @@ -1307,7 +1340,7 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, yield _res(eng, "cpu", False, "DRAM bandwidth gives %.1f tok/s, under the %g floor" % (toks, MIN_DRAM_TOK_S), lines) return - c, n = size_kv(kv, left, ladder, want_seqs, 1.0, relaxed) + c, n = size_kv(kv, left, ladder, want_seqs, 1.0, relaxed, host_ckpt=True) if not c: yield _res(eng, "cpu", False, "KV room %s holds no %d-token sequence" % (g1(left), ladder[-1]), lines) return @@ -1315,8 +1348,11 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, if eng == "vllm": kvused = int(math.ceil(kvused / GIB)) * GIB # VLLM_CPU_KVCACHE_SPACE is whole GiB lines.append("KV: %d seqs x %s at %d tokens = %s" % (n, _mb(per_seq(kv, c)), c, g1(kvused))) - hb = int(m["serve_bytes"] + act + HOST_RUNTIME[eng] + kvused) - yield _res(eng, "cpu", True, "", lines, ctx=c, seqs=n, host_bytes=hb, kv_bytes=int(kvused), + ck = ckpt_seq(kv, c) * n + if ck: + lines.append("context checkpoints: %d seqs x %s = %s" % (n, _mb(ckpt_seq(kv, c)), g1(ck))) + hb = int(m["serve_bytes"] + act + HOST_RUNTIME[eng] + kvused + ck) + yield _res(eng, "cpu", True, "", lines, ctx=c, seqs=n, host_bytes=hb, kv_bytes=int(kvused), ckpt_bytes=ck, dram_tok_s=round(toks, 1) if toks else None, flags=_flags(eng, "cpu", c, n, 1, 0, [], kvused, m, 0, kv)) @@ -1347,6 +1383,8 @@ def _flags(eng, place, ctx, seqs, k, gpu_use, cards, kvused, m, n_cpu_moe, kv=No out += ["-ts", ",".join(str(max(int(c["budget"] // GIB), 1)) for c in cards)] if place == "moe-offload": out += ["--cpu-moe"] if n_cpu_moe >= m["layers"] else ["--n-cpu-moe", str(n_cpu_moe)] + # -cram/--cache-ram, -ctxcp/--ctx-checkpoints: the host caches the fit counts (LLAMA_CTX_CHECKPOINTS) + out += ["--cache-ram", "0", "-ctxcp", str(LLAMA_CTX_CHECKPOINTS)] t = {"fp16": "f16"}.get(kv.get("dtype"), kv.get("dtype")) if t in ("f16", "bf16", "q8_0", "q4_0"): out += ["-ctk", t, "-ctv", t] # -ctk/--cache-type-k, -ctv/--cache-type-v diff --git a/docs/fit.md b/docs/fit.md index ab06e5f1..3f8c7254 100644 --- a/docs/fit.md +++ b/docs/fit.md @@ -71,6 +71,7 @@ KV per token = full layers x 2 x kv heads x head dim x bytes + indexer keys (sparse attention) + sliding layers x 2 x kv heads x head dim x bytes, for at most the window state per seq = linear layers x (v heads x k dim x v dim x 4 B + conv state) +checkpoints = llama.cpp: 4 per slot in host RAM, each the slot's SWA cache and state overhead (GPU) = runtime + activations (batch x widest matmul x 2 x 2 B) + logits (seqs x vocab x 4 B) host reserve = max(10% of MemTotal, 4 GiB); host budget = min(MemAvailable, cgroup room) - reserve GPU reserve = max(5% of VRAM, 1 GiB); GPU budget = VRAM - in use by others - reserve @@ -87,6 +88,15 @@ llama.cpp keeps the token embeddings and the n-gram table on the CPU by design bytes on every llama.cpp placement. OpenVINO's GPU plugin compiles from a host copy, so an OVMS GPU placement counts the weights as host bytes too. +llama-server b11749 keeps two host caches beside the KV cache: a prompt cache of up to +8192 MiB (`--cache-ram`) and up to 32 context checkpoints per slot (`-ctxcp`), each a +host copy of the slot's SWA cache and recurrent state (`tools/server/server-context.cpp`, +`create_checkpoint`). The fit turns the prompt cache off (`--cache-ram 0`), keeps 4 +checkpoints per slot (a request makes them at the last user message and just before its +prompt's end, which is what the next turn restores from), and counts slots x 4 x one +checkpoint in the host bytes and the pod limit. A model with no sliding-window and no +linear-attention layers makes none. + The overhead is an estimate from the formula, not a measurement; `K3D_GPU_FIT_OVERHEAD_GIB` replaces it with a measured value. @@ -96,7 +106,7 @@ The overhead is an estimate from the formula, not a measurement; | --- | --- | | vLLM v0.32.0 | `--tensor-parallel-size`, `--max-model-len`, `--max-num-seqs`, `--gpu-memory-utilization` (vLLM still profiles activations inside that share), `--kv-cache-dtype fp8` when the checkpoint's `kv_cache_scheme` made the fit count one byte. CPU build (EAT): `VLLM_CPU_KVCACHE_SPACE` | | OVMS 2026.4.1 | `--max_num_seqs`, `--cache_size` (a static pool: the dynamic one grows past the fit) | -| llama.cpp | `-c` (context x slots), `-np`, `-ngl`, `-ts`, `--n-cpu-moe` or `--cpu-moe` | +| llama.cpp | `-c` (context x slots), `-np`, `-ngl`, `-ts`, `--n-cpu-moe` or `--cpu-moe`, `--cache-ram 0`, `-ctxcp 4` | `agent up` applies them, sets the pod's memory request and limit to the fit, and annotates the pod `k3d-gpu.io/agent-fit`. `K3D_GPU_AGENT_MAX_LEN`, `K3D_GPU_AGENT_MAX_NUM_SEQS` diff --git a/tests/fit_test.py b/tests/fit_test.py index 56bf7440..6bb3b4fb 100755 --- a/tests/fit_test.py +++ b/tests/fit_test.py @@ -377,6 +377,58 @@ def test_no_moe_offload_without_amx_or_avx512(self): self.assertIn("exceed the host budget", d["reason"]) +class LlamaHostCaches(unittest.TestCase): + """llama-server b11749 keeps an 8192 MiB host prompt cache (--cache-ram) and up to 32 + context checkpoints per slot (-ctxcp), each a slot's SWA cache or recurrent state, in + host RAM. The fit turns the prompt cache off, bounds the checkpoints, and counts them.""" + + def flag(self, d, name): + i = d["flags"].index(name) + return int(d["flags"][i + 1]) + + def test_gemma_qat_cpu_on_the_desktop(self): + d = plan("desktop-i9-nogpu", "google__gemma-4-26B-A4B-it-qat-q4_0-gguf") + self.assertTrue(d["fits"], d["summary"]) + self.assertEqual((d["engine"], d["placement"]), ("llama.cpp", "cpu")) + self.assertEqual(self.flag(d, "--cache-ram"), 0) + k = self.flag(d, "-ctxcp") + self.assertGreaterEqual(k, 1) + # one checkpoint: 25 SWA layers x 2 x 8 kv heads x 256 x 2 B over the slot's SWA + # cache, GGML_PAD(1024 window + 512 ubatch, 256) = 1536 cells + ckpt = 25 * 2 * 8 * 256 * 2 * 1536 + self.assertGreaterEqual(d["ckpt_bytes"], d["seqs"] * k * ckpt) + cfg = config("google__gemma-4-26B-A4B-it-qat-q4_0-gguf") + w = json.load(open(os.path.join(mdir("google__gemma-4-26B-A4B-it-qat-q4_0-gguf"), "weights.json"))) + m = fit.model_profile(cfg, w) + self.assertGreaterEqual(d["host_bytes"], m["serve_bytes"] + fit.HOST_RUNTIME["llama.cpp"] + d["kv_bytes"] + + d["seqs"] * k * ckpt) + # the pod limit holds the checkpoints and stays inside the host budget + budget, _, _ = fit.host_budget(host("desktop-i9-nogpu")) + self.assertGreaterEqual(int(d["k8s"]["memory_limit"][:-2]) * GIB, d["host_bytes"]) + self.assertLessEqual(int(d["k8s"]["memory_limit"][:-2]) * GIB, budget) + # and on every guard rung + for r, (rs, rc) in enumerate(d["guard"]["rungs"]): + self.assertGreaterEqual(int(d["guard"]["rung_memory_limit"][r][:-2]) * GIB, + d["host_bytes"] - d["kv_bytes"] - d["ckpt_bytes"] + + d["guard"]["rung_kv_bytes"][r] + rs * k * ckpt, r) + + def test_recurrent_state_checkpoints_on_the_host(self): + # Flash-Next: 36 gated-DeltaNet layers, each checkpoint the per-sequence state + d = plan("xeon6741p-4xb70", "ggml-org__Qwen3.8-Flash-Next-GGUF") + self.assertTrue(d["fits"], d["summary"]) + k = self.flag(d, "-ctxcp") + state = 36 * (48 * 128 * 128 * 4 + 3 * (2 * 16 * 128 + 48 * 128) * 2) + self.assertEqual(d["ckpt_bytes"], d["seqs"] * k * state) + self.assertIn("--cache-ram", d["flags"]) + + def test_full_attention_makes_no_checkpoints(self): + cfg = config("Qwen__Qwen3.8-27B") + cfg = dict(cfg, text_config=dict(cfg.get("text_config") or cfg, layer_types=None)) + kv = fit.kv_profile(cfg, 2) + kv["ckpts"] = 4 + self.assertEqual(fit.ckpt_seq(kv, 65536), 0) + + class FitLaptop(unittest.TestCase): """Core Ultra 7 258V, 16 GB, 9 GB available, swap in use.""" diff --git a/tests/fit_test.sh b/tests/fit_test.sh index b655baf5..355a1a3c 100755 --- a/tests/fit_test.sh +++ b/tests/fit_test.sh @@ -69,6 +69,23 @@ if [ "${rc}" -eq 0 ] && python3 -c 'import json, sys; d = json.loads(sys.argv[1] ok "fit --json: a fit/v1 document, from a local model directory" else fail "fit --json: a fit/v1 document" "${out}"; fi +# llama-server keeps a host prompt cache (8192 MiB by default) and context checkpoints per +# slot (32 by default, each the slot's SWA cache): the fit turns the first off, bounds the +# second, and counts the checkpoints in the host bytes and the pod limit. +out=$(bash "${K3D_GPU}" fit --host "${DESK}" --model google/gemma-4-26B-A4B-it-qat-q4_0-gguf --json 2>&1); rc=$? +if [ "${rc}" -eq 0 ] && python3 -c ' +import json, sys +d = json.loads(sys.argv[1]) +f = d["flags"] +assert d["placement"] == "cpu" and f[f.index("--cache-ram") + 1] == "0", f +k = int(f[f.index("-ctxcp") + 1]) +ckpt = 25 * 2 * 8 * 256 * 2 * 1536 # 25 SWA layers K and V at 2 B over 1536 cells +assert d["ckpt_bytes"] >= d["seqs"] * k * ckpt > 0, (d["ckpt_bytes"], d["seqs"], k) +assert int(d["k8s"]["memory_limit"][:-2]) * 2 ** 30 >= d["host_bytes"] >= d["kv_bytes"] + d["ckpt_bytes"] +' "${out}"; then + ok "fit (llama.cpp, Gemma QAT on the desktop CPU): --cache-ram 0, -ctxcp K, checkpoints in the pod limit" +else fail "fit (llama.cpp host caches) on the desktop (rc ${rc})" "${out}"; fi + out=$(bash "${K3D_GPU}" fit --host "${XEON}" --model not/in-cache-offline 2>&1