From 71cce2965895419a81eeb77e10fe5dd9d85463d7 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Wed, 28 Oct 2026 22:49:18 -0700 Subject: [PATCH] fit: count llama-server's host caches llama-server b11749 keeps two host caches beside the KV cache that the fit did not count: a prompt cache of up to 8192 MiB (--cache-ram) and up to 32 context checkpoints per slot (-ctxcp), each a host copy of the slot's SWA cache and recurrent state. For the Gemma QAT GGUF on the desktop CPU plan (5 slots at 262144, KV filling the room) that is 8 GiB plus about 47 GiB of checkpoints over a 45Gi pod limit. The llama.cpp flags now carry --cache-ram 0 and -ctxcp 4, and the fit counts slots x 4 x one checkpoint (the SWA cache of GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch), 256) cells plus the recurrent state) in the host bytes, the pod limit and every guard rung. On the CPU placement the checkpoints share the KV room, so the slot count accounts for them; on the GPU and experts-in-DRAM placements they are host bytes. A model with neither sliding-window nor linear-attention layers makes none. Tests: the Gemma QAT CPU plan on desktop-i9-nogpu emits --cache-ram 0 and -ctxcp K and counts seqs x K x 300 MiB in the host bytes, the pod limit and each rung's limit; Flash-Next's checkpoints are its recurrent state; a full-attention model makes none; k3d-gpu fit --json on the desktop shows the same. --- console/fit.py | 72 ++++++++++++++++++++++++++++++++++++----------- docs/fit.md | 12 +++++++- tests/fit_test.py | 52 ++++++++++++++++++++++++++++++++++ tests/fit_test.sh | 17 +++++++++++ 4 files changed, 135 insertions(+), 18 deletions(-) diff --git a/console/fit.py b/console/fit.py index 41211e77..9cacae39 100644 --- a/console/fit.py +++ b/console/fit.py @@ -93,6 +93,13 @@ # the agent stack sets it, llama.cpp's default -ub 512). VLLM_BATCHED_TOKENS = 2048 LLAMA_UBATCH = 512 +# llama-server context checkpoints per slot (-ctxcp; b11749 default 32, common/common.h). Each +# is a host-RAM copy of the slot's SWA cache and recurrent state (tools/server/server-context.cpp +# create_checkpoint, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY). A request makes them at the last user +# message and 4 + n_ubatch and 4 tokens before its prompt's end; an appended turn restores from +# those, so four keep the latest request's set. The host prompt cache (--cache-ram, default +# 8192 MiB) is turned off with --cache-ram 0: a slot's KV stays in its own cache. +LLAMA_CTX_CHECKPOINTS = 4 # Fixed runtime per GPU process (device context, allocator pools, collective # buffers) and per pod on the host (Python, torch or the server binary). Estimates. GPU_RUNTIME = 1 * GIB @@ -832,6 +839,11 @@ def kv_profile(cfg, kv_bytes): "state_per_seq": state, "kv_heads": gkvh, "lines": lines} +def _pad(n, to): + """GGML_PAD: n rounded up to a multiple of to.""" + return (n + to - 1) // to * to + + def _num(x): return ("%g" % x) if isinstance(x, float) else str(x) @@ -971,6 +983,15 @@ def per_seq(kv, ctx, scale=1.0): + kv["state_per_seq"] * scale) +def ckpt_seq(kv, ctx): + """Host bytes of one llama-server slot's context checkpoints at context ctx: kv["ckpts"] + of them, each the slot's SWA cache (GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch), 256) + cells, src/llama-kv-cache-iswa.cpp) and recurrent state. A model with neither makes none.""" + w = kv["window"] or ctx + one = kv["swa_per_token"] * min(_pad(ctx, 256), _pad(w + LLAMA_UBATCH, 256)) + kv["state_per_seq"] + return kv.get("ckpts", 0) * one + + def ctx_ladder(max_ctx, want=None): if want: return [min(want, max_ctx)] @@ -981,13 +1002,14 @@ def ctx_ladder(max_ctx, want=None): return out or [max_ctx] -def size_kv(kv, budget, ladder, want_seqs, scale=1.0, relaxed=False): +def size_kv(kv, budget, ladder, want_seqs, scale=1.0, relaxed=False, host_ckpt=False): """(ctx, seqs): the most context at which TARGET_SEQS (or want_seqs) whole sequences fit in budget, then as many sequences as fit at that context (capped at MAX_SEQS). - relaxed: at the ladder's last rung one sequence is enough.""" + relaxed: at the ladder's last rung one sequence is enough. host_ckpt: the budget also + holds each sequence's context checkpoints (KV in host RAM).""" need = want_seqs or TARGET_SEQS for ctx in ladder: - s = per_seq(kv, ctx, scale) + s = per_seq(kv, ctx, scale) + (ckpt_seq(kv, ctx) if host_ckpt else 0) n = int(budget // s) if s else MAX_SEQS if n >= need or (relaxed and ctx == ladder[-1] and n >= 1 and not want_seqs): return ctx, min(want_seqs or n, MAX_SEQS) @@ -1065,9 +1087,14 @@ def fit(host, cfg, weights, engine=None, ctx=None, seqs=None, gpus=None, kv_dtyp kvb = KV_DTYPE_BYTES.get(kv_dtype) or (1 if eng == "vllm" and m["kv_scheme_bits"] == 8 else m["dtype_bytes"]) kvs[eng] = kv_profile(cfg, kvb) kvs[eng]["dtype"] = kv_dtype if kv_dtype != "auto" else ("fp8" if kvb == 1 else "auto") + kvs[eng]["ckpts"] = LLAMA_CTX_CHECKPOINTS if eng == "llama.cpp" else 0 ex.append(" KV cache (%s B per element%s):" % (_num(kvb), ", the checkpoint's kv_cache_scheme" if kvb == 1 and kv_dtype == "auto" else "")) ex += [" " + s for s in kvs[eng]["lines"]] + if eng == "llama.cpp": + ex.append(" host prompt cache off (--cache-ram 0); context checkpoints (-ctxcp %d) in host RAM: " + "%d x the slot's SWA cache and recurrent state = %s per slot at %d tokens" + % (LLAMA_CTX_CHECKPOINTS, LLAMA_CTX_CHECKPOINTS, _mb(ckpt_seq(kvs[eng], full[0])), full[0])) tried, picked, degraded = [], None, False for i, (label, ladder, relaxed) in enumerate(passes): ex.append("placements, %s:" % label) @@ -1104,11 +1131,12 @@ def fit(host, cfg, weights, engine=None, ctx=None, seqs=None, gpus=None, kv_dtyp kb = per_seq(kvp, rc, picked["kv_scale"]) * rs if picked["engine"] == "vllm" and picked["placement"] == "cpu": kb = int(math.ceil(kb / GIB)) * GIB - hb = picked["host_bytes"] - picked["kv_bytes"] + kb if picked["placement"] == "cpu" else picked["host_bytes"] + hb = (picked["host_bytes"] - picked["kv_bytes"] - picked["ckpt_bytes"] + kb + ckpt_seq(kvp, rc) * rs + if picked["placement"] == "cpu" else picked["host_bytes"]) rung_kv.append(int(kb)) rung_limit.append("%dGi" % int(math.ceil(hb * (1 + LIMIT_SLACK) / GIB))) doc.update({k: picked[k] for k in ("engine", "placement", "ctx", "seqs", "tp", "gpus", "gpu_bytes", "host_bytes", - "flags", "dram_tok_s", "note", "kv_bytes")}) + "flags", "dram_tok_s", "note", "kv_bytes", "ckpt_bytes")}) lim = int(math.ceil(picked["host_bytes"] * (1 + LIMIT_SLACK) / GIB)) doc["k8s"] = {"memory_request": "%dGi" % lim, "memory_limit": "%dGi" % lim} doc["guard"] = {"rungs": rungs, "placement": picked["placement"], "some_avg10": GUARD_SOME_AVG10, @@ -1146,7 +1174,7 @@ def _overhead(eng, m, seqs, override): def _res(eng, place, fits, why="", math_lines=(), **kw): r = {"engine": eng, "placement": place, "fits": fits, "why": why, "math": list(math_lines), "ctx": None, "seqs": 0, "tp": 1, "gpus": 0, "gpu_bytes": 0, "host_bytes": 0, "flags": [], "dram_tok_s": None, - "kv_bytes": 0, "kv_scale": 1.0} + "kv_bytes": 0, "kv_scale": 1.0, "ckpt_bytes": 0} r.update(kw) return r @@ -1220,13 +1248,16 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, lines.append("KV: %d seqs x %s at %d tokens = %s%s" % (n, _mb(per_seq(kv, c, scale)), c, g1(kvused), " per card" if eng == "vllm" else "")) # vLLM runs one worker process per tensor-parallel rank, each with its own runtime. - hb = int(hostrt + HOST_RUNTIME[eng] * (k - 1) * (eng == "vllm") + on_host + compile_host) + ck = ckpt_seq(kv, c) * n + if ck: + lines.append("context checkpoints: %d seqs x %s = %s in host RAM" % (n, _mb(ckpt_seq(kv, c)), g1(ck))) + hb = int(hostrt + HOST_RUNTIME[eng] * (k - 1) * (eng == "vllm") + on_host + compile_host + ck) if hb > hfree: yield _res(eng, place, False, "host side %s exceeds the host budget %s" % (g1(hb), g1(hfree)), lines, gpus=k) continue gpu_use = int(w + ov + kvused) if eng == "vllm" else int((w + ov * k + kvused) / k) yield _res(eng, place, True, "", lines, ctx=c, seqs=n, tp=k if eng == "vllm" else 1, gpus=k, - gpu_bytes=gpu_use, host_bytes=hb, kv_bytes=int(kvused), kv_scale=scale, + gpu_bytes=gpu_use, host_bytes=hb, kv_bytes=int(kvused), kv_scale=scale, ckpt_bytes=ck, flags=_flags(eng, place, c, n, k, gpu_use, use, kvused, m, 0, kv)) # MoE experts to DRAM (llama.cpp --n-cpu-moe): every free card, fewest CPU layers. if eng == "llama.cpp" and cards and m["moe"]: @@ -1272,9 +1303,10 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, % (bw, g1(dram_w), m["experts_per_tok"], m["experts"], toks, MIN_DRAM_TOK_S)) kvused = per_seq(kv, c) * n lines.append("KV: %d seqs x %s at %d tokens = %s on GPU" % (n, _mb(per_seq(kv, c)), c, g1(kvused))) - hb = int(dram_w + on_host + HOST_RUNTIME[eng]) - lines.append("host: experts %s + lookup tables %s + runtime %s = %s" - % (g1(dram_w), g1(on_host), g1(HOST_RUNTIME[eng]), g1(hb))) + ck = ckpt_seq(kv, c) * n + hb = int(dram_w + on_host + HOST_RUNTIME[eng] + ck) + lines.append("host: experts %s + lookup tables %s + runtime %s + context checkpoints %d x %s = %s" + % (g1(dram_w), g1(on_host), g1(HOST_RUNTIME[eng]), n, _mb(ckpt_seq(kv, c)), g1(hb))) if toks < MIN_DRAM_TOK_S: yield _res(eng, "moe-offload", False, "DRAM bandwidth gives %.1f tok/s, under the %g floor" % (toks, MIN_DRAM_TOK_S), lines, gpus=k) @@ -1284,7 +1316,7 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, else: gpu_use = int((gw + ov * k + kvused) / k) yield _res(eng, "moe-offload", True, "", lines, ctx=c, seqs=n, gpus=k, gpu_bytes=gpu_use, - host_bytes=hb, dram_tok_s=round(toks, 1), kv_bytes=int(kvused), + host_bytes=hb, dram_tok_s=round(toks, 1), kv_bytes=int(kvused), ckpt_bytes=ck, flags=_flags(eng, "moe-offload", c, n, k, gpu_use, use, kvused, m, n_cpu, kv)) # CPU only: weights, KV and activations all in DRAM (vLLM: its CPU build, the KV # pool sized by VLLM_CPU_KVCACHE_SPACE, as EAT's vLLM chart sets it). @@ -1295,8 +1327,9 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, if m["moe"]: active -= m["bytes"]["experts"] * (1 - m["experts_per_tok"] / m["experts"]) toks = _dram_rate(bw, active) - lines = ["host budget / %.2f slack %s - weights %s - activations %s - runtime %s = %s for KV" - % (1 + LIMIT_SLACK, g1(hfree), g1(m["serve_bytes"]), g1(act), g1(HOST_RUNTIME[eng]), g1(max(left, 0)))] + lines = ["host budget / %.2f slack %s - weights %s - activations %s - runtime %s = %s for KV%s" + % (1 + LIMIT_SLACK, g1(hfree), g1(m["serve_bytes"]), g1(act), g1(HOST_RUNTIME[eng]), g1(max(left, 0)), + " and context checkpoints" if kv.get("ckpts") else "")] if toks: lines.append("DRAM decode bound: %g GB/s / %s read per token = %.1f tok/s" % (bw, g1(active), toks)) if left <= 0: @@ -1307,7 +1340,7 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, yield _res(eng, "cpu", False, "DRAM bandwidth gives %.1f tok/s, under the %g floor" % (toks, MIN_DRAM_TOK_S), lines) return - c, n = size_kv(kv, left, ladder, want_seqs, 1.0, relaxed) + c, n = size_kv(kv, left, ladder, want_seqs, 1.0, relaxed, host_ckpt=True) if not c: yield _res(eng, "cpu", False, "KV room %s holds no %d-token sequence" % (g1(left), ladder[-1]), lines) return @@ -1315,8 +1348,11 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, if eng == "vllm": kvused = int(math.ceil(kvused / GIB)) * GIB # VLLM_CPU_KVCACHE_SPACE is whole GiB lines.append("KV: %d seqs x %s at %d tokens = %s" % (n, _mb(per_seq(kv, c)), c, g1(kvused))) - hb = int(m["serve_bytes"] + act + HOST_RUNTIME[eng] + kvused) - yield _res(eng, "cpu", True, "", lines, ctx=c, seqs=n, host_bytes=hb, kv_bytes=int(kvused), + ck = ckpt_seq(kv, c) * n + if ck: + lines.append("context checkpoints: %d seqs x %s = %s" % (n, _mb(ckpt_seq(kv, c)), g1(ck))) + hb = int(m["serve_bytes"] + act + HOST_RUNTIME[eng] + kvused + ck) + yield _res(eng, "cpu", True, "", lines, ctx=c, seqs=n, host_bytes=hb, kv_bytes=int(kvused), ckpt_bytes=ck, dram_tok_s=round(toks, 1) if toks else None, flags=_flags(eng, "cpu", c, n, 1, 0, [], kvused, m, 0, kv)) @@ -1347,6 +1383,8 @@ def _flags(eng, place, ctx, seqs, k, gpu_use, cards, kvused, m, n_cpu_moe, kv=No out += ["-ts", ",".join(str(max(int(c["budget"] // GIB), 1)) for c in cards)] if place == "moe-offload": out += ["--cpu-moe"] if n_cpu_moe >= m["layers"] else ["--n-cpu-moe", str(n_cpu_moe)] + # -cram/--cache-ram, -ctxcp/--ctx-checkpoints: the host caches the fit counts (LLAMA_CTX_CHECKPOINTS) + out += ["--cache-ram", "0", "-ctxcp", str(LLAMA_CTX_CHECKPOINTS)] t = {"fp16": "f16"}.get(kv.get("dtype"), kv.get("dtype")) if t in ("f16", "bf16", "q8_0", "q4_0"): out += ["-ctk", t, "-ctv", t] # -ctk/--cache-type-k, -ctv/--cache-type-v diff --git a/docs/fit.md b/docs/fit.md index ab06e5f1..3f8c7254 100644 --- a/docs/fit.md +++ b/docs/fit.md @@ -71,6 +71,7 @@ KV per token = full layers x 2 x kv heads x head dim x bytes + indexer keys (sparse attention) + sliding layers x 2 x kv heads x head dim x bytes, for at most the window state per seq = linear layers x (v heads x k dim x v dim x 4 B + conv state) +checkpoints = llama.cpp: 4 per slot in host RAM, each the slot's SWA cache and state overhead (GPU) = runtime + activations (batch x widest matmul x 2 x 2 B) + logits (seqs x vocab x 4 B) host reserve = max(10% of MemTotal, 4 GiB); host budget = min(MemAvailable, cgroup room) - reserve GPU reserve = max(5% of VRAM, 1 GiB); GPU budget = VRAM - in use by others - reserve @@ -87,6 +88,15 @@ llama.cpp keeps the token embeddings and the n-gram table on the CPU by design bytes on every llama.cpp placement. OpenVINO's GPU plugin compiles from a host copy, so an OVMS GPU placement counts the weights as host bytes too. +llama-server b11749 keeps two host caches beside the KV cache: a prompt cache of up to +8192 MiB (`--cache-ram`) and up to 32 context checkpoints per slot (`-ctxcp`), each a +host copy of the slot's SWA cache and recurrent state (`tools/server/server-context.cpp`, +`create_checkpoint`). The fit turns the prompt cache off (`--cache-ram 0`), keeps 4 +checkpoints per slot (a request makes them at the last user message and just before its +prompt's end, which is what the next turn restores from), and counts slots x 4 x one +checkpoint in the host bytes and the pod limit. A model with no sliding-window and no +linear-attention layers makes none. + The overhead is an estimate from the formula, not a measurement; `K3D_GPU_FIT_OVERHEAD_GIB` replaces it with a measured value. @@ -96,7 +106,7 @@ The overhead is an estimate from the formula, not a measurement; | --- | --- | | vLLM v0.32.0 | `--tensor-parallel-size`, `--max-model-len`, `--max-num-seqs`, `--gpu-memory-utilization` (vLLM still profiles activations inside that share), `--kv-cache-dtype fp8` when the checkpoint's `kv_cache_scheme` made the fit count one byte. CPU build (EAT): `VLLM_CPU_KVCACHE_SPACE` | | OVMS 2026.4.1 | `--max_num_seqs`, `--cache_size` (a static pool: the dynamic one grows past the fit) | -| llama.cpp | `-c` (context x slots), `-np`, `-ngl`, `-ts`, `--n-cpu-moe` or `--cpu-moe` | +| llama.cpp | `-c` (context x slots), `-np`, `-ngl`, `-ts`, `--n-cpu-moe` or `--cpu-moe`, `--cache-ram 0`, `-ctxcp 4` | `agent up` applies them, sets the pod's memory request and limit to the fit, and annotates the pod `k3d-gpu.io/agent-fit`. `K3D_GPU_AGENT_MAX_LEN`, `K3D_GPU_AGENT_MAX_NUM_SEQS` diff --git a/tests/fit_test.py b/tests/fit_test.py index 56bf7440..6bb3b4fb 100755 --- a/tests/fit_test.py +++ b/tests/fit_test.py @@ -377,6 +377,58 @@ def test_no_moe_offload_without_amx_or_avx512(self): self.assertIn("exceed the host budget", d["reason"]) +class LlamaHostCaches(unittest.TestCase): + """llama-server b11749 keeps an 8192 MiB host prompt cache (--cache-ram) and up to 32 + context checkpoints per slot (-ctxcp), each a slot's SWA cache or recurrent state, in + host RAM. The fit turns the prompt cache off, bounds the checkpoints, and counts them.""" + + def flag(self, d, name): + i = d["flags"].index(name) + return int(d["flags"][i + 1]) + + def test_gemma_qat_cpu_on_the_desktop(self): + d = plan("desktop-i9-nogpu", "google__gemma-4-26B-A4B-it-qat-q4_0-gguf") + self.assertTrue(d["fits"], d["summary"]) + self.assertEqual((d["engine"], d["placement"]), ("llama.cpp", "cpu")) + self.assertEqual(self.flag(d, "--cache-ram"), 0) + k = self.flag(d, "-ctxcp") + self.assertGreaterEqual(k, 1) + # one checkpoint: 25 SWA layers x 2 x 8 kv heads x 256 x 2 B over the slot's SWA + # cache, GGML_PAD(1024 window + 512 ubatch, 256) = 1536 cells + ckpt = 25 * 2 * 8 * 256 * 2 * 1536 + self.assertGreaterEqual(d["ckpt_bytes"], d["seqs"] * k * ckpt) + cfg = config("google__gemma-4-26B-A4B-it-qat-q4_0-gguf") + w = json.load(open(os.path.join(mdir("google__gemma-4-26B-A4B-it-qat-q4_0-gguf"), "weights.json"))) + m = fit.model_profile(cfg, w) + self.assertGreaterEqual(d["host_bytes"], m["serve_bytes"] + fit.HOST_RUNTIME["llama.cpp"] + d["kv_bytes"] + + d["seqs"] * k * ckpt) + # the pod limit holds the checkpoints and stays inside the host budget + budget, _, _ = fit.host_budget(host("desktop-i9-nogpu")) + self.assertGreaterEqual(int(d["k8s"]["memory_limit"][:-2]) * GIB, d["host_bytes"]) + self.assertLessEqual(int(d["k8s"]["memory_limit"][:-2]) * GIB, budget) + # and on every guard rung + for r, (rs, rc) in enumerate(d["guard"]["rungs"]): + self.assertGreaterEqual(int(d["guard"]["rung_memory_limit"][r][:-2]) * GIB, + d["host_bytes"] - d["kv_bytes"] - d["ckpt_bytes"] + + d["guard"]["rung_kv_bytes"][r] + rs * k * ckpt, r) + + def test_recurrent_state_checkpoints_on_the_host(self): + # Flash-Next: 36 gated-DeltaNet layers, each checkpoint the per-sequence state + d = plan("xeon6741p-4xb70", "ggml-org__Qwen3.8-Flash-Next-GGUF") + self.assertTrue(d["fits"], d["summary"]) + k = self.flag(d, "-ctxcp") + state = 36 * (48 * 128 * 128 * 4 + 3 * (2 * 16 * 128 + 48 * 128) * 2) + self.assertEqual(d["ckpt_bytes"], d["seqs"] * k * state) + self.assertIn("--cache-ram", d["flags"]) + + def test_full_attention_makes_no_checkpoints(self): + cfg = config("Qwen__Qwen3.8-27B") + cfg = dict(cfg, text_config=dict(cfg.get("text_config") or cfg, layer_types=None)) + kv = fit.kv_profile(cfg, 2) + kv["ckpts"] = 4 + self.assertEqual(fit.ckpt_seq(kv, 65536), 0) + + class FitLaptop(unittest.TestCase): """Core Ultra 7 258V, 16 GB, 9 GB available, swap in use.""" diff --git a/tests/fit_test.sh b/tests/fit_test.sh index b655baf5..355a1a3c 100755 --- a/tests/fit_test.sh +++ b/tests/fit_test.sh @@ -69,6 +69,23 @@ if [ "${rc}" -eq 0 ] && python3 -c 'import json, sys; d = json.loads(sys.argv[1] ok "fit --json: a fit/v1 document, from a local model directory" else fail "fit --json: a fit/v1 document" "${out}"; fi +# llama-server keeps a host prompt cache (8192 MiB by default) and context checkpoints per +# slot (32 by default, each the slot's SWA cache): the fit turns the first off, bounds the +# second, and counts the checkpoints in the host bytes and the pod limit. +out=$(bash "${K3D_GPU}" fit --host "${DESK}" --model google/gemma-4-26B-A4B-it-qat-q4_0-gguf --json 2>&1); rc=$? +if [ "${rc}" -eq 0 ] && python3 -c ' +import json, sys +d = json.loads(sys.argv[1]) +f = d["flags"] +assert d["placement"] == "cpu" and f[f.index("--cache-ram") + 1] == "0", f +k = int(f[f.index("-ctxcp") + 1]) +ckpt = 25 * 2 * 8 * 256 * 2 * 1536 # 25 SWA layers K and V at 2 B over 1536 cells +assert d["ckpt_bytes"] >= d["seqs"] * k * ckpt > 0, (d["ckpt_bytes"], d["seqs"], k) +assert int(d["k8s"]["memory_limit"][:-2]) * 2 ** 30 >= d["host_bytes"] >= d["kv_bytes"] + d["ckpt_bytes"] +' "${out}"; then + ok "fit (llama.cpp, Gemma QAT on the desktop CPU): --cache-ram 0, -ctxcp K, checkpoints in the pod limit" +else fail "fit (llama.cpp host caches) on the desktop (rc ${rc})" "${out}"; fi + out=$(bash "${K3D_GPU}" fit --host "${XEON}" --model not/in-cache-offline 2>&1