diff --git a/console/fit.py b/console/fit.py index dd9448f9..112bca65 100644 --- a/console/fit.py +++ b/console/fit.py @@ -48,6 +48,7 @@ """ import argparse +import bisect import glob import json import math @@ -94,6 +95,13 @@ # the agent stack sets it, llama.cpp's default -ub 512). VLLM_BATCHED_TOKENS = 2048 LLAMA_UBATCH = 512 +# llama-server context checkpoints per slot (-ctxcp; v0.8.0 default 32, common/common.h). Each +# is a host-RAM copy of the slot's SWA cache and recurrent state (tools/server/server-context.cpp +# create_checkpoint, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY). A request makes them at the last user +# message and 4 + n_ubatch and 4 tokens before its prompt's end; an appended turn restores from +# those, so four keep the latest request's set. The host prompt cache (--cache-ram, default +# 8192 MiB) is turned off with --cache-ram 0: a slot's KV stays in its own cache. +LLAMA_CTX_CHECKPOINTS = 4 # Fixed runtime per GPU process (device context, allocator pools, collective # buffers) and per pod on the host (Python, torch or the server binary). Estimates. GPU_RUNTIME = 1 * GIB @@ -118,6 +126,12 @@ # KV cache types each engine takes and the fit sets a flag for (vLLM --kv-cache-dtype, # llama.cpp -ctk/-ctv; OVMS's KV_CACHE_PRECISION is not set by the fit). KV_DTYPES_OF = {"vllm": ("fp8", "fp16", "bf16"), "llama.cpp": ("f16", "fp16", "bf16", "q8_0", "q4_0")} +# Tokens a sliding-window layer's cache holds per sequence (swa_tokens). vLLM: the window. +# llama.cpp v0.8.0: GGML_PAD(n_swa + n_ubatch, 256) cells per stream (src/llama-kv-cache-iswa.cpp). +# OVMS: OpenVINO GenAI 2026.4 has only KV_CACHE and LINEAR_ATTENTION_CACHE types +# (src/cpp/src/continuous_batching/cache/cache_type.hpp), and a KV block spans every +# attention layer (kv_cache_manager.hpp), so sliding-window layers hold the whole context. +SWA_CACHE = {"vllm": "window", "llama.cpp": "llama.cpp", "ovms": "full"} # Tensor classes by name: the parts a placement moves separately. CLASSES = ( @@ -789,19 +803,20 @@ def model_type(cfg): return str(text_config(cfg).get("model_type") or cfg.get("model_type") or "").replace("_text", "") -def kv_profile(cfg, kv_bytes): +def kv_profile(cfg, kv_bytes, engine=None): """Per-token KV bytes for full-attention layers, per-token-within-window bytes for - sliding-window layers, and per-sequence state bytes for linear-attention layers.""" + sliding-window layers, and per-sequence state bytes for linear-attention layers. + engine sets how many tokens a sliding-window layer holds (swa_tokens).""" tc = text_config(cfg) n = tc.get("num_hidden_layers") types = tc.get("layer_types") or ["full_attention"] * n heads = tc.get("num_attention_heads") kvh = tc.get("num_key_value_heads") or heads hd = tc.get("head_dim") or tc["hidden_size"] // heads - full = sum(1 for t in types if t == "full_attention") - swa = sum(1 for t in types if t in ("sliding_attention", "sliding_window_attention", "local_attention")) - lin = sum(1 for t in types if t in ("linear_attention", "mamba", "gated_delta_net")) - full += len(types) - full - swa - lin # an unknown layer type counts as full attention (upper bound) + # an unknown layer type counts as full attention (upper bound) + kinds = ["swa" if t in ("sliding_attention", "sliding_window_attention", "local_attention") else + "linear" if t in ("linear_attention", "mamba", "gated_delta_net") else "full" for t in types] + full, swa, lin = kinds.count("full"), kinds.count("swa"), kinds.count("linear") gkvh = tc.get("num_global_key_value_heads") or kvh ghd = tc.get("global_head_dim") or hd lines = [] @@ -817,9 +832,16 @@ def kv_profile(cfg, kv_bytes): % (full, tc["indexer_kv_heads"], tc["indexer_head_dim"], _num(kv_bytes), full * idx)) window = tc.get("sliding_window") or 0 per_swa = 2 * kvh * hd * kv_bytes + swa_cache = SWA_CACHE.get(engine, "window") if swa: - lines.append("sliding window: %d layers x 2 x %d kv heads x %d head dim x %s B = %d B/token, at most %d tokens" - % (swa, kvh, hd, _num(kv_bytes), swa * per_swa, window)) + lines.append("sliding window: %d layers x 2 x %d kv heads x %d head dim x %s B = %d B/token, %s" + % (swa, kvh, hd, _num(kv_bytes), swa * per_swa, { + "full": "for the whole context: OpenVINO GenAI 2026.4 has no sliding-window cache type" + " (continuous_batching/cache/cache_type.hpp), every attention layer gets every token", + "llama.cpp": "for GGML_PAD(%d window + %d ubatch, 256) = %d tokens per sequence at most" + " (src/llama-kv-cache-iswa.cpp)" % (window, LLAMA_UBATCH, + _pad(window + LLAMA_UBATCH, 256)), + "window": "at most %d tokens" % window}[swa_cache])) state = 0 if lin: vh, kd = tc.get("linear_num_value_heads", 0), tc.get("linear_key_head_dim", 0) @@ -833,8 +855,13 @@ def kv_profile(cfg, kv_bytes): lines.append("linear attention state: %d layers x (%d v heads x %d x %d x %d B + conv (%d-1) x %d ch x 2 B)" " = %s per sequence, any length" % (lin, vh, kd, vd, ssm_b, ck, ch, _mb(state))) return {"full_layers": full, "full_per_token": full * (per_full + idx), "swa_layers": swa, - "swa_per_token": swa * per_swa, "window": window, "linear_layers": lin, - "state_per_seq": state, "kv_heads": gkvh, "lines": lines} + "swa_per_token": swa * per_swa, "window": window, "swa_cache": swa_cache, "linear_layers": lin, + "state_per_seq": state, "kv_heads": gkvh, "kinds": kinds, "lines": lines} + + +def _pad(n, to): + """GGML_PAD: n rounded up to a multiple of to.""" + return (n + to - 1) // to * to def _num(x): @@ -862,6 +889,7 @@ def model_profile(cfg, weights, tokenizer_cfg=None): b = {k: cls.get(k, {}).get("bytes", 0) for k in ("experts", "embed", "ngram", "lm_head", "vision", "mtp", "other")} # The MTP head loads only for speculative decoding; a GGUF loads every tensor it holds. serve = sum(v for k, v in b.items() if k != "mtp" or weights["format"] == "gguf") + mtp_blocks = (tc.get("mtp_num_hidden_layers") or tc.get("num_nextn_predict_layers") or 1) if b["mtp"] else 0 ne = tc.get("num_experts") or tc.get("n_routed_experts") or tc.get("num_local_experts") or 0 k = tc.get("num_experts_per_tok") or tc.get("top_k_experts") or tc.get("num_experts_per_token") or 0 dtype = str(tc.get("dtype") or tc.get("torch_dtype") or cfg.get("torch_dtype") or "bfloat16") @@ -876,6 +904,10 @@ def model_profile(cfg, weights, tokenizer_cfg=None): "intermediate": max(tc.get("intermediate_size") or 0, (tc.get("moe_intermediate_size") or 0) * max(k, 1)), "max_ctx": cap, "ctx_why": capwhy, "experts": ne, "experts_per_tok": k, "moe": bool(ne and b["experts"]), + # MTP (nextn) blocks a GGUF holds after its trunk (llama.cpp's converter keeps them all + # or none): llama.cpp counts them in its layers (block_count = trunk + nextn_predict_layers). + # mtp_blocks: the MTP blocks the weights have, which a GGUF conversion keeps (llama_profile). + "nextn": mtp_blocks if weights["format"] == "gguf" else 0, "mtp_blocks": mtp_blocks, "bytes": b, "serve_bytes": serve, "lookup_bytes": b["embed"] + b["ngram"], "format": weights["format"], "dtype_bytes": 2 if dtype in ("bfloat16", "float16") else 4, # compressed-tensors kv_cache_scheme: vLLM switches its KV cache to that type (fp8) on its own. @@ -883,6 +915,16 @@ def model_profile(cfg, weights, tokenizer_cfg=None): } +def llama_profile(m): + """The model llama.cpp serves: a GGUF as it is; other weights as the GGUF the agent's + conversion writes from them. convert_hf_to_gguf.py runs without --no-mtp, so that GGUF + keeps the MTP blocks after the trunk (llama.cpp v0.8.0 conversion/qwen.py _QwenMtpMixin + adds mtp_num_hidden_layers to block_count) and is served like a GGUF that holds them.""" + if m["format"] == "gguf" or not m["mtp_blocks"]: + return m + return dict(m, nextn=m["mtp_blocks"], serve_bytes=m["serve_bytes"] + m["bytes"]["mtp"]) + + # ---- engines --------------------------------------------------------------------------- # Which engine serves which architecture on Intel GPUs or the CPU, from each engine's # own release notes and recipes. An architecture not listed is not refused (the engine @@ -1013,9 +1055,158 @@ def act_bytes(m, tokens, seqs): return tokens * width * 2 * 2 + seqs * m["vocab"] * 4 +def swa_tokens(kv, ctx): + """Tokens one sequence's sliding-window layers hold at context ctx. vLLM: the window. + llama.cpp: its SWA cache of GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch), 256) cells per + stream (one stream per slot with -np set). OVMS: the whole context.""" + w = kv["window"] or ctx + mode = kv.get("swa_cache", "window") + if mode == "full": + return ctx + if mode == "llama.cpp": + return min(_pad(ctx, 256), _pad(w + LLAMA_UBATCH, 256)) + return min(ctx, w) + + +def kv_parts(kv, ctx): + """(full attention, sliding window, linear state) KV bytes of one sequence at context ctx.""" + return kv["full_per_token"] * ctx, kv["swa_per_token"] * swa_tokens(kv, ctx), kv["state_per_seq"] + + def per_seq(kv, ctx, scale=1.0): - return int((kv["full_per_token"] * ctx + kv["swa_per_token"] * min(ctx, kv["window"] or ctx)) * scale - + kv["state_per_seq"] * scale) + f, w, st = kv_parts(kv, ctx) + return int((f + w) * scale + st * scale) + + +# ---- llama.cpp over several GPUs ---------------------------------------------------------- +def llama_layer_devices(ts, n_layer): + """Device of each repeating layer and of the output layer (the last entry), as llama.cpp + v0.8.0 assigns them with -ngl past the layer count (src/llama-model.cpp load_tensors): + upper_bound(cumsum(ts) / sum(ts), il / (n_layer + 1)). Integer layer counts summing to + n_layer + 1 give each device exactly its count, the output layer on the last one.""" + total, cum, splits = float(sum(ts)), 0.0, [] + for t in ts: + cum += t + splits.append(cum / total) + return [bisect.bisect_right(splits, il / (n_layer + 1)) for il in range(n_layer + 1)] + + +def llama_output_copy(m): + """Bytes of the token embedding llama.cpp loads a second time as the output tensor when + the model ties its embeddings (no output.weight / lm_head): TENSOR_DUPLICATED, placed on + the output layer's device (src/llama-model-loader.cpp). On the CPU it maps the same file + pages; on a GPU it is a copy.""" + return m["bytes"]["embed"] if not m["bytes"]["lm_head"] else 0 + + +def llama_layer_bytes(m, kv, ctx, seqs, n_cpu_moe): + """([weights], [KV]) on the GPU per repeating layer, the output layer last: the class + totals spread evenly over the trunk's layers, the experts of the first n_cpu_moe layers in + host RAM (--n-cpu-moe), and each layer's own KV (its device holds it) for seqs sequences. + A GGUF's MTP (nextn) blocks follow the trunk as repeating layers (llama.cpp v0.8.0 counts + them in n_layer_all, src/llama-model.cpp) with their share of the MTP tensors and no KV: + the cache covers il < n_layer(), the trunk. The weights add up to the bytes to serve less + the lookup tables, plus the output copy of tied embeddings. The output layer holds lm_head + (or that copy); the vision projector counts with layer 0, on the first card: + tools/mtmd/clip.cpp loads it on the first GPU.""" + n, nx = max(m["layers"], 1), m["nextn"] + b = m["bytes"] + exp, other = b["experts"] / n, b["other"] / n + f, w, st = kv_parts(kv, ctx) + per = {"full": f / max(kv["full_layers"], 1), "swa": w / max(kv["swa_layers"], 1), + "linear": st / max(kv["linear_layers"], 1)} + kinds = kv.get("kinds") or [] + wl = [other + (exp if il >= n_cpu_moe else 0) for il in range(n)] + [b["mtp"] / max(nx, 1)] * nx + wl[0] += b["vision"] + kl = [per[kinds[il]] * seqs if il < len(kinds) else 0 for il in range(n)] + [0] * nx + out = (m["serve_bytes"] - m["lookup_bytes"] - b["experts"] - b["other"] - b["vision"] - (b["mtp"] if nx else 0) + + llama_output_copy(m)) + return wl + [out], kl + [0] + + +def llama_split(layer_bytes, k, room): + """-ts for k cards as integer layer counts (the last card's count includes the output + layer) that keep every card's layers within room bytes, its largest load as small as it + can be; None when no split does.""" + last = len(layer_bytes) - 1 + + def fill(cap): + ts, i = [], 0 + for _ in range(k - 1): + start, used = i, 0 + while i < last and used + layer_bytes[i] <= cap: + used += layer_bytes[i] + i += 1 + ts.append(i - start) + if sum(layer_bytes[i:]) > cap: + return None + return ts + [len(layer_bytes) - i] + + if fill(room) is None: + return None + lo, hi = max(layer_bytes), room + while hi - lo > MIB: + mid = (lo + hi) / 2 + lo, hi = (lo, mid) if fill(mid) else (mid, hi) + return fill(hi) + + +def llama_cards(ts, layer_bytes): + """Each card's bytes for ts: the layers llama_layer_devices puts on it.""" + out = [0] * len(ts) + for il, dev in enumerate(llama_layer_devices(ts, len(layer_bytes) - 1)): + out[dev] += layer_bytes[il] + return out + + +def _llama_search(m, kv, ladder, relaxed, want_seqs, k, room, n_cpus): + """(ctx, seqs, n_cpu_moe, ts) for llama.cpp on k cards of room bytes each: the most + context at which TARGET_SEQS (or want_seqs) sequences fit, then the fewest layers whose + experts leave the GPU (from n_cpus), then as many sequences as fit (capped at MAX_SEQS). + relaxed: at the ladder's last rung one sequence is enough. None when nothing fits.""" + for ctx in ladder: + need = want_seqs or (1 if relaxed and ctx == ladder[-1] else TARGET_SEQS) + for n_cpu in n_cpus: + def split(n): + wl, kl = llama_layer_bytes(m, kv, ctx, n, n_cpu) + return llama_split([a + b for a, b in zip(wl, kl)], k, room) + if not split(need): + continue + n = want_seqs + if not n: + lo, hi = need, MAX_SEQS + if split(hi): + lo = hi + while hi - lo > 1: + mid = (lo + hi) // 2 + lo, hi = (mid, hi) if split(mid) else (lo, mid) + n = lo + return ctx, n, n_cpu, split(n) + return None + + +def _llama_card_lines(use, ts, wl, kl, ov, m): + """One explain line per card of a llama.cpp split.""" + lines, first = [], 0 + for c, t, w, kvb in zip(use, ts, llama_cards(ts, wl), llama_cards(ts, kl)): + lines.append(" %s (-ts %d): layers %s, weights %s + KV %s + overhead %s = %s, budget %s" + % (c["card"], t, "%d-%d" % (first, first + t - 1) if t else "none", g1(w), g1(kvb), g1(ov), + g1(w + kvb + ov), g1(c["budget"]))) + first += t + if m["nextn"]: + a, z = m["layers"], m["layers"] + m["nextn"] - 1 + lines.append(" (%s the GGUF's MTP %s: nextn tensors, no KV)" + % ("layer %d is" % a if a == z else "layers %d-%d are" % (a, z), "block" if a == z else "blocks")) + lines.append(" (layer %d is the output layer: %s)" % (len(wl) - 1, "lm_head" if not llama_output_copy(m) else + "a %s copy of the tied token embedding" % g1(llama_output_copy(m)))) + return lines + + +def ckpt_seq(kv, ctx): + """Host bytes of one llama-server slot's context checkpoints at context ctx: kv["ckpts"] + of them, each the slot's SWA cache (GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch), 256) + cells, src/llama-kv-cache-iswa.cpp) and recurrent state. A model with neither makes none.""" + return kv.get("ckpts", 0) * (kv["swa_per_token"] * swa_tokens(kv, ctx) + kv["state_per_seq"]) def ctx_ladder(max_ctx, want=None): @@ -1028,13 +1219,14 @@ def ctx_ladder(max_ctx, want=None): return out or [max_ctx] -def size_kv(kv, budget, ladder, want_seqs, scale=1.0, relaxed=False): +def size_kv(kv, budget, ladder, want_seqs, scale=1.0, relaxed=False, host_ckpt=False): """(ctx, seqs): the most context at which TARGET_SEQS (or want_seqs) whole sequences fit in budget, then as many sequences as fit at that context (capped at MAX_SEQS). - relaxed: at the ladder's last rung one sequence is enough.""" + relaxed: at the ladder's last rung one sequence is enough. host_ckpt: the budget also + holds each sequence's context checkpoints (KV in host RAM).""" need = want_seqs or TARGET_SEQS for ctx in ladder: - s = per_seq(kv, ctx, scale) + s = per_seq(kv, ctx, scale) + (ckpt_seq(kv, ctx) if host_ckpt else 0) n = int(budget // s) if s else MAX_SEQS if n >= need or (relaxed and ctx == ladder[-1] and n >= 1 and not want_seqs): return ctx, min(want_seqs or n, MAX_SEQS) @@ -1101,20 +1293,30 @@ def fit(host, cfg, weights, engine=None, ctx=None, seqs=None, gpus=None, kv_dtyp "summary": "fit: %s does not fit this host: %s cannot serve these %s weights (%s)" % (name, engine, m["format"], why)} cands = asked - kvs = {} + kvs, ms = {}, {} for eng, note in cands: ex.append("engine %s: %s" % (eng or FORMAT_ENGINE.get(m["format"], "?"), note)) if not eng: continue + ms[eng] = llama_profile(m) if eng == "llama.cpp" else m + if ms[eng] is not m: + ex.append(" the GGUF conversion keeps the MTP head (convert_hf_to_gguf.py without --no-mtp): " + "%d block(s) after the trunk, %s more to serve, %s" + % (ms[eng]["nextn"], g1(m["bytes"]["mtp"]), g1(ms[eng]["serve_bytes"]))) if kv_dtype != "auto" and kv_dtype not in KV_DTYPES_OF.get(eng, ()): raise ValueError("--kv-dtype %s is not a KV cache type of %s (%s)" % (kv_dtype, eng, ", ".join(KV_DTYPES_OF.get(eng, ())) or "auto only")) kvb = KV_DTYPE_BYTES.get(kv_dtype) or (1 if eng == "vllm" and m["kv_scheme_bits"] == 8 else m["dtype_bytes"]) - kvs[eng] = kv_profile(cfg, kvb) + kvs[eng] = kv_profile(cfg, kvb, eng) kvs[eng]["dtype"] = kv_dtype if kv_dtype != "auto" else ("fp8" if kvb == 1 else "auto") + kvs[eng]["ckpts"] = LLAMA_CTX_CHECKPOINTS if eng == "llama.cpp" else 0 ex.append(" KV cache (%s B per element%s):" % (_num(kvb), ", the checkpoint's kv_cache_scheme" if kvb == 1 and kv_dtype == "auto" else "")) ex += [" " + s for s in kvs[eng]["lines"]] + if eng == "llama.cpp": + ex.append(" host prompt cache off (--cache-ram 0); context checkpoints (-ctxcp %d) in host RAM: " + "%d x the slot's SWA cache and recurrent state = %s per slot at %d tokens" + % (LLAMA_CTX_CHECKPOINTS, LLAMA_CTX_CHECKPOINTS, _mb(ckpt_seq(kvs[eng], full[0])), full[0])) tried, picked, degraded = [], None, False for i, (label, ladder, relaxed) in enumerate(passes): ex.append("placements, %s:" % label) @@ -1124,7 +1326,7 @@ def fit(host, cfg, weights, engine=None, ctx=None, seqs=None, gpus=None, kv_dtyp # vLLM's CPU build is a placement only when asked for (--device cpu: the EAT # overlay); the agent's vLLM images are the XPU and Gaudi builds. cpu_ok = device == "cpu" if eng == "vllm" else device != "gpu" - for res in _placements(eng, m, kvs[eng], ladder, relaxed, seqs, gpus, cards, hfree, host, overhead, + for res in _placements(eng, ms[eng], kvs[eng], ladder, relaxed, seqs, gpus, cards, hfree, host, overhead, cpu_ok): res["note"] = note tried.append(res) @@ -1141,7 +1343,13 @@ def fit(host, cfg, weights, engine=None, ctx=None, seqs=None, gpus=None, kv_dtyp doc = {"schema": FIT_SCHEMA, "model": name, "format": m["format"], "arch": m["arch"], "fits": bool(picked), "degraded": degraded, "host_reserve": hres, "explain": ex} if not picked: - why = tried[-1]["why"] if tried else (cands[0][1] if cands else "no engine serves %s weights" % m["format"]) + if tried: + why = tried[-1]["why"] + elif any(e for e, _ in cands): + # No placement was tried: no card left after the busy ones, and the CPU not allowed. + why = "no Intel GPU with device-local memory is free" + else: + why = cands[0][1] if cands else "no engine serves %s weights" % m["format"] doc.update(reason=why, summary="fit: %s does not fit this host: %s" % (name, why)) return doc rungs = guard_rungs(picked["ctx"], picked["seqs"]) @@ -1149,13 +1357,14 @@ def fit(host, cfg, weights, engine=None, ctx=None, seqs=None, gpus=None, kv_dtyp rung_kv, rung_limit = [], [] for rs, rc in rungs: kb = per_seq(kvp, rc, picked["kv_scale"]) * rs - if picked["engine"] == "vllm" and picked["placement"] == "cpu": + if _whole_gib_pool(picked["engine"], picked["placement"]): kb = int(math.ceil(kb / GIB)) * GIB - hb = picked["host_bytes"] - picked["kv_bytes"] + kb if picked["placement"] == "cpu" else picked["host_bytes"] + hb = (picked["host_bytes"] - picked["kv_bytes"] - picked["ckpt_bytes"] + kb + ckpt_seq(kvp, rc) * rs + if picked["placement"] == "cpu" else picked["host_bytes"]) rung_kv.append(int(kb)) rung_limit.append("%dGi" % int(math.ceil(hb * (1 + LIMIT_SLACK) / GIB))) doc.update({k: picked[k] for k in ("engine", "placement", "ctx", "seqs", "tp", "gpus", "gpu_bytes", "host_bytes", - "flags", "dram_tok_s", "note", "kv_bytes")}) + "flags", "dram_tok_s", "note", "kv_bytes", "ckpt_bytes", "card_bytes")}) doc["source"] = weights.get("source") doc["weight_bytes"] = weights.get("total_bytes") or m["serve_bytes"] if m["format"] == "gguf": @@ -1201,11 +1410,27 @@ def _overhead(eng, m, seqs, override): def _res(eng, place, fits, why="", math_lines=(), **kw): r = {"engine": eng, "placement": place, "fits": fits, "why": why, "math": list(math_lines), "ctx": None, "seqs": 0, "tp": 1, "gpus": 0, "gpu_bytes": 0, "host_bytes": 0, "flags": [], "dram_tok_s": None, - "kv_bytes": 0, "kv_scale": 1.0} + "kv_bytes": 0, "kv_scale": 1.0, "ckpt_bytes": 0, "card_bytes": []} r.update(kw) return r +def _whole_gib_pool(eng, place): + """KV pools set in whole GiB: OVMS --cache_size, vLLM's CPU VLLM_CPU_KVCACHE_SPACE.""" + return eng == "ovms" or (eng == "vllm" and place == "cpu") + + +def _kv_room(eng, place, left): + """The KV room sequences are sized in: whole GiB for a whole-GiB pool, so the pool + rounded up to hold them stays inside left.""" + return left // GIB * GIB if _whole_gib_pool(eng, place) else left + + +def _kv_pool(eng, place, kvbytes): + """The KV bytes the engine sets aside for kvbytes of sequences.""" + return int(math.ceil(kvbytes / GIB)) * GIB if _whole_gib_pool(eng, place) else kvbytes + + def _dram_rate(bw, nbytes): return bw * 1e9 / nbytes if bw and nbytes > 0 else None @@ -1227,7 +1452,11 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, compile_host = m["serve_bytes"] if eng == "ovms" else 0 gpu_w_all = m["serve_bytes"] - on_host hostrt = HOST_RUNTIME[eng] + (VLLM_SHM if eng == "vllm" else 0) # times the rank count for vLLM, below - if cards: + if cards and eng == "ovms" and want_gpus and want_gpus > 1: + # The agent's OVMS pod holds one gpu.intel.com/xe; multi-GPU tensor parallel on Arc + # is not documented (model_server docs/parameters.md). + yield _res(eng, "gpu", False, "OVMS serves one GPU: %d asked" % want_gpus, gpus=want_gpus) + elif cards: if eng == "vllm": counts = _tp_sizes(m, len(cards)) elif eng == "llama.cpp": @@ -1242,6 +1471,9 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, yield _res(eng, place, False, "%d GPUs asked, %d not busy" % (k, len(cards)), gpus=k) continue use = cards[:k] + if eng == "llama.cpp" and k > 1: + yield _llama_multi(m, kv, ladder, relaxed, want_seqs, use, ov, ovl, [0], on_host, hfree, bw) + continue if eng == "vllm": # Every rank holds 1/k of the weights; KV heads split across ranks, or one # per rank (replicated) with fewer KV heads than ranks. @@ -1252,95 +1484,69 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, lines = ["per card: budget %s - weights %s / %d - overhead %s = %s for KV" % (g1(budget), g1(gpu_w_all), k, ovl, g1(max(left, 0)))] else: - # One card, or llama.cpp's layer split (-sm layer, the default): weights and - # KV spread over the cards in proportion to -ts; each card runs its own buffers. + # One card (llama.cpp on several is sized card by card in _llama_multi). budget = sum(c["budget"] for c in use) - w = gpu_w_all + w = gpu_w_all + (llama_output_copy(m) if eng == "llama.cpp" else 0) left = budget - w - ov * k scale = 1.0 lines = ["%s: budget %s - weights %s - overhead %d x %s = %s for KV" % ("+".join(c["card"] for c in use), g1(budget), g1(w), k, ovl, g1(max(left, 0)))] + if eng == "llama.cpp" and llama_output_copy(m): + lines.append("weights include the output tensor: a %s copy of the tied token embedding" + % g1(llama_output_copy(m))) if on_host: lines.append("lookup tables %s stay in host RAM" % g1(on_host)) if left <= 0: yield _res(eng, place, False, "weights + overhead %s exceed the GPU budget %s" % (g1(w + ov * (1 if eng == "vllm" else k)), g1(budget)), lines, gpus=k, tp=k if eng == "vllm" else 1) continue - c, n = size_kv(kv, left, ladder, want_seqs, scale, relaxed) + c, n = size_kv(kv, _kv_room(eng, place, left), ladder, want_seqs, scale, relaxed) if not c: yield _res(eng, place, False, "KV room %s holds no %d-token sequence" % (g1(left), ladder[-1]), lines, gpus=k) continue - kvused = per_seq(kv, c, scale) * n + kvused = _kv_pool(eng, place, per_seq(kv, c, scale) * n) lines.append("KV: %d seqs x %s at %d tokens = %s%s" % (n, _mb(per_seq(kv, c, scale)), c, g1(kvused), " per card" if eng == "vllm" else "")) # vLLM runs one worker process per tensor-parallel rank, each with its own runtime. - hb = int(hostrt + HOST_RUNTIME[eng] * (k - 1) * (eng == "vllm") + on_host + compile_host) + ck = ckpt_seq(kv, c) * n + if ck: + lines.append("context checkpoints: %d seqs x %s = %s in host RAM" % (n, _mb(ckpt_seq(kv, c)), g1(ck))) + hb = int(hostrt + HOST_RUNTIME[eng] * (k - 1) * (eng == "vllm") + on_host + compile_host + ck) if hb > hfree: yield _res(eng, place, False, "host side %s exceeds the host budget %s" % (g1(hb), g1(hfree)), lines, gpus=k) continue gpu_use = int(w + ov + kvused) if eng == "vllm" else int((w + ov * k + kvused) / k) yield _res(eng, place, True, "", lines, ctx=c, seqs=n, tp=k if eng == "vllm" else 1, gpus=k, - gpu_bytes=gpu_use, host_bytes=hb, kv_bytes=int(kvused), kv_scale=scale, + gpu_bytes=gpu_use, host_bytes=hb, kv_bytes=int(kvused), kv_scale=scale, ckpt_bytes=ck, + card_bytes=[gpu_use] * k, flags=_flags(eng, place, c, n, k, gpu_use, use, kvused, m, 0, kv)) - # MoE experts to DRAM (llama.cpp --n-cpu-moe): every free card, fewest CPU layers. + # MoE experts to DRAM (llama.cpp --n-cpu-moe). Every card is held to the smallest budget + # of the cards used, so one more card can leave less room per card: the free cards by + # budget, 1, 2, ... of them (or --gpus), and the plan with the most context, then the + # fewest CPU-expert layers, then the most sequences, then the fewest cards. if eng == "llama.cpp" and cards and m["moe"]: - k = want_gpus or len(cards) - use = cards[:k] - budget = sum(c["budget"] for c in use) - exp_layer = m["bytes"]["experts"] / max(m["layers"], 1) - base = gpu_w_all - m["bytes"]["experts"] - lines = ["GPU (%s, budget %s) keeps %s non-expert weights + overhead %d x %s" - % ("+".join(c["card"] for c in use), g1(budget), g1(base), k, ovl), - "lookup tables %s in host RAM" % g1(on_host)] - if not (cpu["amx"] or cpu["avx512"]): - yield _res(eng, "moe-offload", False, "the CPU has neither AMX nor AVX-512 for expert compute", lines, gpus=k) + ks = [want_gpus] if want_gpus else list(range(1, len(cards) + 1)) + if want_gpus and want_gpus > len(cards): + yield _res(eng, "moe-offload", False, "%d GPUs asked, %d not busy" % (want_gpus, len(cards)), gpus=want_gpus) + elif not (cpu["amx"] or cpu["avx512"]): + yield _res(eng, "moe-offload", False, "the CPU has neither AMX nor AVX-512 for expert compute", gpus=ks[-1]) elif not bw: yield _res(eng, "moe-offload", False, "DRAM bandwidth unknown (no STREAM run cached, SMBIOS unreadable)", - lines, gpus=k) + gpus=ks[-1]) else: - # Most context first (as everywhere), then the fewest layers whose experts leave the GPU. - best = None - for rung in ladder: - for n_cpu in range(1, m["layers"] + 1): - gw = base + exp_layer * (m["layers"] - n_cpu) - left = budget - gw - ov * k - if left <= 0: - continue - c, n = size_kv(kv, left, [rung], want_seqs, 1.0, relaxed and rung == ladder[-1]) - if c: - best = (n_cpu, gw, c, n) - break - if best: - break - if not best: - yield _res(eng, "moe-offload", False, "non-expert weights + KV do not fit %s of VRAM" % g1(budget), - lines, gpus=k) + tries = [_llama_multi(m, kv, ladder, relaxed, want_seqs, cards[:k], ov, ovl, range(1, m["layers"] + 1), + on_host, hfree, bw) for k in ks] + ok = [r for r in tries if r["fits"]] + if not ok: + yield tries[-1] else: - n_cpu, gw, c, n = best - dram_w = exp_layer * n_cpu - read = dram_w * m["experts_per_tok"] / m["experts"] - toks = _dram_rate(bw, read) - lines.append("experts of the first %d of %d layers in DRAM: %s; on GPU %s" - % (n_cpu, m["layers"], g1(dram_w), g1(gw))) - lines.append("DRAM decode bound: %g GB/s / (%s x %d/%d read per token) = %.0f tok/s (floor %g)" - % (bw, g1(dram_w), m["experts_per_tok"], m["experts"], toks, MIN_DRAM_TOK_S)) - kvused = per_seq(kv, c) * n - lines.append("KV: %d seqs x %s at %d tokens = %s on GPU" % (n, _mb(per_seq(kv, c)), c, g1(kvused))) - hb = int(dram_w + on_host + HOST_RUNTIME[eng]) - lines.append("host: experts %s + lookup tables %s + runtime %s = %s" - % (g1(dram_w), g1(on_host), g1(HOST_RUNTIME[eng]), g1(hb))) - if toks < MIN_DRAM_TOK_S: - yield _res(eng, "moe-offload", False, "DRAM bandwidth gives %.1f tok/s, under the %g floor" - % (toks, MIN_DRAM_TOK_S), lines, gpus=k) - elif hb > hfree: - yield _res(eng, "moe-offload", False, "DRAM side needs %s, host budget %s" % (g1(hb), g1(hfree)), - lines, gpus=k) - else: - gpu_use = int((gw + ov * k + kvused) / k) - yield _res(eng, "moe-offload", True, "", lines, ctx=c, seqs=n, gpus=k, gpu_bytes=gpu_use, - host_bytes=hb, dram_tok_s=round(toks, 1), kv_bytes=int(kvused), - flags=_flags(eng, "moe-offload", c, n, k, gpu_use, use, kvused, m, n_cpu, kv)) + best = max(ok, key=lambda r: (r["ctx"], -r["n_cpu_moe"], r["seqs"], -r["gpus"])) + if len(tries) > 1: + best["math"].append("cards tried, by budget: " + "; ".join( + "%d: %d x %d, experts of %d layers in DRAM" % (r["gpus"], r["ctx"], r["seqs"], r["n_cpu_moe"]) + if r["fits"] else "%d: no" % r["gpus"] for r in tries)) + yield best # CPU only: weights, KV and activations all in DRAM (vLLM: its CPU build, the KV # pool sized by VLLM_CPU_KVCACHE_SPACE, as EAT's vLLM chart sets it). if cpu_ok: @@ -1350,8 +1556,9 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, if m["moe"]: active -= m["bytes"]["experts"] * (1 - m["experts_per_tok"] / m["experts"]) toks = _dram_rate(bw, active) - lines = ["host budget / %.2f slack %s - weights %s - activations %s - runtime %s = %s for KV" - % (1 + LIMIT_SLACK, g1(hfree), g1(m["serve_bytes"]), g1(act), g1(HOST_RUNTIME[eng]), g1(max(left, 0)))] + lines = ["host budget / %.2f slack %s - weights %s - activations %s - runtime %s = %s for KV%s" + % (1 + LIMIT_SLACK, g1(hfree), g1(m["serve_bytes"]), g1(act), g1(HOST_RUNTIME[eng]), g1(max(left, 0)), + " and context checkpoints" if kv.get("ckpts") else "")] if toks: lines.append("DRAM decode bound: %g GB/s / %s read per token = %.1f tok/s" % (bw, g1(active), toks)) if left <= 0: @@ -1362,24 +1569,79 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, yield _res(eng, "cpu", False, "DRAM bandwidth gives %.1f tok/s, under the %g floor" % (toks, MIN_DRAM_TOK_S), lines) return - c, n = size_kv(kv, left, ladder, want_seqs, 1.0, relaxed) + c, n = size_kv(kv, _kv_room(eng, "cpu", left), ladder, want_seqs, 1.0, relaxed, host_ckpt=True) if not c: yield _res(eng, "cpu", False, "KV room %s holds no %d-token sequence" % (g1(left), ladder[-1]), lines) return - kvused = per_seq(kv, c) * n - if eng == "vllm": - kvused = int(math.ceil(kvused / GIB)) * GIB # VLLM_CPU_KVCACHE_SPACE is whole GiB + kvused = _kv_pool(eng, "cpu", per_seq(kv, c) * n) lines.append("KV: %d seqs x %s at %d tokens = %s" % (n, _mb(per_seq(kv, c)), c, g1(kvused))) - hb = int(m["serve_bytes"] + act + HOST_RUNTIME[eng] + kvused) - yield _res(eng, "cpu", True, "", lines, ctx=c, seqs=n, host_bytes=hb, kv_bytes=int(kvused), + ck = ckpt_seq(kv, c) * n + if ck: + lines.append("context checkpoints: %d seqs x %s = %s" % (n, _mb(ckpt_seq(kv, c)), g1(ck))) + hb = int(m["serve_bytes"] + act + HOST_RUNTIME[eng] + kvused + ck) + yield _res(eng, "cpu", True, "", lines, ctx=c, seqs=n, host_bytes=hb, kv_bytes=int(kvused), ckpt_bytes=ck, dram_tok_s=round(toks, 1) if toks else None, flags=_flags(eng, "cpu", c, n, 1, 0, [], kvused, m, 0, kv)) -def _flags(eng, place, ctx, seqs, k, gpu_use, cards, kvused, m, n_cpu_moe, kv=None): +def _llama_multi(m, kv, ladder, relaxed, want_seqs, use, ov, ovl, n_cpus, on_host, hfree, bw): + """llama.cpp on the cards in use, each holding its own layers, their KV and its own + overhead within the smallest budget among them (the device plugin, not the fit, orders + the cards inside the pod): a layer split (n_cpus [0]) or experts in DRAM (n_cpus 1..).""" + k = len(use) + place = "gpu-split" if list(n_cpus) == [0] else "moe-offload" + budget = min(c["budget"] for c in use) + room = budget - ov + lines = ["%s: layer il on card upper_bound(cumsum(-ts) / sum, il / (layers + 1)) (src/llama-model.cpp); " + "each card: budget %s (the smallest of them) - overhead %s = %s for its layers" + % ("+".join(c["card"] for c in use), g1(budget), ovl, g1(max(room, 0)))] + if on_host: + lines.append("lookup tables %s stay in host RAM" % g1(on_host)) + found = _llama_search(m, kv, ladder, relaxed, want_seqs, k, room, n_cpus) if room > 0 else None + if not found: + why = ("weights + KV for %d-token sequences do not fit %d card(s) of %s each, layer by layer" + % (ladder[-1], k, g1(budget))) + return _res("llama.cpp", place, False, why, lines, gpus=k) + c, n, n_cpu, ts = found + wl, kl = llama_layer_bytes(m, kv, c, n, n_cpu) + cards_b = [int(b + ov) for b in llama_cards(ts, [a + b for a, b in zip(wl, kl)])] + lines += _llama_card_lines(use, ts, wl, kl, ov, m) + if m["bytes"]["vision"]: + lines.append(" (%s holds the vision projector, %s: tools/mtmd/clip.cpp loads it on the first GPU)" + % (use[0]["card"], g1(m["bytes"]["vision"]))) + kvused = per_seq(kv, c) * n + lines.append("KV: %d seqs x %s at %d tokens = %s over the cards" % (n, _mb(per_seq(kv, c)), c, g1(kvused))) + # llama-server's context checkpoints stay in host RAM whichever card holds the layers. + ck = ckpt_seq(kv, c) * n + hb = int(HOST_RUNTIME["llama.cpp"] + on_host + ck) + toks = None + if n_cpu: + exp_layer = m["bytes"]["experts"] / max(m["layers"], 1) + dram_w = exp_layer * n_cpu + toks = _dram_rate(bw, dram_w * m["experts_per_tok"] / m["experts"]) + lines.append("experts of the first %d of %d layers in DRAM: %s" % (n_cpu, m["layers"], g1(dram_w))) + lines.append("DRAM decode bound: %g GB/s / (%s x %d/%d read per token) = %.0f tok/s (floor %g)" + % (bw, g1(dram_w), m["experts_per_tok"], m["experts"], toks, MIN_DRAM_TOK_S)) + hb += int(dram_w) + lines.append("host: experts %s + lookup tables %s + runtime %s + context checkpoints %d x %s = %s" + % (g1(dram_w), g1(on_host), g1(HOST_RUNTIME["llama.cpp"]), n, _mb(ckpt_seq(kv, c)), g1(hb))) + if toks < MIN_DRAM_TOK_S: + return _res("llama.cpp", place, False, "DRAM bandwidth gives %.1f tok/s, under the %g floor" + % (toks, MIN_DRAM_TOK_S), lines, gpus=k) + elif ck: + lines.append("context checkpoints: %d seqs x %s = %s in host RAM" % (n, _mb(ckpt_seq(kv, c)), g1(ck))) + if hb > hfree: + return _res("llama.cpp", place, False, "host side %s exceeds the host budget %s" % (g1(hb), g1(hfree)), + lines, gpus=k) + return _res("llama.cpp", place, True, "", lines, ctx=c, seqs=n, gpus=k, gpu_bytes=max(cards_b), card_bytes=cards_b, + host_bytes=hb, dram_tok_s=round(toks, 1) if toks else None, kv_bytes=int(kvused), ckpt_bytes=ck, + n_cpu_moe=n_cpu, flags=_flags("llama.cpp", place, c, n, k, max(cards_b), use, kvused, m, n_cpu, kv, ts)) + + +def _flags(eng, place, ctx, seqs, k, gpu_use, cards, kvused, m, n_cpu_moe, kv=None, ts=None): """Engine memory flags from the fit. Spellings from each engine's own argument parser: vLLM v0.32.0 vllm/engine/arg_utils.py, OVMS v2026.4.1 src/graph_export/graph_cli_parser.cpp, - llama.cpp 1242be6 common/arg.cpp.""" + llama.cpp v0.8.0 (b11960) common/arg.cpp.""" kv = kv or {} if eng == "vllm" and place == "cpu": return ["--max-model-len=%d" % ctx, "--max-num-seqs=%d" % seqs, @@ -1398,10 +1660,12 @@ def _flags(eng, place, ctx, seqs, k, gpu_use, cards, kvused, m, n_cpu_moe, kv=No # -c is the total context; with -np set explicitly each slot gets c/np (kv-unified is # on only when the slot count is auto). out = ["-c", str(ctx * seqs), "-np", str(seqs), "-ngl", "0" if place == "cpu" else "999"] - if k > 1: - out += ["-ts", ",".join(str(max(int(c["budget"] // GIB), 1)) for c in cards)] + if ts and k > 1: + out += ["-ts", ",".join(str(t) for t in ts)] # layers per card (llama_layer_devices) if place == "moe-offload": out += ["--cpu-moe"] if n_cpu_moe >= m["layers"] else ["--n-cpu-moe", str(n_cpu_moe)] + # -cram/--cache-ram, -ctxcp/--ctx-checkpoints: the host caches the fit counts (LLAMA_CTX_CHECKPOINTS) + out += ["--cache-ram", "0", "-ctxcp", str(LLAMA_CTX_CHECKPOINTS)] t = {"fp16": "f16"}.get(kv.get("dtype"), kv.get("dtype")) if t in ("f16", "bf16", "q8_0", "q4_0"): out += ["-ctk", t, "-ctv", t] # -ctk/--cache-type-k, -ctv/--cache-type-v diff --git a/docs/fit.md b/docs/fit.md index 36358501..67419485 100644 --- a/docs/fit.md +++ b/docs/fit.md @@ -68,20 +68,27 @@ in this order, and the first that fits wins: 1. one GPU; 2. several GPUs: vLLM tensor parallel (heads divisible by the size), or llama.cpp's - layer split; + layer split, sized card by card (below); OVMS serves one GPU; 3. MoE experts in DRAM (llama.cpp `--n-cpu-moe N`): only when the GPU placements do not fit, the CPU has AMX or AVX-512, and the DRAM side's bandwidth bound per token stays - above a floor. The fit takes the fewest layers whose experts leave the GPU; + above a floor. The fit takes the fewest layers whose experts leave the GPU, trying + 1, 2, ... of the free cards (largest budget first) and keeping the plan with the most + context, then the fewest such layers, then the most sequences, then the fewest cards; 4. CPU only. Per placement: ``` -weights = bytes of each tensor class to serve (MTP left out unless the GGUF holds it) +weights = bytes of each tensor class to serve (MTP left out unless the GGUF holds it, + as llama.cpp's conversion of safetensors does) KV per token = full layers x 2 x kv heads x head dim x bytes + indexer keys (sparse attention) - + sliding layers x 2 x kv heads x head dim x bytes, for at most the window + + sliding layers x 2 x kv heads x head dim x bytes, for: + vLLM at most the window + llama.cpp at most GGML_PAD(window + 512 ubatch, 256) tokens per slot + OVMS the whole context (OpenVINO GenAI has no sliding-window cache) state per seq = linear layers x (v heads x k dim x v dim x 4 B + conv state) +checkpoints = llama.cpp: 4 per slot in host RAM, each the slot's SWA cache and state overhead (GPU) = runtime + activations (batch x widest matmul x 2 x 2 B) + logits (seqs x vocab x 4 B) host reserve = max(10% of MemTotal, 4 GiB); host budget = min(MemAvailable, cgroup room) - reserve GPU reserve = max(5% of VRAM, 1 GiB); GPU budget = VRAM - in use by others - reserve @@ -92,12 +99,57 @@ Context first: the largest context, halving from the model's cap, at which 4 seq fit at once and that is at least 32768 tokens (or the cap). Only when no placement reaches that does the fit take less, down to one sequence of 4096 tokens, and the summary says it is below the target. Then as many sequences as fit at that context. +When no placement can be tried at all (every card busy or none with device-local memory, +and the CPU not allowed), the refusal says that no Intel GPU with device-local memory is +free. llama.cpp keeps the token embeddings and the n-gram table on the CPU by design (`src/llama-model.cpp`, input layer "always keep it on the CPU"), so those count as host bytes on every llama.cpp placement. OpenVINO's GPU plugin compiles from a host copy, so an OVMS GPU placement counts the weights as host bytes too. +A sliding-window layer's cache is engine-specific. llama.cpp v0.8.0 gives each slot +`GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch), 256)` cells (`src/llama-kv-cache-iswa.cpp`), +so Gemma 4's 1024-token window holds 1536 cells. OpenVINO GenAI 2026.4 has only a +full-attention KV cache and a linear-attention state cache +(`src/cpp/src/continuous_batching/cache/cache_type.hpp`), and one KV block spans every +attention layer, so OVMS holds every token in the sliding-window layers too: Gemma 4 +costs 225280 B per token there (5 full and 25 sliding layers, K and V at 2 B). + +llama-server v0.8.0 keeps two host caches beside the KV cache: a prompt cache of up to +8192 MiB (`--cache-ram`) and up to 32 context checkpoints per slot (`-ctxcp`), each a +host copy of the slot's SWA cache and recurrent state (`tools/server/server-context.cpp`, +`create_checkpoint`). The fit turns the prompt cache off (`--cache-ram 0`), keeps 4 +checkpoints per slot (a request makes them at the last user message and just before its +prompt's end, which is what the next turn restores from), and counts slots x 4 x one +checkpoint in the host bytes and the pod limit. A model with no sliding-window and no +linear-attention layers makes none. + +On several GPUs llama.cpp places whole layers, so the fit sizes each card from the +layers it gets, the way llama.cpp v0.8.0 assigns them with `-ngl 999` +(`src/llama-model.cpp`, `load_tensors`). Layer `il` goes to the device +`upper_bound(cumsum(ts) / sum(ts), il / (layers + 1))`, where layers is the GGUF's block +count (`n_layer_all`): the trunk, then the MTP (nextn) blocks a GGUF holds. Qwen3.8 27B's +GGUF keeps one (`qwen35.block_count` 65 for 64 trunk layers), and so does the GGUF the agent +converts from safetensors: `convert_hf_to_gguf.py` runs without `--no-mtp`, so +Qwen3.8-Flash-Next's has 49 blocks for 48 trunk layers. An MTP block counts its MTP +tensors and no KV, since the cache covers the trunk only (`il < n_layer()`). The output +layer (`il = layers`, holding `lm_head`) goes the same way, which is the last card. A +model with tied embeddings (Gemma 4) has no `lm_head`: llama.cpp loads +`token_embd` a second time as the output tensor on that device +(`src/llama-model-loader.cpp`, `TENSOR_DUPLICATED`), so the output layer counts that copy, +on one card as on several. Each layer's weights are its share of the class totals, minus its experts +when it is one of the first `--n-cpu-moe` layers. The vision projector counts on the first +card: `tools/mtmd/clip.cpp` loads it on the first GPU. Each layer's KV and recurrent state +sit on its card, a sliding-window layer's at its padded cell count (above); the context +checkpoints stay in host RAM. Every card also carries its own overhead. +The fit emits `-ts` as whole layer counts summing to layers + 1, so each card gets exactly +the layers it was sized for. It keeps every card within the smallest budget of the cards +used, because the device plugin, not the fit, decides which card is device 0 inside the +pod, and it balances the largest card's load. When no split fits, the placement is refused. +`--explain` lists each card's layers, weights, KV and total, and `card_bytes` in `--json` +has the same numbers. + The overhead is an estimate from the formula, not a measurement; `K3D_GPU_FIT_OVERHEAD_GIB` replaces it with a measured value. @@ -106,12 +158,15 @@ The overhead is an estimate from the formula, not a measurement; | Engine | Flags (spellings from each engine's own argument parser) | | --- | --- | | vLLM v0.32.0 | `--tensor-parallel-size`, `--max-model-len`, `--max-num-seqs`, `--gpu-memory-utilization` (vLLM still profiles activations inside that share), `--kv-cache-dtype fp8` when the checkpoint's `kv_cache_scheme` made the fit count one byte. CPU build (EAT): `VLLM_CPU_KVCACHE_SPACE` | -| OVMS 2026.4.1 | `--max_num_seqs`, `--cache_size` (a static pool: the dynamic one grows past the fit) | -| llama.cpp v0.8.0 | `-c` (context x slots), `-np`, `-ngl`, `-ts`, `--n-cpu-moe` or `--cpu-moe`, `-ctk`/`-ctv` | +| OVMS 2026.4.1 | `--max_num_seqs`, `--cache_size` (a static pool in whole GiB, rounded up to hold every sequence: the dynamic one grows past the fit) | +| llama.cpp v0.8.0 | `-c` (context x slots), `-np`, `-ngl`, `-ts` (layers per card), `--n-cpu-moe` or `--cpu-moe`, `--cache-ram 0`, `-ctxcp 4`, `-ctk`/`-ctv` | `agent up` applies them, sets the pod's memory request and limit to the fit, and annotates the pod `k3d-gpu.io/agent-fit`. `K3D_GPU_AGENT_MAX_LEN`, `K3D_GPU_AGENT_MAX_NUM_SEQS` -and `K3D_GPU_AGENT_GPUS` are inputs the fit checks, not values that replace it. With +and `K3D_GPU_AGENT_GPUS` are inputs the fit checks, not values that replace it. +`K3D_GPU_AGENT_GPUS` reaches the fit unless OVMS serves the model (the backend set to OVMS +or, with it unset, an OpenVINO IR): an OVMS pod holds one GPU, and `fit --gpus 2` with OVMS +is refused as "OVMS serves one GPU". With `K3D_GPU_AGENT_BACKEND` unset, the engine the fit picks serves the model: OVMS, vLLM or llama-server. With `--model`, the model optimizer first picks the variant for the default engine (OVMS: Gemma 4's OpenVINO IR, for example). When that engine has nothing to serve, diff --git a/scripts/k3d-gpu b/scripts/k3d-gpu index efd27ea6..d2f9c55c 100755 --- a/scripts/k3d-gpu +++ b/scripts/k3d-gpu @@ -423,7 +423,8 @@ Environment: K3D_GPU_AUTOQUANT auto | off: on OVMS/CPU with no K3D_GPU_AGENT_MODEL, serve the precision measured for this host fingerprint (default: auto) K3D_GPU_AUTOQUANT_CACHE measured-pick cache (default: \$XDG_STATE_HOME/k3d-gpu/autoquant.json) - K3D_GPU_AGENT_GPUS vllm/llm-scaler: GPUs (or Gaudis) to shard the model across (default: 1) + K3D_GPU_AGENT_GPUS vllm/llm-scaler/llama.cpp: GPUs (or Gaudis) to shard or split the model + across (default: 1; OVMS serves one GPU) K3D_GPU_AGENT_MAX_LEN vllm/llm-scaler: max context tokens (default: 8192) K3D_GPU_FIT auto | on | off: size the agent model with 'fit' before agent up renders it (default: auto = a model you chose; on also sizes the @@ -7634,8 +7635,9 @@ agent_config # OVMS only: DEBUG also prints the compiled model's EXECUTION_DEVICES (e.g. GPU.0: # Intel(R) UHD Graphics 770), and every HTTP request, readiness probes included. AGENT_LOG_LEVEL="${K3D_GPU_AGENT_LOG_LEVEL:-INFO}" -# vLLM only: GPUs (or Gaudis) the model is sharded across (--tensor-parallel-size), -# and the longest context, sized so the default model's KV cache fits a 12 GB card. +# vLLM: GPUs (or Gaudis) the model is sharded across (--tensor-parallel-size); llama.cpp: +# the cards its layers are split over. And the longest context, sized so the default +# model's KV cache fits a 12 GB card. AGENT_GPUS="${K3D_GPU_AGENT_GPUS:-1}" AGENT_MAX_LEN="${K3D_GPU_AGENT_MAX_LEN:-8192}" @@ -11200,7 +11202,17 @@ agent_fit() { || { fit_soft "${mode}" "fit: no metadata for ${AGENT_MODEL} (offline, private or gated: set HF_TOKEN)"; return 0; } [ -z "${K3D_GPU_AGENT_MAX_LEN:-}" ] || args+=(--ctx "${K3D_GPU_AGENT_MAX_LEN}") [ -z "${K3D_GPU_AGENT_MAX_NUM_SEQS:-}" ] || args+=(--seqs "${K3D_GPU_AGENT_MAX_NUM_SEQS}") - [ -z "${K3D_GPU_AGENT_GPUS:-}" ] || args+=(--gpus "${K3D_GPU_AGENT_GPUS}") + # K3D_GPU_AGENT_GPUS shards a vLLM model or splits a llama.cpp one; an OVMS pod holds + # one GPU whatever it says. OVMS serves the model when the backend is set to it or, with + # the backend unset (the fit picks the engine), when it is an OpenVINO IR: only OVMS + # serves one, and the fit picks vLLM or llama.cpp for any other weights. + if [ -n "${K3D_GPU_AGENT_GPUS:-}" ]; then + if [ -n "${AGENT_BACKEND_SET}" ]; then + [ "${AGENT_APP}" = ovms ] || args+=(--gpus "${K3D_GPU_AGENT_GPUS}") + else + [ "$(fit_dir_format "${dir}")" = openvino ] || args+=(--gpus "${K3D_GPU_AGENT_GPUS}") + fi + fi # agent up replaces the running server (the Deployment is Recreate, and a backend # switch deletes the other one): it is stopped before the probe, so the memory it holds # is measured as free, not as someone else's. A model that then does not fit gets the @@ -11610,7 +11622,12 @@ agent_model_format() { local d command -v python3 >/dev/null 2>&1 || return 0 d=$(fit_model_dir "$1" "${K3D_GPU_AGENT_REVISION:-main}" 2>/dev/null) || return 0 - python3 -c 'import json, sys; print(json.load(open(sys.argv[1])).get("format") or "")' "${d}/weights.json" 2>/dev/null || true + fit_dir_format "${d}" +} + +# fit_dir_format : the weight format a fit metadata directory names, or nothing. +fit_dir_format() { + python3 -c 'import json, sys; print(json.load(open(sys.argv[1])).get("format") or "")' "$1/weights.json" 2>/dev/null || true } # Applies the optimizer's choices for AGENT_OPT_MODEL to the agent settings. diff --git a/tests/fit_test.py b/tests/fit_test.py index 341c6e4a..3590fe0a 100755 --- a/tests/fit_test.py +++ b/tests/fit_test.py @@ -12,6 +12,7 @@ class bytes from the real safetensors/GGUF headers at a pinned revision), Usage: python3 tests/fit_test.py """ +import bisect import gzip import json import os @@ -246,6 +247,11 @@ def config(model): return json.load(f) +def weights(model): + with open(os.path.join(mdir(model), "weights.json"), encoding="utf-8") as f: + return json.load(f) + + class KV(unittest.TestCase): """KV bytes from config.json, worked out by hand from each config's fields.""" @@ -258,6 +264,19 @@ def test_gemma4_hybrid_swa(self): self.assertEqual(kv["state_per_seq"], 0) self.assertEqual(fit.per_seq(kv, 131072), 5 * 2 * 2 * 512 * 2 * 131072 + 25 * 2 * 8 * 256 * 2 * 1024) + def test_swa_cache_per_engine(self): + cfg = config("google__gemma-4-26B-A4B-it") + full, swa = 5 * 2 * 2 * 512 * 2, 25 * 2 * 8 * 256 * 2 + self.assertEqual(full + swa, 225280) + # vLLM: the window + self.assertEqual(fit.per_seq(fit.kv_profile(cfg, 2, "vllm"), 131072), full * 131072 + swa * 1024) + # llama.cpp v0.8.0: GGML_PAD(min(n_ctx_seq, n_swa + n_ubatch 512), 256) cells per stream + kv = fit.kv_profile(cfg, 2, "llama.cpp") + self.assertEqual(fit.per_seq(kv, 131072), full * 131072 + swa * 1536) + self.assertEqual(fit.per_seq(kv, 1024), (full + swa) * 1024) + # OVMS (OpenVINO GenAI 2026.4): no sliding-window cache type, every layer at full context + self.assertEqual(fit.per_seq(fit.kv_profile(cfg, 2, "ovms"), 131072), 225280 * 131072) + def test_qwen38_gated_deltanet(self): kv = fit.kv_profile(config("Qwen__Qwen3.8-27B"), 2) self.assertEqual(kv["full_per_token"], 16 * 2 * 4 * 256 * 2) @@ -378,10 +397,28 @@ def test_gemma_qat_gguf(self): d = plan("xeon6741p-4xb70", "google__gemma-4-26B-A4B-it-qat-q4_0-gguf") self.assertTrue(d["fits"], d["summary"]) self.assertEqual((d["engine"], d["placement"], d["gpus"]), ("llama.cpp", "gpu", 1)) - self.assertEqual((d["ctx"], d["seqs"]), (131072, 6)) + # each slot's SWA cache is GGML_PAD(1024 + 512, 256) = 1536 cells, not the 1024 window + self.assertEqual((d["ctx"], d["seqs"]), (131072, 5)) self.assertFalse(d["degraded"]) self.assertIn("-ngl", d["flags"]) + def test_gemma_ovms_cache_holds_every_token(self): + # OVMS gives Gemma 4's 25 sliding-window layers the whole context: 225280 B per token, + # and --cache_size (whole GiB) holds context x sequences at that, on every guard rung. + one = max(c["budget"] for c in fit.gpu_budget(host("xeon6741p-4xb70"))[0]) + for hname, dev in (("xeon6741p-4xb70", "gpu"), ("xeon6741p-4xb70", "cpu"), ("desktop-i9-nogpu", None)): + d = plan(hname, "OpenVINO__gemma-4-26b-a4b-it-int4-ov", device=dev) + self.assertTrue(d["fits"], (hname, d["summary"])) + self.assertEqual(d["engine"], "ovms") + cache = [int(f.split("=")[1]) for f in d["flags"] if f.startswith("--cache_size=")][0] + self.assertGreaterEqual(cache * GIB, d["ctx"] * d["seqs"] * 225280, (hname, d["flags"])) + self.assertEqual(cache * GIB, d["kv_bytes"], hname) + if d["placement"] == "gpu": + self.assertLessEqual(d["gpu_bytes"], one) + for r, (s, c) in enumerate(d["guard"]["rungs"]): + k = dict(line.split("=", 1) for line in fit.knobs(d, r).splitlines()) + self.assertGreaterEqual(int(k["cache_gib"]) * GIB, s * c * 225280, (hname, r)) + def test_vllm_host_side_per_rank(self): d = plan("xeon6741p-4xb70", "Qwen__Qwen3.8-27B") # one runtime per tensor-parallel rank, plus the /dev/shm emptyDir @@ -439,6 +476,58 @@ def test_no_moe_offload_without_amx_or_avx512(self): self.assertIn("exceed the host budget", d["reason"]) +class LlamaHostCaches(unittest.TestCase): + """llama-server v0.8.0 keeps an 8192 MiB host prompt cache (--cache-ram) and up to 32 + context checkpoints per slot (-ctxcp), each a slot's SWA cache or recurrent state, in + host RAM. The fit turns the prompt cache off, bounds the checkpoints, and counts them.""" + + def flag(self, d, name): + i = d["flags"].index(name) + return int(d["flags"][i + 1]) + + def test_gemma_qat_cpu_on_the_desktop(self): + d = plan("desktop-i9-nogpu", "google__gemma-4-26B-A4B-it-qat-q4_0-gguf") + self.assertTrue(d["fits"], d["summary"]) + self.assertEqual((d["engine"], d["placement"]), ("llama.cpp", "cpu")) + self.assertEqual(self.flag(d, "--cache-ram"), 0) + k = self.flag(d, "-ctxcp") + self.assertGreaterEqual(k, 1) + # one checkpoint: 25 SWA layers x 2 x 8 kv heads x 256 x 2 B over the slot's SWA + # cache, GGML_PAD(1024 window + 512 ubatch, 256) = 1536 cells + ckpt = 25 * 2 * 8 * 256 * 2 * 1536 + self.assertGreaterEqual(d["ckpt_bytes"], d["seqs"] * k * ckpt) + cfg = config("google__gemma-4-26B-A4B-it-qat-q4_0-gguf") + w = json.load(open(os.path.join(mdir("google__gemma-4-26B-A4B-it-qat-q4_0-gguf"), "weights.json"))) + m = fit.model_profile(cfg, w) + self.assertGreaterEqual(d["host_bytes"], m["serve_bytes"] + fit.HOST_RUNTIME["llama.cpp"] + d["kv_bytes"] + + d["seqs"] * k * ckpt) + # the pod limit holds the checkpoints and stays inside the host budget + budget, _, _ = fit.host_budget(host("desktop-i9-nogpu")) + self.assertGreaterEqual(int(d["k8s"]["memory_limit"][:-2]) * GIB, d["host_bytes"]) + self.assertLessEqual(int(d["k8s"]["memory_limit"][:-2]) * GIB, budget) + # and on every guard rung + for r, (rs, rc) in enumerate(d["guard"]["rungs"]): + self.assertGreaterEqual(int(d["guard"]["rung_memory_limit"][r][:-2]) * GIB, + d["host_bytes"] - d["kv_bytes"] - d["ckpt_bytes"] + + d["guard"]["rung_kv_bytes"][r] + rs * k * ckpt, r) + + def test_recurrent_state_checkpoints_on_the_host(self): + # Flash-Next: 36 gated-DeltaNet layers, each checkpoint the per-sequence state + d = plan("xeon6741p-4xb70", "ggml-org__Qwen3.8-Flash-Next-GGUF") + self.assertTrue(d["fits"], d["summary"]) + k = self.flag(d, "-ctxcp") + state = 36 * (48 * 128 * 128 * 4 + 3 * (2 * 16 * 128 + 48 * 128) * 2) + self.assertEqual(d["ckpt_bytes"], d["seqs"] * k * state) + self.assertIn("--cache-ram", d["flags"]) + + def test_full_attention_makes_no_checkpoints(self): + cfg = config("Qwen__Qwen3.8-27B") + cfg = dict(cfg, text_config=dict(cfg.get("text_config") or cfg, layer_types=None)) + kv = fit.kv_profile(cfg, 2) + kv["ckpts"] = 4 + self.assertEqual(fit.ckpt_seq(kv, 65536), 0) + + class FitLaptop(unittest.TestCase): """Core Ultra 7 258V, 16 GB, 9 GB available, swap in use.""" @@ -501,6 +590,33 @@ def test_kv_scheme_follows_checkpoint(self): self.assertTrue(any("kv_cache_scheme" in s for s in d["explain"])) +class OvmsOneGpu(unittest.TestCase): + """OVMS serves one GPU: --gpus 2 is refused, never sized as a layer split of two cards.""" + + def test_two_gpus_asked_is_refused(self): + for mname in ("OpenVINO__gemma-4-26b-a4b-it-int4-ov", "OpenVINO__Qwen3.8-27B-int4-ov"): + d = plan("xeon6741p-4xb70", mname, gpus=2, device="gpu") + self.assertFalse(d["fits"], (mname, d["summary"])) + self.assertIn("OVMS serves one GPU", d["reason"]) + + def test_cache_size_within_one_card(self): + one = max(c["budget"] for c in fit.gpu_budget(host("xeon6741p-4xb70"))[0]) + for gpus in (None, 1, 2): + d = plan("xeon6741p-4xb70", "OpenVINO__gemma-4-26b-a4b-it-int4-ov", gpus=gpus, device="gpu") + if not d["fits"]: + continue + self.assertEqual((d["placement"], d["gpus"]), ("gpu", 1), gpus) + cache = [int(f.split("=")[1]) for f in d["flags"] if f.startswith("--cache_size=")] + self.assertLessEqual(cache[0] * GIB, one, (gpus, d["flags"])) + + def test_no_free_gpu_says_so(self): + # device gpu on a host with no Intel GPU of its own memory: no placement is tried at all + d = plan("desktop-i9-nogpu", "OpenVINO__Qwen3.8-27B-int4-ov", device="gpu") + self.assertFalse(d["fits"]) + self.assertIn("no Intel GPU with device-local memory is free", d["reason"]) + self.assertNotIn("release notes", d["reason"]) + + class Guard(unittest.TestCase): """guard_step under simulated PSI: a CPU-placed server sheds rungs; a VRAM one holds or stops.""" @@ -703,5 +819,245 @@ def test_falls_to_fewer_bits_when_int4_does_not_fit(self): self.assertTrue(any("IQ4_NL" in s for s in d["explain"]) and any("Q2_K" in s for s in d["explain"])) +def gguf_nextn(cfg, w): + """MTP (nextn) blocks a GGUF holds after its trunk: the config's count when the GGUF has + MTP tensors (llama.cpp's converter keeps or drops them all).""" + tc = cfg.get("text_config") or cfg + if w["format"] != "gguf" or not w["classes"].get("mtp", {}).get("bytes"): + return 0 + return tc.get("mtp_num_hidden_layers") or tc.get("num_nextn_predict_layers") or 1 + + +def llama_cards(d, cfg, w, kv_layer): + """Each card's GPU bytes for a llama.cpp plan, worked out here from its flags the way + llama.cpp v0.8.0 assigns them (src/llama-model.cpp load_tensors, -ngl past the layer + count): layer il on device upper_bound(cumsum(ts) / sum(ts), il / (n_layer_all + 1)), the + output layer (il = n_layer_all) the same way, the experts of the first --n-cpu-moe layers + in host RAM, each layer's KV on its device, and each card's own overhead. n_layer_all is + the GGUF's block count: the trunk, then the MTP blocks a GGUF holds (gguf_nextn). Weights + per trunk layer are the class totals over the trunk; an MTP block holds its share of the + MTP tensors and no KV; the output layer holds lm_head (with tied embeddings a copy of + token_embd), and the vision projector is on the first GPU (tools/mtmd/clip.cpp). + kv_layer(il, ctx) is one sequence's KV of layer il.""" + tc = cfg.get("text_config") or cfg + n = tc["num_hidden_layers"] + nx = gguf_nextn(cfg, w) + f = d["flags"] + ts = [int(x) for x in f[f.index("-ts") + 1].split(",")] + ncpu = int(f[f.index("--n-cpu-moe") + 1]) if "--n-cpu-moe" in f else n if "--cpu-moe" in f else 0 + cls = {k: v["bytes"] for k, v in w["classes"].items()} + cum, c = [], 0 + for t in ts: + c += t + cum.append(c / sum(ts)) + cards = [0] * len(ts) + cards[0] += cls.get("vision", 0) + for il in range(n + nx + 1): + dev = bisect.bisect_right(cum, il / (n + nx + 1)) + if il == n + nx: + # the output weight: lm_head, or with tied embeddings (no output.weight) a copy of + # token_embd that llama.cpp loads as the output tensor (src/llama-model-loader.cpp) + cards[dev] += cls.get("lm_head", 0) or cls.get("embed", 0) + elif il >= n: + cards[dev] += cls["mtp"] / nx + else: + cards[dev] += (cls.get("other", 0) / n + (cls.get("experts", 0) / n if il >= ncpu else 0) + + kv_layer(il, d["ctx"]) * d["seqs"]) + m = fit.model_profile(cfg, w) + ov = fit._overhead("llama.cpp", m, fit.TARGET_SEQS, None)[0] + return ts, [int(b + ov) for b in cards] + + +def flash_next_kv(il, ctx): + # every 4th layer full attention: 2 x 2 kv heads x 256 x 2 B + indexer 128 x 2 B per token; + # the rest gated DeltaNet: 48 x 128 x 128 x 4 B state + 3 x 10240 conv ch x 2 B per sequence + return (2 * 2 * 256 * 2 + 128 * 2) * ctx if il % 4 == 3 else 48 * 128 * 128 * 4 + 3 * 10240 * 2 + + +def qwen38_27b_kv(il, ctx): + # every 4th layer full attention: 2 x 4 kv heads x 256 x 2 B per token; the rest gated + # DeltaNet: 48 x 128 x 128 x 4 B state + 3 x (2 x 16 x 128 + 48 x 128) conv ch x 2 B per sequence + return 2 * 4 * 256 * 2 * ctx if il % 4 == 3 else 48 * 128 * 128 * 4 + 3 * (2 * 16 * 128 + 48 * 128) * 2 + + +def gemma4_kv(il, ctx): + # every 6th layer global: 2 x 2 kv heads x 512 x 2 B per token; the rest a 1024-token window, + # whose llama.cpp v0.8.0 cache is GGML_PAD(min(n_ctx_seq, 1024 + 512 ubatch), 256) cells per + # slot (src/llama-kv-cache-iswa.cpp), n_ctx_seq itself padded to 256 (src/llama-context.cpp) + if il % 6 == 5: + return 2 * 2 * 512 * 2 * ctx + return 2 * 8 * 256 * 2 * min((ctx + 255) // 256 * 256, 1536) + + +class LlamaSplit(unittest.TestCase): + """llama.cpp over several GPUs: each card's own layers, KV and overhead within its budget.""" + + def budget(self, k): + return min(c["budget"] for c in fit.gpu_budget(host("xeon6741p-4xb70"))[0][:k]) + + def test_layer_devices(self): + # upper_bound(cumsum(ts)/sum, il/(n_layer+1)): -ts 30,30 over 48 layers puts 25 on the first card + self.assertEqual(fit.llama_layer_devices([30, 30], 48), [0] * 25 + [1] * 24) + # integer layer counts that sum to n_layer + 1 give exactly those counts, the output last + self.assertEqual(fit.llama_layer_devices([28, 21], 48), [0] * 28 + [1] * 21) + self.assertEqual(fit.llama_layer_devices([10, 0, 21], 30), [0] * 10 + [2] * 21) + + def test_old_split_overflows_the_second_card(self): + # The plan this replaces: -ts 30,30 --n-cpu-moe 28 at 262144 x 4 puts ~43.5 GiB on a 32 GiB card. + cfg = config(FLASH_GGUF) + w = weights(FLASH_GGUF)["groups"]["IQ4_NL"] + old = {"flags": ["-ts", "30,30", "--n-cpu-moe", "28"], "ctx": 262144, "seqs": 4} + _, cards = llama_cards(old, cfg, w, flash_next_kv) + self.assertGreater(cards[1], 32 * GIB) + + def check(self, d, cfg, w, kv_layer): + n = (cfg.get("text_config") or cfg)["num_hidden_layers"] + ts, cards = llama_cards(d, cfg, w, kv_layer) + self.assertEqual(sum(ts), n + gguf_nextn(cfg, w) + 1, ts) + self.assertEqual(len(ts), d["gpus"]) + budget = self.budget(d["gpus"]) + for i, b in enumerate(cards): + self.assertLessEqual(b, budget, (i, ts, d["flags"])) + # the fit's own per-card bytes agree with the ones worked out here + self.assertEqual(len(d["card_bytes"]), len(ts)) + for got, want in zip(d["card_bytes"], cards): + self.assertAlmostEqual(got / GIB, want / GIB, delta=0.01) + self.assertEqual(d["gpu_bytes"], max(d["card_bytes"])) + # the KV over the cards is the plan's KV + self.assertAlmostEqual(sum(kv_layer(il, d["ctx"]) for il in range(n)) * d["seqs"] / GIB, d["kv_bytes"] / GIB, + delta=0.01) + + def test_flash_next_iq4nl_cards_within_budget(self): + d = plan("xeon6741p-4xb70", FLASH_GGUF) + self.assertTrue(d["fits"], d["summary"]) + self.assertEqual((d["placement"], d["gpus"]), ("moe-offload", 2)) + w = weights(FLASH_GGUF)["groups"]["IQ4_NL"] + self.check(d, config(FLASH_GGUF), w, flash_next_kv) + + def test_gemma_layer_split_cards_within_budget(self): + g = "google__gemma-4-26B-A4B-it-qat-q4_0-gguf" + d = plan("xeon6741p-4xb70", g, ctx=262144, seqs=6) + self.assertTrue(d["fits"], d["summary"]) + self.assertEqual((d["placement"], d["gpus"]), ("gpu-split", 2)) + self.check(d, config(g), weights(g), gemma4_kv) + + def test_mtp_blocks_take_layer_slots(self): + # Intel/Qwen3.8-27B-q4km-AutoRound's GGUF holds Qwen3.8's MTP block after its 64-layer + # trunk (its header at 4b30b30e: qwen35.block_count 65, qwen35.nextn_predict_layers 1, + # blk.64.nextn.*). llama.cpp v0.8.0 assigns devices over all 65 blocks plus the output + # layer (src/llama-model.cpp, n_layer_all) and gives the MTP block no KV (the cache + # covers il < n_layer()), so -ts counts 66 layers: counts summing to 65 put one more + # trunk layer on the first card than the fit sized it for. + g = "Intel__Qwen3.8-27B-q4km-AutoRound" + d = plan("xeon6741p-4xb70", g, gpus=2, device="gpu") + self.assertTrue(d["fits"], d["summary"]) + self.assertEqual((d["engine"], d["placement"], d["gpus"]), ("llama.cpp", "gpu-split", 2)) + self.check(d, config(g), weights(g), qwen38_27b_kv) + self.assertTrue(any("layer 64" in s and "MTP" in s for s in d["explain"]), d["explain"]) + + def test_tied_output_copy_on_one_card(self): + # Gemma 4 ties its embeddings: no output.weight, so llama.cpp loads token_embd a second + # time as the output tensor, on the output layer's device (src/llama-model-loader.cpp, + # TENSOR_DUPLICATED). On one card that copy is GPU bytes; the input copy stays on the host. + g = "google__gemma-4-26B-A4B-it-qat-q4_0-gguf" + d = plan("xeon6741p-4xb70", g) + self.assertEqual((d["engine"], d["placement"], d["gpus"]), ("llama.cpp", "gpu", 1)) + m = fit.model_profile(config(g), weights(g)) + self.assertEqual(m["bytes"]["lm_head"], 0) + ov = fit._overhead("llama.cpp", m, fit.TARGET_SEQS, None)[0] + want = m["serve_bytes"] - m["lookup_bytes"] + m["bytes"]["embed"] + ov + d["kv_bytes"] + self.assertAlmostEqual(d["gpu_bytes"] / GIB, want / GIB, delta=0.01) + self.assertLessEqual(d["gpu_bytes"], max(c["budget"] for c in fit.gpu_budget(host("xeon6741p-4xb70"))[0])) + + def test_split_counts_the_context_checkpoints_on_the_host(self): + # llama-server keeps -ctxcp checkpoints of each slot's SWA cache in host RAM whatever + # card holds the layers: a split plan counts them in its host bytes, as one card does. + g = "google__gemma-4-26B-A4B-it-qat-q4_0-gguf" + d = plan("xeon6741p-4xb70", g, ctx=262144, seqs=6) + self.assertEqual(d["placement"], "gpu-split") + f = d["flags"] + self.assertEqual(f[f.index("--cache-ram") + 1], "0") + k = int(f[f.index("-ctxcp") + 1]) + ckpt = 25 * 2 * 8 * 256 * 2 * 1536 # 25 SWA layers, K and V at 2 B, over 1536 cells + self.assertEqual(d["ckpt_bytes"], d["seqs"] * k * ckpt) + m = fit.model_profile(config(g), weights(g)) + self.assertEqual(d["host_bytes"], int(fit.HOST_RUNTIME["llama.cpp"] + m["lookup_bytes"] + d["ckpt_bytes"])) + self.assertGreaterEqual(int(d["k8s"]["memory_limit"][:-2]) * GIB, d["host_bytes"]) + + def test_layer_bytes_are_the_bytes_to_serve(self): + # Sized from safetensors for a GGUF conversion: vLLM and OVMS leave the MTP head out + # (model_profile serve_bytes), but the GGUF the agent converts keeps it + # (convert_hf_to_gguf.py without --no-mtp), so llama.cpp's profile serves it as one + # more block after the trunk. The vision projector loads on the first GPU + # (tools/mtmd/clip.cpp takes the first GPU backend), not on the output layer's card. + cfg = config("Qwen__Qwen3.8-Flash-Next") + m = fit.model_profile(cfg, weights("Qwen__Qwen3.8-Flash-Next")) + self.assertGreater(m["bytes"]["mtp"], GIB) + self.assertGreater(m["bytes"]["vision"], GIB / 2) + lm = fit.llama_profile(m) + self.assertEqual((m["nextn"], lm["nextn"]), (0, 1)) + self.assertEqual(lm["serve_bytes"], m["serve_bytes"] + m["bytes"]["mtp"]) + wl, kl = fit.llama_layer_bytes(lm, fit.kv_profile(cfg, 2), 4096, 1, 0) + self.assertEqual(len(wl), 48 + 1 + 1) + self.assertAlmostEqual(sum(wl), lm["serve_bytes"] - lm["lookup_bytes"], delta=1) + self.assertAlmostEqual(wl[48], m["bytes"]["mtp"], delta=1) + self.assertEqual(kl[48], 0) + self.assertAlmostEqual(wl[-1], m["bytes"]["lm_head"], delta=1) + self.assertAlmostEqual(wl[0] - wl[1], m["bytes"]["vision"], delta=1) + # a GGUF is served as it is + g = fit.model_profile(config(FLASH_GGUF), weights(FLASH_GGUF)["groups"]["IQ4_NL"]) + self.assertIs(fit.llama_profile(g), g) + + def test_conversion_keeps_the_mtp_blocks(self): + # The agent converts safetensors only llama.cpp runs here (Qwen3.8-Flash-Next) with + # convert_hf_to_gguf.py and no --no-mtp, so the GGUF it serves holds the MTP block: + # llama.cpp v0.8.0 conversion/qwen.py (_QwenMtpMixin) extends block_count by + # mtp_num_hidden_layers. llama.cpp assigns devices over that block too, so -ts counts + # it as for a GGUF repo that holds one, and the card it lands on holds its tensors. + g = "Qwen__Qwen3.8-Flash-Next" + d = plan("xeon6741p-4xb70", g) + self.assertTrue(d["fits"], d["summary"]) + self.assertEqual((d["engine"], d["placement"]), ("llama.cpp", "moe-offload")) + self.assertEqual(d["convert"]["args"], ["--outtype", "bf16"]) + self.check(d, config(g), dict(weights(g), format="gguf"), flash_next_kv) # the converted GGUF + self.assertTrue(any("layer 48 is the GGUF's MTP block" in s for s in d["explain"]), d["explain"]) + + def test_moe_offload_takes_the_best_number_of_cards(self): + # Every card is held to the smallest budget of the cards used, so two smaller cards + # beside two large ones can leave less room than the two large ones alone. The fit + # tries 1, 2, ... of the free cards by budget and keeps the plan with the fewest + # CPU-expert layers, not every free card. + h = host("xeon6741p-4xb70") + for g, used in zip(h["gpus"], (0, 0, 12, 14)): + g["vram_used"], g["users"] = used * GIB, [] + model = mdir("Intel__Qwen3.8-Flash-Next-W4A16-AutoRound") + cfg = config("Intel__Qwen3.8-Flash-Next-W4A16-AutoRound") + n = (cfg.get("text_config") or cfg)["num_hidden_layers"] + + def ncpu(d): + f = d["flags"] + return int(f[f.index("--n-cpu-moe") + 1]) if "--n-cpu-moe" in f else n + + kw = dict(ctx=262144, seqs=6, device="gpu") + auto = fit.plan_with_variants(h, model, [], **kw) + self.assertTrue(auto["fits"], auto["summary"]) + self.assertEqual(auto["placement"], "moe-offload") + each = [fit.plan_with_variants(h, model, [], gpus=k, **kw) for k in range(1, 5)] + self.assertEqual(ncpu(auto), min(ncpu(d) for d in each if d["fits"])) + self.assertLess(auto["gpus"], 4) + budget = min(c["budget"] for c in fit.gpu_budget(h)[0][:auto["gpus"]]) + self.assertLessEqual(max(auto["card_bytes"]), budget) + + def test_moe_offload_takes_the_cards_asked_for(self): + # --gpus 3 on the Xeon, where two cards are free: refused, not two cards. + d = plan("xeon6741p-4xb70", FLASH_GGUF, gpus=3, device="gpu") + self.assertFalse(d["fits"]) + self.assertIn("3 GPUs asked, 2 not busy", d["reason"]) + + def test_too_much_for_any_split_is_refused(self): + d = plan("xeon6741p-4xb70", "google__gemma-4-26B-A4B-it-qat-q4_0-gguf", ctx=262144, seqs=12, device="gpu") + self.assertFalse(d["fits"]) + + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/tests/fit_test.sh b/tests/fit_test.sh index bdb42439..0c5f58ef 100755 --- a/tests/fit_test.sh +++ b/tests/fit_test.sh @@ -69,6 +69,31 @@ if [ "${rc}" -eq 0 ] && python3 -c 'import json, sys; d = json.loads(sys.argv[1] ok "fit --json: a fit/v1 document, from a local model directory" else fail "fit --json: a fit/v1 document" "${out}"; fi +# llama-server keeps a host prompt cache (8192 MiB by default) and context checkpoints per +# slot (32 by default, each the slot's SWA cache): the fit turns the first off, bounds the +# second, and counts the checkpoints in the host bytes and the pod limit. +out=$(bash "${K3D_GPU}" fit --host "${DESK}" --model google/gemma-4-26B-A4B-it-qat-q4_0-gguf --json 2>&1); rc=$? +if [ "${rc}" -eq 0 ] && python3 -c ' +import json, sys +d = json.loads(sys.argv[1]) +f = d["flags"] +assert d["placement"] == "cpu" and f[f.index("--cache-ram") + 1] == "0", f +k = int(f[f.index("-ctxcp") + 1]) +ckpt = 25 * 2 * 8 * 256 * 2 * 1536 # 25 SWA layers K and V at 2 B over 1536 cells +assert d["ckpt_bytes"] >= d["seqs"] * k * ckpt > 0, (d["ckpt_bytes"], d["seqs"], k) +assert int(d["k8s"]["memory_limit"][:-2]) * 2 ** 30 >= d["host_bytes"] >= d["kv_bytes"] + d["ckpt_bytes"] +' "${out}"; then + ok "fit (llama.cpp, Gemma QAT on the desktop CPU): --cache-ram 0, -ctxcp K, checkpoints in the pod limit" +else fail "fit (llama.cpp host caches) on the desktop (rc ${rc})" "${out}"; fi + +# Gemma 4 ties its embeddings: llama.cpp loads token_embd again as the output tensor, on the +# output layer's device, so one card holds that copy beside the layers. +out=$(bash "${K3D_GPU}" fit --host "${XEON}" --model google/gemma-4-26B-A4B-it-qat-q4_0-gguf --explain 2>&1); rc=$? +if [ "${rc}" -eq 0 ] && grep -q 'llama.cpp/gpu: FITS' <<< "${out}" \ + && grep -q 'weights include the output tensor: a 0.6 GiB copy of the tied token embedding' <<< "${out}"; then + ok "fit (llama.cpp, Gemma 4 on one card): the tied embedding's output copy counts as GPU bytes" +else fail "fit (llama.cpp, tied output copy) on the Xeon (rc ${rc})" "$(grep -E 'llama.cpp/|weights' <<< "${out}")"; fi + out=$(bash "${K3D_GPU}" fit --host "${XEON}" --model not/in-cache-offline 2>&1 &1); rc=$? if [ "${rc}" -eq 0 ] && grep -q "^fit: ${FLASH}:IQ4_NL on 2 GPU(s) + experts and lookup tables in DRAM" <<< "${out}"; then ok "fit: a multi-GGUF repo needs no flag (IQ4_NL picked, split shards summed)" else fail "fit: multi-GGUF repo auto-pick (rc ${rc})" "${out}"; fi +# Its two-card plan sizes each card the way llama.cpp assigns layers: -ts is integer layer +# counts (48 layers + the output layer), and no card's own bytes pass its budget. The plan +# this replaces, -ts 30,30 --n-cpu-moe 28, put 43.5 GiB on the second 32 GiB card. +out=$(bash "${K3D_GPU}" fit --host "${XEON}" --model "${FLASH}" --json 2>&1); rc=$? +if [ "${rc}" -eq 0 ] && ts=$(python3 -c ' +import json, sys +sys.path.insert(0, sys.argv[1]) +import fit +d = json.loads(sys.argv[2]) +f = d["flags"] +ts = [int(x) for x in f[f.index("-ts") + 1].split(",")] +budget = min(c["budget"] for c in fit.gpu_budget(json.load(open(sys.argv[3])))[0][:d["gpus"]]) +assert d["placement"] == "moe-offload" and len(ts) == d["gpus"] == 2 and sum(ts) == 49, (ts, d["gpus"]) +assert ts != [30, 30] and "--n-cpu-moe" in f, f +assert len(d["card_bytes"]) == 2 and max(d["card_bytes"]) <= budget, (d["card_bytes"], budget) +# context checkpoints (here the recurrent state) stay in host RAM on a split too +k = int(f[f.index("-ctxcp") + 1]) +assert f[f.index("--cache-ram") + 1] == "0" and 0 < d["seqs"] * k <= d["ckpt_bytes"] < d["host_bytes"], (k, d["ckpt_bytes"]) +print(f[f.index("-ts") + 1]) +' "${ROOT}/console" "${out}" "${XEON}"); then + ok "fit: Flash-Next IQ4_NL on two cards, -ts ${ts} layers, every card within its budget, checkpoints on the host" +else fail "fit: Flash-Next IQ4_NL per-card split (rc ${rc})" "$(grep -E '"-ts"|card_bytes' -A3 <<< "${out}")"; fi out=$(bash "${K3D_GPU}" fit --host "${XEON}" --model "${FLASH}" --file Qwen3.8-Flash-Next-Q8_0-00002-of-00002.gguf --explain 2>&1); rc=$? if [ "${rc}" -eq 0 ] && grep -q ':Q8_0 on' <<< "${out}" && grep -q -- "-hf ${FLASH}:Q8_0" <<< "${out}"; then ok "fit --file: a shard name picks its whole split model" @@ -156,6 +203,39 @@ if render ovms STUB_ALLOC='cpu 32' K3D_GPU_FIT_HOST="${DESK}" K3D_GPU_AGENT_MODE ok "agent manifest (ovms, CPU): static KV pool, seqs and pod memory limit from the fit" else fail "agent manifest (ovms, CPU) from the fit" "$(cat "${WORK}/ovms.err" "${WORK}/ovms.yaml" 2>/dev/null | head -60)"; fi +# OVMS serves one GPU: K3D_GPU_AGENT_GPUS=2 is a vLLM knob, so the OVMS pod (one +# gpu.intel.com/xe) gets a KV pool that fits one card, not two cards' worth. +one_card=$(python3 -c 'import json, sys; sys.path.insert(0, sys.argv[1]); import fit +print(max(c["budget"] for c in fit.gpu_budget(json.load(open(sys.argv[2])))[0]) // fit.GIB)' "${ROOT}/console" "${XEON}") +if render ovms2 STUB_ALLOC=$'cpu 64\ngpu.intel.com/xe 4' STUB_NODE_LABELS=gpu.intel.com/xmx=true K3D_GPU_FIT_HOST="${XEON}" \ + K3D_GPU_AGENT_MODEL=OpenVINO/gemma-4-26b-a4b-it-int4-ov K3D_GPU_AGENT_GPUS=2 -- \ + && cache=$(sed -n 's/^ *- --cache_size=\([0-9]*\)$/\1/p' "${WORK}/ovms2.yaml") && [ -n "${cache}" ] \ + && [ "${cache}" -le "${one_card}" ] && has ovms2 '^ gpu.intel.com/xe: 1$' && ! has ovms2 'gpu-split'; then + ok "agent manifest (ovms, K3D_GPU_AGENT_GPUS=2): --cache_size=${cache} fits one card's ${one_card} GiB budget" +else fail "agent manifest (ovms, K3D_GPU_AGENT_GPUS=2) sized for one card (cache ${cache:-?}, one card ${one_card})" \ + "$(cat "${WORK}/ovms2.err"; grep -E 'cache_size|agent-fit|gpu.intel.com/xe' "${WORK}/ovms2.yaml")"; fi +# With K3D_GPU_AGENT_BACKEND unset the fit picks the engine, and K3D_GPU_AGENT_GPUS still +# reaches it for a model OVMS does not serve: RedHatAI's INT4 checkpoint fits one B70, so +# only the two GPUs asked put it on two (vLLM tensor parallel), as with the backend set. +if render vllm2 STUB_ALLOC=$'cpu 64\ngpu.intel.com/xe 4' STUB_NODE_LABELS=gpu.intel.com/xmx=true K3D_GPU_FIT_HOST="${XEON}" \ + K3D_GPU_MODEL_OPT=off K3D_GPU_AGENT_GPUS=2 -- --model RedHatAI/Qwen3.8-27B-INT4 \ + && grep -q 'fit: RedHatAI/Qwen3.8-27B-INT4 on 2 GPUs, tensor parallel via vllm' "${WORK}/vllm2.err" \ + && has vllm2 '--tensor-parallel-size=2$' && has vllm2 '^ gpu.intel.com/xe: 2$'; then + ok "agent manifest (backend unset, K3D_GPU_AGENT_GPUS=2): the fit's vLLM pick shards over the two GPUs asked" +else fail "agent manifest (backend unset, K3D_GPU_AGENT_GPUS=2) reaches the fit" \ + "$(cat "${WORK}/vllm2.err"; grep -E 'tensor-parallel|agent-fit|gpu.intel.com/xe' "${WORK}/vllm2.yaml")"; fi + +# OVMS has no sliding-window cache: Gemma 4's 25 SWA layers hold the whole context, so the +# static pool holds seqs x context x 225280 B (5 full + 25 SWA layers' K and V at 2 B). +if render ovms-swa STUB_ALLOC=$'cpu 64\ngpu.intel.com/xe 4' STUB_NODE_LABELS=gpu.intel.com/xmx=true K3D_GPU_FIT_HOST="${XEON}" \ + K3D_GPU_AGENT_MODEL=OpenVINO/gemma-4-26b-a4b-it-int4-ov -- \ + && cache=$(sed -n 's/^ *- --cache_size=\([0-9]*\)$/\1/p' "${WORK}/ovms-swa.yaml") \ + && sc=$(sed -n 's/^ *k3d-gpu.io\/agent-fit: "gpu \([0-9]*x[0-9]*\)"$/\1/p' "${WORK}/ovms-swa.yaml") \ + && [ -n "${cache}" ] && [ -n "${sc}" ] && [ $(( cache * 1073741824 )) -ge $(( ${sc%x*} * ${sc#*x} * 225280 )) ]; then + ok "agent manifest (ovms, Gemma 4 on a GPU): --cache_size=${cache} holds ${sc} tokens at 225280 B each" +else fail "agent manifest (ovms, Gemma 4): --cache_size holds seqs x context at 225280 B/token (cache ${cache:-?}, ${sc:-?})" \ + "$(cat "${WORK}/ovms-swa.err"; grep -E 'cache_size|agent-fit' "${WORK}/ovms-swa.yaml")"; fi + if render tier STUB_ALLOC='cpu 32' K3D_GPU_FIT_HOST="${LAPTOP}" K3D_GPU_AGENT_MODEL=OpenVINO/Qwen3.8-27B-int8-ov \ K3D_GPU_AGENT_MODEL_TIER=1 -- && ! has tier 'agent-fit'; then ok "a quickstart tier pick (K3D_GPU_AGENT_MODEL_TIER=1) is not sized: it is no user choice" diff --git a/tests/llamacpp_test.sh b/tests/llamacpp_test.sh index 0b864630..21fc7efe 100755 --- a/tests/llamacpp_test.sh +++ b/tests/llamacpp_test.sh @@ -92,15 +92,16 @@ pair() { server_args "$1" | awk -v o="$2" -v v="$3" 'p && $0 == v { f = 1 } { p # llama-server's parser (llama.cpp v0.8.0 common/arg.cpp parse_cli_args, :827-865) takes an # option's value from the next argument, never from "--opt=value", and refuses a name it does # not define. llama_argv_ok : the server args parse so, with the names and arities the -# renderer uses (common/arg.cpp v0.8.0: -t :1528, -tb :1538, -c :1650, -ctk :2449, -ctv :2462, -# -np :2556, --load-mode :2703, --cpu-moe :2772, --n-cpu-moe :2779, -ngl :2811, -ts :2853, -# --alias :3033, --model-url :3066, --host :3343, --port :3359, --jinja :3727, -# --reasoning :3746, --spec-draft-model :4322, --spec-type :4330, --model :3057). +# renderer uses (common/arg.cpp v0.8.0: -t :1528, -tb :1538, -c :1650, -ctxcp :1709, +# --cache-ram :1727, -ctk :2449, -ctv :2462, -np :2556, --load-mode :2703, --cpu-moe :2772, +# --n-cpu-moe :2779, -ngl :2811, -ts :2853, --alias :3033, --model-url :3066, --host :3343, +# --port :3359, --jinja :3727, --reasoning :3746, --spec-draft-model :4322, --spec-type :4330, +# --model :3057). llama_argv_ok() { server_args "$1" | python3 -c 'import sys arity = {"--model": 1, "--model-url": 1, "--alias": 1, "--host": 1, "--port": 1, "--jinja": 0, "--reasoning": 1, "-c": 1, "-np": 1, "-ngl": 1, "-ts": 1, "--n-cpu-moe": 1, "--cpu-moe": 0, - "-ctk": 1, "-ctv": 1, "--threads": 1, "--threads-batch": 1, "--load-mode": 1, + "-ctk": 1, "-ctv": 1, "--threads": 1, "--threads-batch": 1, "--load-mode": 1, "--cache-ram": 1, "-ctxcp": 1, "--spec-type": 1, "--spec-draft-model": 1} argv = sys.stdin.read().split("\n")[:-1] i = 0 @@ -133,7 +134,7 @@ if render gguf "${XEON_ENV[@]}" -- --model "${GEMMA_GGUF}" \ && pair gguf --model "/models/gguf/google__gemma-4-26B-A4B-it-qat-q4_0-gguf@${GREV}/gemma-4-26B_q4_0-it.gguf" \ && pair gguf --model-url "https://huggingface.co/${GEMMA_GGUF}/resolve/${GREV}/gemma-4-26B_q4_0-it.gguf" \ && pair gguf --alias 'gemma-4-26B-A4B-it-qat-q4_0-gguf' && arg gguf '--jinja' && pair gguf --reasoning 'off' \ - && arg gguf '-c' && arg gguf '786432' && arg gguf '-np' && arg gguf '6' && arg gguf '-ngl' && arg gguf '999' \ + && pair gguf -c 655360 && pair gguf -np 5 && pair gguf -ngl 999 && pair gguf --cache-ram 0 && pair gguf -ctxcp 4 \ && pair gguf --port '8000' && server_args gguf | grep -A1 -x -- '--threads' | tail -n1 | grep -qE '^[0-9]+$' \ && ! arg gguf '-ts' && ! has gguf 'spec-type' && ! has gguf 'kind: Job' && ! has gguf 'load-mode' && ! arg gguf '-hf' \ && grep -A3 '^ limits:$' "${WORK}/gguf.yaml" | grep -q '^ gpu.intel.com/xe: 1$' \ @@ -189,7 +190,7 @@ FN=Qwen/Qwen3.8-Flash-Next FNREV=de4b8e4d43b917e7706784d8bb445c9af86a3540 if render fn "${XEON_ENV[@]}" -- --model "${FN}" \ && pair fn --model "/models/gguf/Qwen__Qwen3.8-Flash-Next@${FNREV}/model.gguf" && ! has fn 'model-url' \ - && arg fn '--n-cpu-moe' && arg fn '44' && arg fn '-ts' && arg fn '30,30' && arg fn '-np' && arg fn '4' \ + && pair fn --n-cpu-moe 45 && pair fn -ts 39,11 && pair fn -np 4 && pair fn --cache-ram 0 && pair fn -ctxcp 4 \ && grep -A3 '^ limits:$' "${WORK}/fn.yaml" | grep -q '^ gpu.intel.com/xe: 2$' \ && has fn 'kind: Job' && has fn "image: ghcr.io/ggml-org/llama.cpp:full-b11960@${FULL}" \ && has fn '^ - "Qwen/Qwen3.8-Flash-Next"$' && has fn "^ - \"${FNREV}\"\$" \ @@ -378,6 +379,22 @@ else fail "an explicit backend is kept" "$(show set)"; fi if render setl "${XEON_ENV[@]}" K3D_GPU_AGENT_BACKEND=llama.cpp -- --model "${GEMMA_GGUF}" && has setl '^ name: llamacpp$'; then ok "K3D_GPU_AGENT_BACKEND=llama.cpp serves the GGUF" else fail "K3D_GPU_AGENT_BACKEND=llama.cpp" "$(show setl)"; fi +# K3D_GPU_AGENT_GPUS is the fit's --gpus for llama.cpp as for vLLM (an OVMS pod holds one +# GPU): the QAT GGUF, one card's model, is split layer by layer over the two free B70s asked. +if render setl2 "${XEON_ENV[@]}" K3D_GPU_AGENT_BACKEND=llama.cpp K3D_GPU_AGENT_GPUS=2 -- --model "${GEMMA_GGUF}" \ + && grep -q "fit: ${GEMMA_GGUF}:Q4_0 on 2 GPUs, layer split via llama.cpp" "${WORK}/setl2.err" \ + && server_args setl2 | grep -A1 -x -- '-ts' | tail -n1 | grep -qE '^[0-9]+,[0-9]+$' \ + && grep -A3 '^ limits:$' "${WORK}/setl2.yaml" | grep -q '^ gpu.intel.com/xe: 2$'; then + ok "K3D_GPU_AGENT_BACKEND=llama.cpp K3D_GPU_AGENT_GPUS=2: the fit splits the GGUF over two B70s, -ts in layers" +else fail "K3D_GPU_AGENT_GPUS=2 reaches the llama.cpp fit" "$(show setl2)"; fi +# With the backend unset and no optimizer (K3D_GPU_AGENT_MODEL, not --model), the backend is +# OVMS's until the fit picks llama.cpp for the GGUF, and K3D_GPU_AGENT_GPUS still reaches the +# fit: only OVMS, which serves an OpenVINO IR alone, holds one GPU whatever it says. +if render unset2 "${XEON_ENV[@]}" K3D_GPU_AGENT_GPUS=2 K3D_GPU_AGENT_MODEL="${GEMMA_GGUF}" -- && has unset2 '^ name: llamacpp$' \ + && grep -q "fit: ${GEMMA_GGUF}:Q4_0 on 2 GPUs, layer split via llama.cpp" "${WORK}/unset2.err" \ + && grep -A3 '^ limits:$' "${WORK}/unset2.yaml" | grep -q '^ gpu.intel.com/xe: 2$'; then + ok "K3D_GPU_AGENT_BACKEND unset, K3D_GPU_AGENT_GPUS=2: the fit's llama.cpp pick splits the GGUF over two B70s" +else fail "K3D_GPU_AGENT_GPUS=2 reaches the fit with the backend unset" "$(show unset2)"; fi # --- MTP on request --------------------------------------------------------------------- if render mtp "${XEON_ENV[@]}" K3D_GPU_AGENT_DRAFT=mtp -- --model "${GEMMA_GGUF}" \