diff --git a/console/fit.py b/console/fit.py index 41211e77..b0afaf4d 100644 --- a/console/fit.py +++ b/console/fit.py @@ -47,6 +47,7 @@ """ import argparse +import bisect import glob import json import math @@ -793,10 +794,10 @@ def kv_profile(cfg, kv_bytes): heads = tc.get("num_attention_heads") kvh = tc.get("num_key_value_heads") or heads hd = tc.get("head_dim") or tc["hidden_size"] // heads - full = sum(1 for t in types if t == "full_attention") - swa = sum(1 for t in types if t in ("sliding_attention", "sliding_window_attention", "local_attention")) - lin = sum(1 for t in types if t in ("linear_attention", "mamba", "gated_delta_net")) - full += len(types) - full - swa - lin # an unknown layer type counts as full attention (upper bound) + # an unknown layer type counts as full attention (upper bound) + kinds = ["swa" if t in ("sliding_attention", "sliding_window_attention", "local_attention") else + "linear" if t in ("linear_attention", "mamba", "gated_delta_net") else "full" for t in types] + full, swa, lin = kinds.count("full"), kinds.count("swa"), kinds.count("linear") gkvh = tc.get("num_global_key_value_heads") or kvh ghd = tc.get("global_head_dim") or hd lines = [] @@ -829,7 +830,7 @@ def kv_profile(cfg, kv_bytes): " = %s per sequence, any length" % (lin, vh, kd, vd, ssm_b, ck, ch, _mb(state))) return {"full_layers": full, "full_per_token": full * (per_full + idx), "swa_layers": swa, "swa_per_token": swa * per_swa, "window": window, "linear_layers": lin, - "state_per_seq": state, "kv_heads": gkvh, "lines": lines} + "state_per_seq": state, "kv_heads": gkvh, "kinds": kinds, "lines": lines} def _num(x): @@ -966,9 +967,121 @@ def act_bytes(m, tokens, seqs): return tokens * width * 2 * 2 + seqs * m["vocab"] * 4 +def kv_parts(kv, ctx): + """(full attention, sliding window, linear state) KV bytes of one sequence at context ctx.""" + return kv["full_per_token"] * ctx, kv["swa_per_token"] * min(ctx, kv["window"] or ctx), kv["state_per_seq"] + + def per_seq(kv, ctx, scale=1.0): - return int((kv["full_per_token"] * ctx + kv["swa_per_token"] * min(ctx, kv["window"] or ctx)) * scale - + kv["state_per_seq"] * scale) + f, w, st = kv_parts(kv, ctx) + return int((f + w) * scale + st * scale) + + +# ---- llama.cpp over several GPUs ---------------------------------------------------------- +def llama_layer_devices(ts, n_layer): + """Device of each repeating layer and of the output layer (the last entry), as llama.cpp + b11749 assigns them with -ngl past the layer count (src/llama-model.cpp load_tensors): + upper_bound(cumsum(ts) / sum(ts), il / (n_layer + 1)). Integer layer counts summing to + n_layer + 1 give each device exactly its count, the output layer on the last one.""" + total, cum, splits = float(sum(ts)), 0.0, [] + for t in ts: + cum += t + splits.append(cum / total) + return [bisect.bisect_right(splits, il / (n_layer + 1)) for il in range(n_layer + 1)] + + +def llama_layer_bytes(m, kv, ctx, seqs, n_cpu_moe): + """([weights], [KV]) on the GPU per repeating layer, the output layer last: the class + totals spread evenly over the layers, the experts of the first n_cpu_moe layers in host + RAM (--n-cpu-moe), and each layer's own KV (its device holds it) for seqs sequences. + The weights add up to the bytes to serve less the lookup tables (MTP only when the GGUF + holds it). The output layer holds lm_head and the MTP tensors; the vision projector + counts with layer 0, on the first card: tools/mtmd/clip.cpp loads it on the first GPU.""" + n = max(m["layers"], 1) + b = m["bytes"] + exp, other = b["experts"] / n, b["other"] / n + f, w, st = kv_parts(kv, ctx) + per = {"full": f / max(kv["full_layers"], 1), "swa": w / max(kv["swa_layers"], 1), + "linear": st / max(kv["linear_layers"], 1)} + kinds = kv.get("kinds") or [] + wl = [other + (exp if il >= n_cpu_moe else 0) for il in range(n)] + wl[0] += b["vision"] + kl = [per[kinds[il]] * seqs if il < len(kinds) else 0 for il in range(n)] + out = m["serve_bytes"] - m["lookup_bytes"] - b["experts"] - b["other"] - b["vision"] + return wl + [out], kl + [0] + + +def llama_split(layer_bytes, k, room): + """-ts for k cards as integer layer counts (the last card's count includes the output + layer) that keep every card's layers within room bytes, its largest load as small as it + can be; None when no split does.""" + last = len(layer_bytes) - 1 + + def fill(cap): + ts, i = [], 0 + for _ in range(k - 1): + start, used = i, 0 + while i < last and used + layer_bytes[i] <= cap: + used += layer_bytes[i] + i += 1 + ts.append(i - start) + if sum(layer_bytes[i:]) > cap: + return None + return ts + [len(layer_bytes) - i] + + if fill(room) is None: + return None + lo, hi = max(layer_bytes), room + while hi - lo > MIB: + mid = (lo + hi) / 2 + lo, hi = (lo, mid) if fill(mid) else (mid, hi) + return fill(hi) + + +def llama_cards(ts, layer_bytes): + """Each card's bytes for ts: the layers llama_layer_devices puts on it.""" + out = [0] * len(ts) + for il, dev in enumerate(llama_layer_devices(ts, len(layer_bytes) - 1)): + out[dev] += layer_bytes[il] + return out + + +def _llama_search(m, kv, ladder, relaxed, want_seqs, k, room, n_cpus): + """(ctx, seqs, n_cpu_moe, ts) for llama.cpp on k cards of room bytes each: the most + context at which TARGET_SEQS (or want_seqs) sequences fit, then the fewest layers whose + experts leave the GPU (from n_cpus), then as many sequences as fit (capped at MAX_SEQS). + relaxed: at the ladder's last rung one sequence is enough. None when nothing fits.""" + for ctx in ladder: + need = want_seqs or (1 if relaxed and ctx == ladder[-1] else TARGET_SEQS) + for n_cpu in n_cpus: + def split(n): + wl, kl = llama_layer_bytes(m, kv, ctx, n, n_cpu) + return llama_split([a + b for a, b in zip(wl, kl)], k, room) + if not split(need): + continue + n = want_seqs + if not n: + lo, hi = need, MAX_SEQS + if split(hi): + lo = hi + while hi - lo > 1: + mid = (lo + hi) // 2 + lo, hi = (mid, hi) if split(mid) else (lo, mid) + n = lo + return ctx, n, n_cpu, split(n) + return None + + +def _llama_card_lines(use, ts, wl, kl, ov): + """One explain line per card of a llama.cpp split.""" + lines, first = [], 0 + for c, t, w, kvb in zip(use, ts, llama_cards(ts, wl), llama_cards(ts, kl)): + lines.append(" %s (-ts %d): layers %s, weights %s + KV %s + overhead %s = %s, budget %s" + % (c["card"], t, "%d-%d" % (first, first + t - 1) if t else "none", g1(w), g1(kvb), g1(ov), + g1(w + kvb + ov), g1(c["budget"]))) + first += t + lines.append(" (layer %d is the output layer: lm_head)" % (len(wl) - 1)) + return lines def ctx_ladder(max_ctx, want=None): @@ -1108,7 +1221,7 @@ def fit(host, cfg, weights, engine=None, ctx=None, seqs=None, gpus=None, kv_dtyp rung_kv.append(int(kb)) rung_limit.append("%dGi" % int(math.ceil(hb * (1 + LIMIT_SLACK) / GIB))) doc.update({k: picked[k] for k in ("engine", "placement", "ctx", "seqs", "tp", "gpus", "gpu_bytes", "host_bytes", - "flags", "dram_tok_s", "note", "kv_bytes")}) + "flags", "dram_tok_s", "note", "kv_bytes", "card_bytes")}) lim = int(math.ceil(picked["host_bytes"] * (1 + LIMIT_SLACK) / GIB)) doc["k8s"] = {"memory_request": "%dGi" % lim, "memory_limit": "%dGi" % lim} doc["guard"] = {"rungs": rungs, "placement": picked["placement"], "some_avg10": GUARD_SOME_AVG10, @@ -1146,7 +1259,7 @@ def _overhead(eng, m, seqs, override): def _res(eng, place, fits, why="", math_lines=(), **kw): r = {"engine": eng, "placement": place, "fits": fits, "why": why, "math": list(math_lines), "ctx": None, "seqs": 0, "tp": 1, "gpus": 0, "gpu_bytes": 0, "host_bytes": 0, "flags": [], "dram_tok_s": None, - "kv_bytes": 0, "kv_scale": 1.0} + "kv_bytes": 0, "kv_scale": 1.0, "card_bytes": []} r.update(kw) return r @@ -1187,6 +1300,9 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, yield _res(eng, place, False, "%d GPUs asked, %d not busy" % (k, len(cards)), gpus=k) continue use = cards[:k] + if eng == "llama.cpp" and k > 1: + yield _llama_multi(m, kv, ladder, relaxed, want_seqs, use, ov, ovl, [0], on_host, hfree, bw) + continue if eng == "vllm": # Every rank holds 1/k of the weights; KV heads split across ranks, or one # per rank (replicated) with fewer KV heads than ranks. @@ -1197,8 +1313,7 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, lines = ["per card: budget %s - weights %s / %d - overhead %s = %s for KV" % (g1(budget), g1(gpu_w_all), k, ovl, g1(max(left, 0)))] else: - # One card, or llama.cpp's layer split (-sm layer, the default): weights and - # KV spread over the cards in proportion to -ts; each card runs its own buffers. + # One card (llama.cpp on several is sized card by card in _llama_multi). budget = sum(c["budget"] for c in use) w = gpu_w_all left = budget - w - ov * k @@ -1226,66 +1341,34 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, continue gpu_use = int(w + ov + kvused) if eng == "vllm" else int((w + ov * k + kvused) / k) yield _res(eng, place, True, "", lines, ctx=c, seqs=n, tp=k if eng == "vllm" else 1, gpus=k, - gpu_bytes=gpu_use, host_bytes=hb, kv_bytes=int(kvused), kv_scale=scale, + gpu_bytes=gpu_use, host_bytes=hb, kv_bytes=int(kvused), kv_scale=scale, card_bytes=[gpu_use] * k, flags=_flags(eng, place, c, n, k, gpu_use, use, kvused, m, 0, kv)) - # MoE experts to DRAM (llama.cpp --n-cpu-moe): every free card, fewest CPU layers. + # MoE experts to DRAM (llama.cpp --n-cpu-moe). Every card is held to the smallest budget + # of the cards used, so one more card can leave less room per card: the free cards by + # budget, 1, 2, ... of them (or --gpus), and the plan with the most context, then the + # fewest CPU-expert layers, then the most sequences, then the fewest cards. if eng == "llama.cpp" and cards and m["moe"]: - k = want_gpus or len(cards) - use = cards[:k] - budget = sum(c["budget"] for c in use) - exp_layer = m["bytes"]["experts"] / max(m["layers"], 1) - base = gpu_w_all - m["bytes"]["experts"] - lines = ["GPU (%s, budget %s) keeps %s non-expert weights + overhead %d x %s" - % ("+".join(c["card"] for c in use), g1(budget), g1(base), k, ovl), - "lookup tables %s in host RAM" % g1(on_host)] - if not (cpu["amx"] or cpu["avx512"]): - yield _res(eng, "moe-offload", False, "the CPU has neither AMX nor AVX-512 for expert compute", lines, gpus=k) + ks = [want_gpus] if want_gpus else list(range(1, len(cards) + 1)) + if want_gpus and want_gpus > len(cards): + yield _res(eng, "moe-offload", False, "%d GPUs asked, %d not busy" % (want_gpus, len(cards)), gpus=want_gpus) + elif not (cpu["amx"] or cpu["avx512"]): + yield _res(eng, "moe-offload", False, "the CPU has neither AMX nor AVX-512 for expert compute", gpus=ks[-1]) elif not bw: yield _res(eng, "moe-offload", False, "DRAM bandwidth unknown (no STREAM run cached, SMBIOS unreadable)", - lines, gpus=k) + gpus=ks[-1]) else: - # Most context first (as everywhere), then the fewest layers whose experts leave the GPU. - best = None - for rung in ladder: - for n_cpu in range(1, m["layers"] + 1): - gw = base + exp_layer * (m["layers"] - n_cpu) - left = budget - gw - ov * k - if left <= 0: - continue - c, n = size_kv(kv, left, [rung], want_seqs, 1.0, relaxed and rung == ladder[-1]) - if c: - best = (n_cpu, gw, c, n) - break - if best: - break - if not best: - yield _res(eng, "moe-offload", False, "non-expert weights + KV do not fit %s of VRAM" % g1(budget), - lines, gpus=k) + tries = [_llama_multi(m, kv, ladder, relaxed, want_seqs, cards[:k], ov, ovl, range(1, m["layers"] + 1), + on_host, hfree, bw) for k in ks] + ok = [r for r in tries if r["fits"]] + if not ok: + yield tries[-1] else: - n_cpu, gw, c, n = best - dram_w = exp_layer * n_cpu - read = dram_w * m["experts_per_tok"] / m["experts"] - toks = _dram_rate(bw, read) - lines.append("experts of the first %d of %d layers in DRAM: %s; on GPU %s" - % (n_cpu, m["layers"], g1(dram_w), g1(gw))) - lines.append("DRAM decode bound: %g GB/s / (%s x %d/%d read per token) = %.0f tok/s (floor %g)" - % (bw, g1(dram_w), m["experts_per_tok"], m["experts"], toks, MIN_DRAM_TOK_S)) - kvused = per_seq(kv, c) * n - lines.append("KV: %d seqs x %s at %d tokens = %s on GPU" % (n, _mb(per_seq(kv, c)), c, g1(kvused))) - hb = int(dram_w + on_host + HOST_RUNTIME[eng]) - lines.append("host: experts %s + lookup tables %s + runtime %s = %s" - % (g1(dram_w), g1(on_host), g1(HOST_RUNTIME[eng]), g1(hb))) - if toks < MIN_DRAM_TOK_S: - yield _res(eng, "moe-offload", False, "DRAM bandwidth gives %.1f tok/s, under the %g floor" - % (toks, MIN_DRAM_TOK_S), lines, gpus=k) - elif hb > hfree: - yield _res(eng, "moe-offload", False, "DRAM side needs %s, host budget %s" % (g1(hb), g1(hfree)), - lines, gpus=k) - else: - gpu_use = int((gw + ov * k + kvused) / k) - yield _res(eng, "moe-offload", True, "", lines, ctx=c, seqs=n, gpus=k, gpu_bytes=gpu_use, - host_bytes=hb, dram_tok_s=round(toks, 1), kv_bytes=int(kvused), - flags=_flags(eng, "moe-offload", c, n, k, gpu_use, use, kvused, m, n_cpu, kv)) + best = max(ok, key=lambda r: (r["ctx"], -r["n_cpu_moe"], r["seqs"], -r["gpus"])) + if len(tries) > 1: + best["math"].append("cards tried, by budget: " + "; ".join( + "%d: %d x %d, experts of %d layers in DRAM" % (r["gpus"], r["ctx"], r["seqs"], r["n_cpu_moe"]) + if r["fits"] else "%d: no" % r["gpus"] for r in tries)) + yield best # CPU only: weights, KV and activations all in DRAM (vLLM: its CPU build, the KV # pool sized by VLLM_CPU_KVCACHE_SPACE, as EAT's vLLM chart sets it). if cpu_ok: @@ -1321,7 +1404,57 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, flags=_flags(eng, "cpu", c, n, 1, 0, [], kvused, m, 0, kv)) -def _flags(eng, place, ctx, seqs, k, gpu_use, cards, kvused, m, n_cpu_moe, kv=None): +def _llama_multi(m, kv, ladder, relaxed, want_seqs, use, ov, ovl, n_cpus, on_host, hfree, bw): + """llama.cpp on the cards in use, each holding its own layers, their KV and its own + overhead within the smallest budget among them (the device plugin, not the fit, orders + the cards inside the pod): a layer split (n_cpus [0]) or experts in DRAM (n_cpus 1..).""" + k = len(use) + place = "gpu-split" if list(n_cpus) == [0] else "moe-offload" + budget = min(c["budget"] for c in use) + room = budget - ov + lines = ["%s: layer il on card upper_bound(cumsum(-ts) / sum, il / (layers + 1)) (src/llama-model.cpp); " + "each card: budget %s (the smallest of them) - overhead %s = %s for its layers" + % ("+".join(c["card"] for c in use), g1(budget), ovl, g1(max(room, 0)))] + if on_host: + lines.append("lookup tables %s stay in host RAM" % g1(on_host)) + found = _llama_search(m, kv, ladder, relaxed, want_seqs, k, room, n_cpus) if room > 0 else None + if not found: + why = ("weights + KV for %d-token sequences do not fit %d card(s) of %s each, layer by layer" + % (ladder[-1], k, g1(budget))) + return _res("llama.cpp", place, False, why, lines, gpus=k) + c, n, n_cpu, ts = found + wl, kl = llama_layer_bytes(m, kv, c, n, n_cpu) + cards_b = [int(b + ov) for b in llama_cards(ts, [a + b for a, b in zip(wl, kl)])] + lines += _llama_card_lines(use, ts, wl, kl, ov) + if m["bytes"]["vision"]: + lines.append(" (%s holds the vision projector, %s: tools/mtmd/clip.cpp loads it on the first GPU)" + % (use[0]["card"], g1(m["bytes"]["vision"]))) + kvused = per_seq(kv, c) * n + lines.append("KV: %d seqs x %s at %d tokens = %s over the cards" % (n, _mb(per_seq(kv, c)), c, g1(kvused))) + hb = int(HOST_RUNTIME["llama.cpp"] + on_host) + toks = None + if n_cpu: + exp_layer = m["bytes"]["experts"] / max(m["layers"], 1) + dram_w = exp_layer * n_cpu + toks = _dram_rate(bw, dram_w * m["experts_per_tok"] / m["experts"]) + lines.append("experts of the first %d of %d layers in DRAM: %s" % (n_cpu, m["layers"], g1(dram_w))) + lines.append("DRAM decode bound: %g GB/s / (%s x %d/%d read per token) = %.0f tok/s (floor %g)" + % (bw, g1(dram_w), m["experts_per_tok"], m["experts"], toks, MIN_DRAM_TOK_S)) + hb += int(dram_w) + lines.append("host: experts %s + lookup tables %s + runtime %s = %s" + % (g1(dram_w), g1(on_host), g1(HOST_RUNTIME["llama.cpp"]), g1(hb))) + if toks < MIN_DRAM_TOK_S: + return _res("llama.cpp", place, False, "DRAM bandwidth gives %.1f tok/s, under the %g floor" + % (toks, MIN_DRAM_TOK_S), lines, gpus=k) + if hb > hfree: + return _res("llama.cpp", place, False, "host side %s exceeds the host budget %s" % (g1(hb), g1(hfree)), + lines, gpus=k) + return _res("llama.cpp", place, True, "", lines, ctx=c, seqs=n, gpus=k, gpu_bytes=max(cards_b), card_bytes=cards_b, + host_bytes=hb, dram_tok_s=round(toks, 1) if toks else None, kv_bytes=int(kvused), n_cpu_moe=n_cpu, + flags=_flags("llama.cpp", place, c, n, k, max(cards_b), use, kvused, m, n_cpu, kv, ts)) + + +def _flags(eng, place, ctx, seqs, k, gpu_use, cards, kvused, m, n_cpu_moe, kv=None, ts=None): """Engine memory flags from the fit. Spellings from each engine's own argument parser: vLLM v0.32.0 vllm/engine/arg_utils.py, OVMS v2026.4.1 src/graph_export/graph_cli_parser.cpp, llama.cpp 1242be6 common/arg.cpp.""" @@ -1343,8 +1476,8 @@ def _flags(eng, place, ctx, seqs, k, gpu_use, cards, kvused, m, n_cpu_moe, kv=No # -c is the total context; with -np set explicitly each slot gets c/np (kv-unified is # on only when the slot count is auto). out = ["-c", str(ctx * seqs), "-np", str(seqs), "-ngl", "0" if place == "cpu" else "999"] - if k > 1: - out += ["-ts", ",".join(str(max(int(c["budget"] // GIB), 1)) for c in cards)] + if ts and k > 1: + out += ["-ts", ",".join(str(t) for t in ts)] # layers per card (llama_layer_devices) if place == "moe-offload": out += ["--cpu-moe"] if n_cpu_moe >= m["layers"] else ["--n-cpu-moe", str(n_cpu_moe)] t = {"fp16": "f16"}.get(kv.get("dtype"), kv.get("dtype")) diff --git a/docs/fit.md b/docs/fit.md index ab06e5f1..ee9b6849 100644 --- a/docs/fit.md +++ b/docs/fit.md @@ -57,10 +57,12 @@ in this order, and the first that fits wins: 1. one GPU; 2. several GPUs: vLLM tensor parallel (heads divisible by the size), or llama.cpp's - layer split; + layer split, sized card by card (below); 3. MoE experts in DRAM (llama.cpp `--n-cpu-moe N`): only when the GPU placements do not fit, the CPU has AMX or AVX-512, and the DRAM side's bandwidth bound per token stays - above a floor. The fit takes the fewest layers whose experts leave the GPU; + above a floor. The fit takes the fewest layers whose experts leave the GPU, trying + 1, 2, ... of the free cards (largest budget first) and keeping the plan with the most + context, then the fewest such layers, then the most sequences, then the fewest cards; 4. CPU only. Per placement: @@ -87,6 +89,22 @@ llama.cpp keeps the token embeddings and the n-gram table on the CPU by design bytes on every llama.cpp placement. OpenVINO's GPU plugin compiles from a host copy, so an OVMS GPU placement counts the weights as host bytes too. +On several GPUs llama.cpp places whole layers, so the fit sizes each card from the +layers it gets, the way llama.cpp b11749 assigns them with `-ngl 999` +(`src/llama-model.cpp`, `load_tensors`). Layer `il` goes to the device +`upper_bound(cumsum(ts) / sum(ts), il / (layers + 1))`, and the output layer +(`il = layers`, holding `lm_head` and a GGUF's MTP tensors) goes the same way, which is +the last card. Each layer's weights are its share of the class totals, minus its experts +when it is one of the first `--n-cpu-moe` layers. The vision projector counts on the first +card: `tools/mtmd/clip.cpp` loads it on the first GPU. Each layer's KV and recurrent state +sit on its card. Every card also carries its own overhead. +The fit emits `-ts` as whole layer counts summing to layers + 1, so each card gets exactly +the layers it was sized for. It keeps every card within the smallest budget of the cards +used, because the device plugin, not the fit, decides which card is device 0 inside the +pod, and it balances the largest card's load. When no split fits, the placement is refused. +`--explain` lists each card's layers, weights, KV and total, and `card_bytes` in `--json` +has the same numbers. + The overhead is an estimate from the formula, not a measurement; `K3D_GPU_FIT_OVERHEAD_GIB` replaces it with a measured value. @@ -96,7 +114,7 @@ The overhead is an estimate from the formula, not a measurement; | --- | --- | | vLLM v0.32.0 | `--tensor-parallel-size`, `--max-model-len`, `--max-num-seqs`, `--gpu-memory-utilization` (vLLM still profiles activations inside that share), `--kv-cache-dtype fp8` when the checkpoint's `kv_cache_scheme` made the fit count one byte. CPU build (EAT): `VLLM_CPU_KVCACHE_SPACE` | | OVMS 2026.4.1 | `--max_num_seqs`, `--cache_size` (a static pool: the dynamic one grows past the fit) | -| llama.cpp | `-c` (context x slots), `-np`, `-ngl`, `-ts`, `--n-cpu-moe` or `--cpu-moe` | +| llama.cpp | `-c` (context x slots), `-np`, `-ngl`, `-ts` (layers per card), `--n-cpu-moe` or `--cpu-moe` | `agent up` applies them, sets the pod's memory request and limit to the fit, and annotates the pod `k3d-gpu.io/agent-fit`. `K3D_GPU_AGENT_MAX_LEN`, `K3D_GPU_AGENT_MAX_NUM_SEQS` diff --git a/tests/fit_test.py b/tests/fit_test.py index 56bf7440..c1574b78 100755 --- a/tests/fit_test.py +++ b/tests/fit_test.py @@ -12,6 +12,7 @@ class bytes from the real safetensors/GGUF headers at a pinned revision), Usage: python3 tests/fit_test.py """ +import bisect import gzip import json import os @@ -246,6 +247,11 @@ def config(model): return json.load(f) +def weights(model): + with open(os.path.join(mdir(model), "weights.json"), encoding="utf-8") as f: + return json.load(f) + + class KV(unittest.TestCase): """KV bytes from config.json, worked out by hand from each config's fields.""" @@ -641,5 +647,148 @@ def test_falls_to_fewer_bits_when_int4_does_not_fit(self): self.assertTrue(any("IQ4_NL" in s for s in d["explain"]) and any("Q2_K" in s for s in d["explain"])) +def llama_cards(d, cfg, w, kv_layer): + """Each card's GPU bytes for a llama.cpp plan, worked out here from its flags the way + llama.cpp b11749 assigns them (src/llama-model.cpp load_tensors, -ngl past the layer + count): layer il on device upper_bound(cumsum(ts) / sum(ts), il / (n_layer + 1)), the + output layer (il = n_layer) the same way, the experts of the first --n-cpu-moe layers in + host RAM, each layer's KV on its device, and each card's own overhead. Weights per layer + are the class totals over the layers; the output layer holds lm_head and the MTP tensors + (a GGUF's), and the vision projector is on the first GPU (tools/mtmd/clip.cpp). + kv_layer(il, ctx) is one sequence's KV of layer il.""" + tc = cfg.get("text_config") or cfg + n = tc["num_hidden_layers"] + f = d["flags"] + ts = [int(x) for x in f[f.index("-ts") + 1].split(",")] + ncpu = int(f[f.index("--n-cpu-moe") + 1]) if "--n-cpu-moe" in f else n if "--cpu-moe" in f else 0 + cls = {k: v["bytes"] for k, v in w["classes"].items()} + cum, c = [], 0 + for t in ts: + c += t + cum.append(c / sum(ts)) + cards = [0] * len(ts) + cards[0] += cls.get("vision", 0) + for il in range(n + 1): + dev = bisect.bisect_right(cum, il / (n + 1)) + if il == n: + cards[dev] += cls.get("lm_head", 0) + (cls.get("mtp", 0) if w["format"] == "gguf" else 0) + else: + cards[dev] += (cls.get("other", 0) / n + (cls.get("experts", 0) / n if il >= ncpu else 0) + + kv_layer(il, d["ctx"]) * d["seqs"]) + m = fit.model_profile(cfg, w) + ov = fit._overhead("llama.cpp", m, fit.TARGET_SEQS, None)[0] + return ts, [int(b + ov) for b in cards] + + +def flash_next_kv(il, ctx): + # every 4th layer full attention: 2 x 2 kv heads x 256 x 2 B + indexer 128 x 2 B per token; + # the rest gated DeltaNet: 48 x 128 x 128 x 4 B state + 3 x 10240 conv ch x 2 B per sequence + return (2 * 2 * 256 * 2 + 128 * 2) * ctx if il % 4 == 3 else 48 * 128 * 128 * 4 + 3 * 10240 * 2 + + +def gemma4_kv(il, ctx): + # every 6th layer global: 2 x 2 kv heads x 512 x 2 B per token; the rest a 1024-token window + return 2 * 2 * 512 * 2 * ctx if il % 6 == 5 else 2 * 8 * 256 * 2 * min(ctx, 1024) + + +class LlamaSplit(unittest.TestCase): + """llama.cpp over several GPUs: each card's own layers, KV and overhead within its budget.""" + + def budget(self, k): + return min(c["budget"] for c in fit.gpu_budget(host("xeon6741p-4xb70"))[0][:k]) + + def test_layer_devices(self): + # upper_bound(cumsum(ts)/sum, il/(n_layer+1)): -ts 30,30 over 48 layers puts 25 on the first card + self.assertEqual(fit.llama_layer_devices([30, 30], 48), [0] * 25 + [1] * 24) + # integer layer counts that sum to n_layer + 1 give exactly those counts, the output last + self.assertEqual(fit.llama_layer_devices([28, 21], 48), [0] * 28 + [1] * 21) + self.assertEqual(fit.llama_layer_devices([10, 0, 21], 30), [0] * 10 + [2] * 21) + + def test_old_split_overflows_the_second_card(self): + # The plan this replaces: -ts 30,30 --n-cpu-moe 28 at 262144 x 4 puts ~43.5 GiB on a 32 GiB card. + cfg = config(FLASH_GGUF) + w = weights(FLASH_GGUF)["groups"]["IQ4_NL"] + old = {"flags": ["-ts", "30,30", "--n-cpu-moe", "28"], "ctx": 262144, "seqs": 4} + _, cards = llama_cards(old, cfg, w, flash_next_kv) + self.assertGreater(cards[1], 32 * GIB) + + def check(self, d, cfg, w, kv_layer): + n = (cfg.get("text_config") or cfg)["num_hidden_layers"] + ts, cards = llama_cards(d, cfg, w, kv_layer) + self.assertEqual(sum(ts), n + 1, ts) + self.assertEqual(len(ts), d["gpus"]) + budget = self.budget(d["gpus"]) + for i, b in enumerate(cards): + self.assertLessEqual(b, budget, (i, ts, d["flags"])) + # the fit's own per-card bytes agree with the ones worked out here + self.assertEqual(len(d["card_bytes"]), len(ts)) + for got, want in zip(d["card_bytes"], cards): + self.assertAlmostEqual(got / GIB, want / GIB, delta=0.01) + self.assertEqual(d["gpu_bytes"], max(d["card_bytes"])) + + def test_flash_next_iq4nl_cards_within_budget(self): + d = plan("xeon6741p-4xb70", FLASH_GGUF) + self.assertTrue(d["fits"], d["summary"]) + self.assertEqual((d["placement"], d["gpus"]), ("moe-offload", 2)) + w = weights(FLASH_GGUF)["groups"]["IQ4_NL"] + self.check(d, config(FLASH_GGUF), w, flash_next_kv) + + def test_gemma_layer_split_cards_within_budget(self): + g = "google__gemma-4-26B-A4B-it-qat-q4_0-gguf" + d = plan("xeon6741p-4xb70", g, ctx=262144, seqs=6) + self.assertTrue(d["fits"], d["summary"]) + self.assertEqual((d["placement"], d["gpus"]), ("gpu-split", 2)) + self.check(d, config(g), weights(g), gemma4_kv) + + def test_layer_bytes_are_the_bytes_to_serve(self): + # Sized from safetensors for a GGUF conversion: the MTP head is not served + # (model_profile serve_bytes), and the vision projector loads on the first GPU + # (tools/mtmd/clip.cpp takes the first GPU backend), not on the output layer's card. + cfg = config("Qwen__Qwen3.8-Flash-Next") + m = fit.model_profile(cfg, weights("Qwen__Qwen3.8-Flash-Next")) + self.assertGreater(m["bytes"]["mtp"], GIB) + self.assertGreater(m["bytes"]["vision"], GIB / 2) + wl, _ = fit.llama_layer_bytes(m, fit.kv_profile(cfg, 2), 4096, 1, 0) + self.assertAlmostEqual(sum(wl), m["serve_bytes"] - m["lookup_bytes"], delta=1) + self.assertAlmostEqual(wl[-1], m["bytes"]["lm_head"], delta=1) + self.assertAlmostEqual(wl[0] - wl[1], m["bytes"]["vision"], delta=1) + + def test_moe_offload_takes_the_best_number_of_cards(self): + # Every card is held to the smallest budget of the cards used, so two smaller cards + # beside two large ones can leave less room than the two large ones alone. The fit + # tries 1, 2, ... of the free cards by budget and keeps the plan with the fewest + # CPU-expert layers, not every free card. + h = host("xeon6741p-4xb70") + for g, used in zip(h["gpus"], (0, 0, 12, 14)): + g["vram_used"], g["users"] = used * GIB, [] + model = mdir("Intel__Qwen3.8-Flash-Next-W4A16-AutoRound") + cfg = config("Intel__Qwen3.8-Flash-Next-W4A16-AutoRound") + n = (cfg.get("text_config") or cfg)["num_hidden_layers"] + + def ncpu(d): + f = d["flags"] + return int(f[f.index("--n-cpu-moe") + 1]) if "--n-cpu-moe" in f else n + + kw = dict(ctx=262144, seqs=6, device="gpu") + auto = fit.plan_with_variants(h, model, [], **kw) + self.assertTrue(auto["fits"], auto["summary"]) + self.assertEqual(auto["placement"], "moe-offload") + each = [fit.plan_with_variants(h, model, [], gpus=k, **kw) for k in range(1, 5)] + self.assertEqual(ncpu(auto), min(ncpu(d) for d in each if d["fits"])) + self.assertLess(auto["gpus"], 4) + budget = min(c["budget"] for c in fit.gpu_budget(h)[0][:auto["gpus"]]) + self.assertLessEqual(max(auto["card_bytes"]), budget) + + def test_moe_offload_takes_the_cards_asked_for(self): + # --gpus 3 on the Xeon, where two cards are free: refused, not two cards. + d = plan("xeon6741p-4xb70", FLASH_GGUF, gpus=3, device="gpu") + self.assertFalse(d["fits"]) + self.assertIn("3 GPUs asked, 2 not busy", d["reason"]) + + def test_too_much_for_any_split_is_refused(self): + d = plan("xeon6741p-4xb70", "google__gemma-4-26B-A4B-it-qat-q4_0-gguf", ctx=262144, seqs=12, device="gpu") + self.assertFalse(d["fits"]) + + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/tests/fit_test.sh b/tests/fit_test.sh index b655baf5..64f8ab2b 100755 --- a/tests/fit_test.sh +++ b/tests/fit_test.sh @@ -92,6 +92,25 @@ out=$(bash "${K3D_GPU}" fit --host "${XEON}" --model "${FLASH}" 2>&1); rc=$? if [ "${rc}" -eq 0 ] && grep -q "^fit: ${FLASH}:IQ4_NL on 2 GPU(s) + experts and lookup tables in DRAM" <<< "${out}"; then ok "fit: a multi-GGUF repo needs no flag (IQ4_NL picked, split shards summed)" else fail "fit: multi-GGUF repo auto-pick (rc ${rc})" "${out}"; fi +# Its two-card plan sizes each card the way llama.cpp assigns layers: -ts is integer layer +# counts (48 layers + the output layer), and no card's own bytes pass its budget. The plan +# this replaces, -ts 30,30 --n-cpu-moe 28, put 43.5 GiB on the second 32 GiB card. +out=$(bash "${K3D_GPU}" fit --host "${XEON}" --model "${FLASH}" --json 2>&1); rc=$? +if [ "${rc}" -eq 0 ] && ts=$(python3 -c ' +import json, sys +sys.path.insert(0, sys.argv[1]) +import fit +d = json.loads(sys.argv[2]) +f = d["flags"] +ts = [int(x) for x in f[f.index("-ts") + 1].split(",")] +budget = min(c["budget"] for c in fit.gpu_budget(json.load(open(sys.argv[3])))[0][:d["gpus"]]) +assert d["placement"] == "moe-offload" and len(ts) == d["gpus"] == 2 and sum(ts) == 49, (ts, d["gpus"]) +assert ts != [30, 30] and "--n-cpu-moe" in f, f +assert len(d["card_bytes"]) == 2 and max(d["card_bytes"]) <= budget, (d["card_bytes"], budget) +print(f[f.index("-ts") + 1]) +' "${ROOT}/console" "${out}" "${XEON}"); then + ok "fit: Flash-Next IQ4_NL on two cards, -ts ${ts} layers, every card within its budget" +else fail "fit: Flash-Next IQ4_NL per-card split (rc ${rc})" "$(grep -E '"-ts"|card_bytes' -A3 <<< "${out}")"; fi out=$(bash "${K3D_GPU}" fit --host "${XEON}" --model "${FLASH}" --file Qwen3.8-Flash-Next-Q8_0-00002-of-00002.gguf --explain 2>&1); rc=$? if [ "${rc}" -eq 0 ] && grep -q ':Q8_0 on' <<< "${out}" && grep -q -- "-hf ${FLASH}:Q8_0" <<< "${out}"; then ok "fit --file: a shard name picks its whole split model"