From e4ab3f738ba850c7006e446f686ab7c625db19b0 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Wed, 28 Oct 2026 22:58:19 -0700 Subject: [PATCH 1/2] fit: size llama.cpp's layer split card by card On several GPUs the fit treated llama.cpp's cards as one pool: the summed budgets less the summed weights, KV and overhead. It then wrote -ts as each card's budget in GiB. llama.cpp places whole layers, though, and with --n-cpu-moe the layers whose experts stay on the GPU are the last ones. The Xeon fixture's Flash-Next IQ4_NL plan, -ts 30,30 --n-cpu-moe 28 at 262144 x 4, therefore put 43.5 GiB on the second 32 GiB card. The fit now sizes each card from the layers llama.cpp b11749 gives it with -ngl 999 (src/llama-model.cpp load_tensors): - layer il goes to the device upper_bound(cumsum(ts) / sum, il / (layers + 1)); - the output layer, holding lm_head, goes the same way and lands on the last card; - the first --n-cpu-moe layers keep their experts in host RAM; - each layer's KV or recurrent state sits on its card; - every card carries its own overhead. -ts is now integer layer counts summing to layers + 1, so each card gets exactly the layers it was sized for. Every card is held to the smallest budget of the cards used, because the device plugin orders them inside the pod. The largest card load is balanced, and a split that cannot fit is refused. The experts-in-DRAM placement uses the same per-card search, so -ts and --n-cpu-moe are chosen together. --explain lists each card, and --json has card_bytes. Flash-Next IQ4_NL on the Xeon is now -ts 35,14 --n-cpu-moe 29 at 262144 x 4: 29.9 and 28.9 GiB on cards with 30.4 and 30.2 GiB budgets. Tests: - llama_layer_devices matches the upper_bound rule. - The old -ts 30,30 --n-cpu-moe 28 plan works out to 43.5 GiB on its second card. - For the Flash-Next IQ4_NL plan and a two-card Gemma QAT layer split, each card's bytes, recomputed in the test from the flags, stay within budget and agree with card_bytes. - A split that cannot fit is refused. - fit_test.sh checks the Flash-Next plan's -ts and card_bytes. --- console/fit.py | 244 +++++++++++++++++++++++++++++++++------------- docs/fit.md | 19 +++- tests/fit_test.py | 101 +++++++++++++++++++ tests/fit_test.sh | 19 ++++ 4 files changed, 314 insertions(+), 69 deletions(-) diff --git a/console/fit.py b/console/fit.py index 41211e77..fcad82cd 100644 --- a/console/fit.py +++ b/console/fit.py @@ -47,6 +47,7 @@ """ import argparse +import bisect import glob import json import math @@ -793,10 +794,10 @@ def kv_profile(cfg, kv_bytes): heads = tc.get("num_attention_heads") kvh = tc.get("num_key_value_heads") or heads hd = tc.get("head_dim") or tc["hidden_size"] // heads - full = sum(1 for t in types if t == "full_attention") - swa = sum(1 for t in types if t in ("sliding_attention", "sliding_window_attention", "local_attention")) - lin = sum(1 for t in types if t in ("linear_attention", "mamba", "gated_delta_net")) - full += len(types) - full - swa - lin # an unknown layer type counts as full attention (upper bound) + # an unknown layer type counts as full attention (upper bound) + kinds = ["swa" if t in ("sliding_attention", "sliding_window_attention", "local_attention") else + "linear" if t in ("linear_attention", "mamba", "gated_delta_net") else "full" for t in types] + full, swa, lin = kinds.count("full"), kinds.count("swa"), kinds.count("linear") gkvh = tc.get("num_global_key_value_heads") or kvh ghd = tc.get("global_head_dim") or hd lines = [] @@ -829,7 +830,7 @@ def kv_profile(cfg, kv_bytes): " = %s per sequence, any length" % (lin, vh, kd, vd, ssm_b, ck, ch, _mb(state))) return {"full_layers": full, "full_per_token": full * (per_full + idx), "swa_layers": swa, "swa_per_token": swa * per_swa, "window": window, "linear_layers": lin, - "state_per_seq": state, "kv_heads": gkvh, "lines": lines} + "state_per_seq": state, "kv_heads": gkvh, "kinds": kinds, "lines": lines} def _num(x): @@ -966,9 +967,116 @@ def act_bytes(m, tokens, seqs): return tokens * width * 2 * 2 + seqs * m["vocab"] * 4 +def kv_parts(kv, ctx): + """(full attention, sliding window, linear state) KV bytes of one sequence at context ctx.""" + return kv["full_per_token"] * ctx, kv["swa_per_token"] * min(ctx, kv["window"] or ctx), kv["state_per_seq"] + + def per_seq(kv, ctx, scale=1.0): - return int((kv["full_per_token"] * ctx + kv["swa_per_token"] * min(ctx, kv["window"] or ctx)) * scale - + kv["state_per_seq"] * scale) + f, w, st = kv_parts(kv, ctx) + return int((f + w) * scale + st * scale) + + +# ---- llama.cpp over several GPUs ---------------------------------------------------------- +def llama_layer_devices(ts, n_layer): + """Device of each repeating layer and of the output layer (the last entry), as llama.cpp + b11749 assigns them with -ngl past the layer count (src/llama-model.cpp load_tensors): + upper_bound(cumsum(ts) / sum(ts), il / (n_layer + 1)). Integer layer counts summing to + n_layer + 1 give each device exactly its count, the output layer on the last one.""" + total, cum, splits = float(sum(ts)), 0.0, [] + for t in ts: + cum += t + splits.append(cum / total) + return [bisect.bisect_right(splits, il / (n_layer + 1)) for il in range(n_layer + 1)] + + +def llama_layer_bytes(m, kv, ctx, seqs, n_cpu_moe): + """([weights], [KV]) on the GPU per repeating layer, the output layer last: the class + totals spread evenly over the layers, the experts of the first n_cpu_moe layers in host + RAM (--n-cpu-moe), and each layer's own KV (its device holds it) for seqs sequences. + The output layer holds lm_head, and the MTP and vision tensors when the GGUF has them.""" + n = max(m["layers"], 1) + exp, other = m["bytes"]["experts"] / n, m["bytes"]["other"] / n + f, w, st = kv_parts(kv, ctx) + per = {"full": f / max(kv["full_layers"], 1), "swa": w / max(kv["swa_layers"], 1), + "linear": st / max(kv["linear_layers"], 1)} + kinds = kv.get("kinds") or [] + wl = [other + (exp if il >= n_cpu_moe else 0) for il in range(n)] + kl = [per[kinds[il]] * seqs if il < len(kinds) else 0 for il in range(n)] + return wl + [m["bytes"]["lm_head"] + m["bytes"]["mtp"] + m["bytes"]["vision"]], kl + [0] + + +def llama_split(layer_bytes, k, room): + """-ts for k cards as integer layer counts (the last card's count includes the output + layer) that keep every card's layers within room bytes, its largest load as small as it + can be; None when no split does.""" + last = len(layer_bytes) - 1 + + def fill(cap): + ts, i = [], 0 + for _ in range(k - 1): + start, used = i, 0 + while i < last and used + layer_bytes[i] <= cap: + used += layer_bytes[i] + i += 1 + ts.append(i - start) + if sum(layer_bytes[i:]) > cap: + return None + return ts + [len(layer_bytes) - i] + + if fill(room) is None: + return None + lo, hi = max(layer_bytes), room + while hi - lo > MIB: + mid = (lo + hi) / 2 + lo, hi = (lo, mid) if fill(mid) else (mid, hi) + return fill(hi) + + +def llama_cards(ts, layer_bytes): + """Each card's bytes for ts: the layers llama_layer_devices puts on it.""" + out = [0] * len(ts) + for il, dev in enumerate(llama_layer_devices(ts, len(layer_bytes) - 1)): + out[dev] += layer_bytes[il] + return out + + +def _llama_search(m, kv, ladder, relaxed, want_seqs, k, room, n_cpus): + """(ctx, seqs, n_cpu_moe, ts) for llama.cpp on k cards of room bytes each: the most + context at which TARGET_SEQS (or want_seqs) sequences fit, then the fewest layers whose + experts leave the GPU (from n_cpus), then as many sequences as fit (capped at MAX_SEQS). + relaxed: at the ladder's last rung one sequence is enough. None when nothing fits.""" + for ctx in ladder: + need = want_seqs or (1 if relaxed and ctx == ladder[-1] else TARGET_SEQS) + for n_cpu in n_cpus: + def split(n): + wl, kl = llama_layer_bytes(m, kv, ctx, n, n_cpu) + return llama_split([a + b for a, b in zip(wl, kl)], k, room) + if not split(need): + continue + n = want_seqs + if not n: + lo, hi = need, MAX_SEQS + if split(hi): + lo = hi + while hi - lo > 1: + mid = (lo + hi) // 2 + lo, hi = (mid, hi) if split(mid) else (lo, mid) + n = lo + return ctx, n, n_cpu, split(n) + return None + + +def _llama_card_lines(use, ts, wl, kl, ov): + """One explain line per card of a llama.cpp split.""" + lines, first = [], 0 + for c, t, w, kvb in zip(use, ts, llama_cards(ts, wl), llama_cards(ts, kl)): + lines.append(" %s (-ts %d): layers %s, weights %s + KV %s + overhead %s = %s, budget %s" + % (c["card"], t, "%d-%d" % (first, first + t - 1) if t else "none", g1(w), g1(kvb), g1(ov), + g1(w + kvb + ov), g1(c["budget"]))) + first += t + lines.append(" (layer %d is the output layer: lm_head)" % (len(wl) - 1)) + return lines def ctx_ladder(max_ctx, want=None): @@ -1108,7 +1216,7 @@ def fit(host, cfg, weights, engine=None, ctx=None, seqs=None, gpus=None, kv_dtyp rung_kv.append(int(kb)) rung_limit.append("%dGi" % int(math.ceil(hb * (1 + LIMIT_SLACK) / GIB))) doc.update({k: picked[k] for k in ("engine", "placement", "ctx", "seqs", "tp", "gpus", "gpu_bytes", "host_bytes", - "flags", "dram_tok_s", "note", "kv_bytes")}) + "flags", "dram_tok_s", "note", "kv_bytes", "card_bytes")}) lim = int(math.ceil(picked["host_bytes"] * (1 + LIMIT_SLACK) / GIB)) doc["k8s"] = {"memory_request": "%dGi" % lim, "memory_limit": "%dGi" % lim} doc["guard"] = {"rungs": rungs, "placement": picked["placement"], "some_avg10": GUARD_SOME_AVG10, @@ -1146,7 +1254,7 @@ def _overhead(eng, m, seqs, override): def _res(eng, place, fits, why="", math_lines=(), **kw): r = {"engine": eng, "placement": place, "fits": fits, "why": why, "math": list(math_lines), "ctx": None, "seqs": 0, "tp": 1, "gpus": 0, "gpu_bytes": 0, "host_bytes": 0, "flags": [], "dram_tok_s": None, - "kv_bytes": 0, "kv_scale": 1.0} + "kv_bytes": 0, "kv_scale": 1.0, "card_bytes": []} r.update(kw) return r @@ -1187,6 +1295,9 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, yield _res(eng, place, False, "%d GPUs asked, %d not busy" % (k, len(cards)), gpus=k) continue use = cards[:k] + if eng == "llama.cpp" and k > 1: + yield _llama_multi(m, kv, ladder, relaxed, want_seqs, use, ov, ovl, [0], on_host, hfree, bw) + continue if eng == "vllm": # Every rank holds 1/k of the weights; KV heads split across ranks, or one # per rank (replicated) with fewer KV heads than ranks. @@ -1197,8 +1308,7 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, lines = ["per card: budget %s - weights %s / %d - overhead %s = %s for KV" % (g1(budget), g1(gpu_w_all), k, ovl, g1(max(left, 0)))] else: - # One card, or llama.cpp's layer split (-sm layer, the default): weights and - # KV spread over the cards in proportion to -ts; each card runs its own buffers. + # One card (llama.cpp on several is sized card by card in _llama_multi). budget = sum(c["budget"] for c in use) w = gpu_w_all left = budget - w - ov * k @@ -1226,66 +1336,19 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, continue gpu_use = int(w + ov + kvused) if eng == "vllm" else int((w + ov * k + kvused) / k) yield _res(eng, place, True, "", lines, ctx=c, seqs=n, tp=k if eng == "vllm" else 1, gpus=k, - gpu_bytes=gpu_use, host_bytes=hb, kv_bytes=int(kvused), kv_scale=scale, + gpu_bytes=gpu_use, host_bytes=hb, kv_bytes=int(kvused), kv_scale=scale, card_bytes=[gpu_use] * k, flags=_flags(eng, place, c, n, k, gpu_use, use, kvused, m, 0, kv)) # MoE experts to DRAM (llama.cpp --n-cpu-moe): every free card, fewest CPU layers. if eng == "llama.cpp" and cards and m["moe"]: - k = want_gpus or len(cards) - use = cards[:k] - budget = sum(c["budget"] for c in use) - exp_layer = m["bytes"]["experts"] / max(m["layers"], 1) - base = gpu_w_all - m["bytes"]["experts"] - lines = ["GPU (%s, budget %s) keeps %s non-expert weights + overhead %d x %s" - % ("+".join(c["card"] for c in use), g1(budget), g1(base), k, ovl), - "lookup tables %s in host RAM" % g1(on_host)] + use = cards[:want_gpus or len(cards)] if not (cpu["amx"] or cpu["avx512"]): - yield _res(eng, "moe-offload", False, "the CPU has neither AMX nor AVX-512 for expert compute", lines, gpus=k) + yield _res(eng, "moe-offload", False, "the CPU has neither AMX nor AVX-512 for expert compute", gpus=len(use)) elif not bw: yield _res(eng, "moe-offload", False, "DRAM bandwidth unknown (no STREAM run cached, SMBIOS unreadable)", - lines, gpus=k) + gpus=len(use)) else: - # Most context first (as everywhere), then the fewest layers whose experts leave the GPU. - best = None - for rung in ladder: - for n_cpu in range(1, m["layers"] + 1): - gw = base + exp_layer * (m["layers"] - n_cpu) - left = budget - gw - ov * k - if left <= 0: - continue - c, n = size_kv(kv, left, [rung], want_seqs, 1.0, relaxed and rung == ladder[-1]) - if c: - best = (n_cpu, gw, c, n) - break - if best: - break - if not best: - yield _res(eng, "moe-offload", False, "non-expert weights + KV do not fit %s of VRAM" % g1(budget), - lines, gpus=k) - else: - n_cpu, gw, c, n = best - dram_w = exp_layer * n_cpu - read = dram_w * m["experts_per_tok"] / m["experts"] - toks = _dram_rate(bw, read) - lines.append("experts of the first %d of %d layers in DRAM: %s; on GPU %s" - % (n_cpu, m["layers"], g1(dram_w), g1(gw))) - lines.append("DRAM decode bound: %g GB/s / (%s x %d/%d read per token) = %.0f tok/s (floor %g)" - % (bw, g1(dram_w), m["experts_per_tok"], m["experts"], toks, MIN_DRAM_TOK_S)) - kvused = per_seq(kv, c) * n - lines.append("KV: %d seqs x %s at %d tokens = %s on GPU" % (n, _mb(per_seq(kv, c)), c, g1(kvused))) - hb = int(dram_w + on_host + HOST_RUNTIME[eng]) - lines.append("host: experts %s + lookup tables %s + runtime %s = %s" - % (g1(dram_w), g1(on_host), g1(HOST_RUNTIME[eng]), g1(hb))) - if toks < MIN_DRAM_TOK_S: - yield _res(eng, "moe-offload", False, "DRAM bandwidth gives %.1f tok/s, under the %g floor" - % (toks, MIN_DRAM_TOK_S), lines, gpus=k) - elif hb > hfree: - yield _res(eng, "moe-offload", False, "DRAM side needs %s, host budget %s" % (g1(hb), g1(hfree)), - lines, gpus=k) - else: - gpu_use = int((gw + ov * k + kvused) / k) - yield _res(eng, "moe-offload", True, "", lines, ctx=c, seqs=n, gpus=k, gpu_bytes=gpu_use, - host_bytes=hb, dram_tok_s=round(toks, 1), kv_bytes=int(kvused), - flags=_flags(eng, "moe-offload", c, n, k, gpu_use, use, kvused, m, n_cpu, kv)) + yield _llama_multi(m, kv, ladder, relaxed, want_seqs, use, ov, ovl, range(1, m["layers"] + 1), on_host, + hfree, bw) # CPU only: weights, KV and activations all in DRAM (vLLM: its CPU build, the KV # pool sized by VLLM_CPU_KVCACHE_SPACE, as EAT's vLLM chart sets it). if cpu_ok: @@ -1321,7 +1384,54 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, flags=_flags(eng, "cpu", c, n, 1, 0, [], kvused, m, 0, kv)) -def _flags(eng, place, ctx, seqs, k, gpu_use, cards, kvused, m, n_cpu_moe, kv=None): +def _llama_multi(m, kv, ladder, relaxed, want_seqs, use, ov, ovl, n_cpus, on_host, hfree, bw): + """llama.cpp on the cards in use, each holding its own layers, their KV and its own + overhead within the smallest budget among them (the device plugin, not the fit, orders + the cards inside the pod): a layer split (n_cpus [0]) or experts in DRAM (n_cpus 1..).""" + k = len(use) + place = "gpu-split" if list(n_cpus) == [0] else "moe-offload" + budget = min(c["budget"] for c in use) + room = budget - ov + lines = ["%s: layer il on card upper_bound(cumsum(-ts) / sum, il / (layers + 1)) (src/llama-model.cpp); " + "each card: budget %s (the smallest of them) - overhead %s = %s for its layers" + % ("+".join(c["card"] for c in use), g1(budget), ovl, g1(max(room, 0)))] + if on_host: + lines.append("lookup tables %s stay in host RAM" % g1(on_host)) + found = _llama_search(m, kv, ladder, relaxed, want_seqs, k, room, n_cpus) if room > 0 else None + if not found: + why = ("weights + KV for %d-token sequences do not fit %d card(s) of %s each, layer by layer" + % (ladder[-1], k, g1(budget))) + return _res("llama.cpp", place, False, why, lines, gpus=k) + c, n, n_cpu, ts = found + wl, kl = llama_layer_bytes(m, kv, c, n, n_cpu) + cards_b = [int(b + ov) for b in llama_cards(ts, [a + b for a, b in zip(wl, kl)])] + lines += _llama_card_lines(use, ts, wl, kl, ov) + kvused = per_seq(kv, c) * n + lines.append("KV: %d seqs x %s at %d tokens = %s over the cards" % (n, _mb(per_seq(kv, c)), c, g1(kvused))) + hb = int(HOST_RUNTIME["llama.cpp"] + on_host) + toks = None + if n_cpu: + exp_layer = m["bytes"]["experts"] / max(m["layers"], 1) + dram_w = exp_layer * n_cpu + toks = _dram_rate(bw, dram_w * m["experts_per_tok"] / m["experts"]) + lines.append("experts of the first %d of %d layers in DRAM: %s" % (n_cpu, m["layers"], g1(dram_w))) + lines.append("DRAM decode bound: %g GB/s / (%s x %d/%d read per token) = %.0f tok/s (floor %g)" + % (bw, g1(dram_w), m["experts_per_tok"], m["experts"], toks, MIN_DRAM_TOK_S)) + hb += int(dram_w) + lines.append("host: experts %s + lookup tables %s + runtime %s = %s" + % (g1(dram_w), g1(on_host), g1(HOST_RUNTIME["llama.cpp"]), g1(hb))) + if toks < MIN_DRAM_TOK_S: + return _res("llama.cpp", place, False, "DRAM bandwidth gives %.1f tok/s, under the %g floor" + % (toks, MIN_DRAM_TOK_S), lines, gpus=k) + if hb > hfree: + return _res("llama.cpp", place, False, "host side %s exceeds the host budget %s" % (g1(hb), g1(hfree)), + lines, gpus=k) + return _res("llama.cpp", place, True, "", lines, ctx=c, seqs=n, gpus=k, gpu_bytes=max(cards_b), card_bytes=cards_b, + host_bytes=hb, dram_tok_s=round(toks, 1) if toks else None, kv_bytes=int(kvused), + flags=_flags("llama.cpp", place, c, n, k, max(cards_b), use, kvused, m, n_cpu, kv, ts)) + + +def _flags(eng, place, ctx, seqs, k, gpu_use, cards, kvused, m, n_cpu_moe, kv=None, ts=None): """Engine memory flags from the fit. Spellings from each engine's own argument parser: vLLM v0.32.0 vllm/engine/arg_utils.py, OVMS v2026.4.1 src/graph_export/graph_cli_parser.cpp, llama.cpp 1242be6 common/arg.cpp.""" @@ -1343,8 +1453,8 @@ def _flags(eng, place, ctx, seqs, k, gpu_use, cards, kvused, m, n_cpu_moe, kv=No # -c is the total context; with -np set explicitly each slot gets c/np (kv-unified is # on only when the slot count is auto). out = ["-c", str(ctx * seqs), "-np", str(seqs), "-ngl", "0" if place == "cpu" else "999"] - if k > 1: - out += ["-ts", ",".join(str(max(int(c["budget"] // GIB), 1)) for c in cards)] + if ts and k > 1: + out += ["-ts", ",".join(str(t) for t in ts)] # layers per card (llama_layer_devices) if place == "moe-offload": out += ["--cpu-moe"] if n_cpu_moe >= m["layers"] else ["--n-cpu-moe", str(n_cpu_moe)] t = {"fp16": "f16"}.get(kv.get("dtype"), kv.get("dtype")) diff --git a/docs/fit.md b/docs/fit.md index ab06e5f1..8f03dc06 100644 --- a/docs/fit.md +++ b/docs/fit.md @@ -57,7 +57,7 @@ in this order, and the first that fits wins: 1. one GPU; 2. several GPUs: vLLM tensor parallel (heads divisible by the size), or llama.cpp's - layer split; + layer split, sized card by card (below); 3. MoE experts in DRAM (llama.cpp `--n-cpu-moe N`): only when the GPU placements do not fit, the CPU has AMX or AVX-512, and the DRAM side's bandwidth bound per token stays above a floor. The fit takes the fewest layers whose experts leave the GPU; @@ -87,6 +87,21 @@ llama.cpp keeps the token embeddings and the n-gram table on the CPU by design bytes on every llama.cpp placement. OpenVINO's GPU plugin compiles from a host copy, so an OVMS GPU placement counts the weights as host bytes too. +On several GPUs llama.cpp places whole layers, so the fit sizes each card from the +layers it gets, the way llama.cpp b11749 assigns them with `-ngl 999` +(`src/llama-model.cpp`, `load_tensors`). Layer `il` goes to the device +`upper_bound(cumsum(ts) / sum(ts), il / (layers + 1))`, and the output layer +(`il = layers`, holding `lm_head`) goes the same way, which is the last card. Each +layer's weights are its share of the class totals, minus its experts when it is one of +the first `--n-cpu-moe` layers. Each layer's KV and recurrent state sit on its card. Every +card also carries its own overhead. +The fit emits `-ts` as whole layer counts summing to layers + 1, so each card gets exactly +the layers it was sized for. It keeps every card within the smallest budget of the cards +used, because the device plugin, not the fit, decides which card is device 0 inside the +pod, and it balances the largest card's load. When no split fits, the placement is refused. +`--explain` lists each card's layers, weights, KV and total, and `card_bytes` in `--json` +has the same numbers. + The overhead is an estimate from the formula, not a measurement; `K3D_GPU_FIT_OVERHEAD_GIB` replaces it with a measured value. @@ -96,7 +111,7 @@ The overhead is an estimate from the formula, not a measurement; | --- | --- | | vLLM v0.32.0 | `--tensor-parallel-size`, `--max-model-len`, `--max-num-seqs`, `--gpu-memory-utilization` (vLLM still profiles activations inside that share), `--kv-cache-dtype fp8` when the checkpoint's `kv_cache_scheme` made the fit count one byte. CPU build (EAT): `VLLM_CPU_KVCACHE_SPACE` | | OVMS 2026.4.1 | `--max_num_seqs`, `--cache_size` (a static pool: the dynamic one grows past the fit) | -| llama.cpp | `-c` (context x slots), `-np`, `-ngl`, `-ts`, `--n-cpu-moe` or `--cpu-moe` | +| llama.cpp | `-c` (context x slots), `-np`, `-ngl`, `-ts` (layers per card), `--n-cpu-moe` or `--cpu-moe` | `agent up` applies them, sets the pod's memory request and limit to the fit, and annotates the pod `k3d-gpu.io/agent-fit`. `K3D_GPU_AGENT_MAX_LEN`, `K3D_GPU_AGENT_MAX_NUM_SEQS` diff --git a/tests/fit_test.py b/tests/fit_test.py index 56bf7440..b9845281 100755 --- a/tests/fit_test.py +++ b/tests/fit_test.py @@ -12,6 +12,7 @@ class bytes from the real safetensors/GGUF headers at a pinned revision), Usage: python3 tests/fit_test.py """ +import bisect import gzip import json import os @@ -246,6 +247,11 @@ def config(model): return json.load(f) +def weights(model): + with open(os.path.join(mdir(model), "weights.json"), encoding="utf-8") as f: + return json.load(f) + + class KV(unittest.TestCase): """KV bytes from config.json, worked out by hand from each config's fields.""" @@ -641,5 +647,100 @@ def test_falls_to_fewer_bits_when_int4_does_not_fit(self): self.assertTrue(any("IQ4_NL" in s for s in d["explain"]) and any("Q2_K" in s for s in d["explain"])) +def llama_cards(d, cfg, w, kv_layer): + """Each card's GPU bytes for a llama.cpp plan, worked out here from its flags the way + llama.cpp b11749 assigns them (src/llama-model.cpp load_tensors, -ngl past the layer + count): layer il on device upper_bound(cumsum(ts) / sum(ts), il / (n_layer + 1)), the + output layer (il = n_layer) the same way, the experts of the first --n-cpu-moe layers in + host RAM, each layer's KV on its device, and each card's own overhead. Weights per layer + are the class totals over the layers. kv_layer(il, ctx) is one sequence's KV of layer il.""" + tc = cfg.get("text_config") or cfg + n = tc["num_hidden_layers"] + f = d["flags"] + ts = [int(x) for x in f[f.index("-ts") + 1].split(",")] + ncpu = int(f[f.index("--n-cpu-moe") + 1]) if "--n-cpu-moe" in f else n if "--cpu-moe" in f else 0 + cls = {k: v["bytes"] for k, v in w["classes"].items()} + cum, c = [], 0 + for t in ts: + c += t + cum.append(c / sum(ts)) + cards = [0] * len(ts) + for il in range(n + 1): + dev = bisect.bisect_right(cum, il / (n + 1)) + if il == n: + cards[dev] += cls.get("lm_head", 0) + cls.get("mtp", 0) + cls.get("vision", 0) + else: + cards[dev] += (cls.get("other", 0) / n + (cls.get("experts", 0) / n if il >= ncpu else 0) + + kv_layer(il, d["ctx"]) * d["seqs"]) + m = fit.model_profile(cfg, w) + ov = fit._overhead("llama.cpp", m, fit.TARGET_SEQS, None)[0] + return ts, [int(b + ov) for b in cards] + + +def flash_next_kv(il, ctx): + # every 4th layer full attention: 2 x 2 kv heads x 256 x 2 B + indexer 128 x 2 B per token; + # the rest gated DeltaNet: 48 x 128 x 128 x 4 B state + 3 x 10240 conv ch x 2 B per sequence + return (2 * 2 * 256 * 2 + 128 * 2) * ctx if il % 4 == 3 else 48 * 128 * 128 * 4 + 3 * 10240 * 2 + + +def gemma4_kv(il, ctx): + # every 6th layer global: 2 x 2 kv heads x 512 x 2 B per token; the rest a 1024-token window + return 2 * 2 * 512 * 2 * ctx if il % 6 == 5 else 2 * 8 * 256 * 2 * min(ctx, 1024) + + +class LlamaSplit(unittest.TestCase): + """llama.cpp over several GPUs: each card's own layers, KV and overhead within its budget.""" + + def budget(self, k): + return min(c["budget"] for c in fit.gpu_budget(host("xeon6741p-4xb70"))[0][:k]) + + def test_layer_devices(self): + # upper_bound(cumsum(ts)/sum, il/(n_layer+1)): -ts 30,30 over 48 layers puts 25 on the first card + self.assertEqual(fit.llama_layer_devices([30, 30], 48), [0] * 25 + [1] * 24) + # integer layer counts that sum to n_layer + 1 give exactly those counts, the output last + self.assertEqual(fit.llama_layer_devices([28, 21], 48), [0] * 28 + [1] * 21) + self.assertEqual(fit.llama_layer_devices([10, 0, 21], 30), [0] * 10 + [2] * 21) + + def test_old_split_overflows_the_second_card(self): + # The plan this replaces: -ts 30,30 --n-cpu-moe 28 at 262144 x 4 puts ~43.5 GiB on a 32 GiB card. + cfg = config(FLASH_GGUF) + w = weights(FLASH_GGUF)["groups"]["IQ4_NL"] + old = {"flags": ["-ts", "30,30", "--n-cpu-moe", "28"], "ctx": 262144, "seqs": 4} + _, cards = llama_cards(old, cfg, w, flash_next_kv) + self.assertGreater(cards[1], 32 * GIB) + + def check(self, d, cfg, w, kv_layer): + n = (cfg.get("text_config") or cfg)["num_hidden_layers"] + ts, cards = llama_cards(d, cfg, w, kv_layer) + self.assertEqual(sum(ts), n + 1, ts) + self.assertEqual(len(ts), d["gpus"]) + budget = self.budget(d["gpus"]) + for i, b in enumerate(cards): + self.assertLessEqual(b, budget, (i, ts, d["flags"])) + # the fit's own per-card bytes agree with the ones worked out here + self.assertEqual(len(d["card_bytes"]), len(ts)) + for got, want in zip(d["card_bytes"], cards): + self.assertAlmostEqual(got / GIB, want / GIB, delta=0.01) + self.assertEqual(d["gpu_bytes"], max(d["card_bytes"])) + + def test_flash_next_iq4nl_cards_within_budget(self): + d = plan("xeon6741p-4xb70", FLASH_GGUF) + self.assertTrue(d["fits"], d["summary"]) + self.assertEqual((d["placement"], d["gpus"]), ("moe-offload", 2)) + w = weights(FLASH_GGUF)["groups"]["IQ4_NL"] + self.check(d, config(FLASH_GGUF), w, flash_next_kv) + + def test_gemma_layer_split_cards_within_budget(self): + g = "google__gemma-4-26B-A4B-it-qat-q4_0-gguf" + d = plan("xeon6741p-4xb70", g, ctx=262144, seqs=6) + self.assertTrue(d["fits"], d["summary"]) + self.assertEqual((d["placement"], d["gpus"]), ("gpu-split", 2)) + self.check(d, config(g), weights(g), gemma4_kv) + + def test_too_much_for_any_split_is_refused(self): + d = plan("xeon6741p-4xb70", "google__gemma-4-26B-A4B-it-qat-q4_0-gguf", ctx=262144, seqs=12, device="gpu") + self.assertFalse(d["fits"]) + + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/tests/fit_test.sh b/tests/fit_test.sh index b655baf5..64f8ab2b 100755 --- a/tests/fit_test.sh +++ b/tests/fit_test.sh @@ -92,6 +92,25 @@ out=$(bash "${K3D_GPU}" fit --host "${XEON}" --model "${FLASH}" 2>&1); rc=$? if [ "${rc}" -eq 0 ] && grep -q "^fit: ${FLASH}:IQ4_NL on 2 GPU(s) + experts and lookup tables in DRAM" <<< "${out}"; then ok "fit: a multi-GGUF repo needs no flag (IQ4_NL picked, split shards summed)" else fail "fit: multi-GGUF repo auto-pick (rc ${rc})" "${out}"; fi +# Its two-card plan sizes each card the way llama.cpp assigns layers: -ts is integer layer +# counts (48 layers + the output layer), and no card's own bytes pass its budget. The plan +# this replaces, -ts 30,30 --n-cpu-moe 28, put 43.5 GiB on the second 32 GiB card. +out=$(bash "${K3D_GPU}" fit --host "${XEON}" --model "${FLASH}" --json 2>&1); rc=$? +if [ "${rc}" -eq 0 ] && ts=$(python3 -c ' +import json, sys +sys.path.insert(0, sys.argv[1]) +import fit +d = json.loads(sys.argv[2]) +f = d["flags"] +ts = [int(x) for x in f[f.index("-ts") + 1].split(",")] +budget = min(c["budget"] for c in fit.gpu_budget(json.load(open(sys.argv[3])))[0][:d["gpus"]]) +assert d["placement"] == "moe-offload" and len(ts) == d["gpus"] == 2 and sum(ts) == 49, (ts, d["gpus"]) +assert ts != [30, 30] and "--n-cpu-moe" in f, f +assert len(d["card_bytes"]) == 2 and max(d["card_bytes"]) <= budget, (d["card_bytes"], budget) +print(f[f.index("-ts") + 1]) +' "${ROOT}/console" "${out}" "${XEON}"); then + ok "fit: Flash-Next IQ4_NL on two cards, -ts ${ts} layers, every card within its budget" +else fail "fit: Flash-Next IQ4_NL per-card split (rc ${rc})" "$(grep -E '"-ts"|card_bytes' -A3 <<< "${out}")"; fi out=$(bash "${K3D_GPU}" fit --host "${XEON}" --model "${FLASH}" --file Qwen3.8-Flash-Next-Q8_0-00002-of-00002.gguf --explain 2>&1); rc=$? if [ "${rc}" -eq 0 ] && grep -q ':Q8_0 on' <<< "${out}" && grep -q -- "-hf ${FLASH}:Q8_0" <<< "${out}"; then ok "fit --file: a shard name picks its whole split model" From a8d9b66ead5e6652834625fbf67ebb297422ec34 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Wed, 28 Oct 2026 23:28:37 -0700 Subject: [PATCH 2/2] fit: llama.cpp per-card bytes follow the bytes to serve; moe-offload picks its card count The per-layer weights counted the MTP head for every format, so a model sized from safetensors for a GGUF conversion carried 4.9 GiB (Qwen3.8-Flash-Next) on its output layer that model_profile leaves out of the bytes to serve. One-card moe-offload plans therefore no longer matched the old arithmetic: Intel/Qwen3.8-Flash-Next-W4A16-AutoRound with --gpus 1 on the fixture Xeon went from --n-cpu-moe 43 to 47. The output layer now holds the bytes to serve less the lookup tables, experts, other and vision: lm_head, plus MTP only when the GGUF holds it. The one-card plan is --n-cpu-moe 43 again. The vision projector was put on the output layer, the last card. llama.cpp loads the projector through tools/mtmd/clip.cpp, which takes the first GPU backend, so it now counts with layer 0 on the first card, and --explain says so. moe-offload always used every free card. Each card is held to the smallest budget of the cards used, so a smaller card beside two large ones lowers the room on all of them. On the fixture Xeon with 0, 0, 12 and 14 GiB in use on its cards (budgets 30.4, 30.4, 18.4 and 16.4 GiB), Intel/Qwen3.8-Flash-Next-W4A16-AutoRound at 262144 x 6 with --device gpu was refused ("do not fit 4 card(s) of 16.4 GiB each") while --gpus 2 fit. The fit now tries 1, 2, ... of the free cards by budget (or the --gpus count) and keeps the plan with the most context, then the fewest CPU-expert layers, then the most sequences, then the fewest cards; that case is now two cards, -ts 31,18 --n-cpu-moe 41. --gpus above the free card count is refused, as the GPU placements already refuse it, instead of running on fewer cards. The fixture Xeon's GGUF plan is unchanged: -ts 35,14 --n-cpu-moe 29 at 262144 x 4. Tests (each fails on the previous commit): - llama_layer_bytes adds up to the bytes to serve less the lookup tables, its output layer is lm_head alone and layer 0 carries the vision projector, for Qwen3.8-Flash-Next from safetensors. - On the mixed-budget Xeon, the automatic moe-offload plan fits, has no more CPU-expert layers than any --gpus 1..4 plan, uses fewer than four cards and keeps every card in budget. - --gpus 3 on the fixture Xeon, two cards free, is refused for moe-offload. - The test's own per-card arithmetic follows the same vision and MTP rule. --- console/fit.py | 45 ++++++++++++++++++++++++++++++---------- docs/fit.md | 13 +++++++----- tests/fit_test.py | 52 +++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 92 insertions(+), 18 deletions(-) diff --git a/console/fit.py b/console/fit.py index fcad82cd..b0afaf4d 100644 --- a/console/fit.py +++ b/console/fit.py @@ -994,16 +994,21 @@ def llama_layer_bytes(m, kv, ctx, seqs, n_cpu_moe): """([weights], [KV]) on the GPU per repeating layer, the output layer last: the class totals spread evenly over the layers, the experts of the first n_cpu_moe layers in host RAM (--n-cpu-moe), and each layer's own KV (its device holds it) for seqs sequences. - The output layer holds lm_head, and the MTP and vision tensors when the GGUF has them.""" + The weights add up to the bytes to serve less the lookup tables (MTP only when the GGUF + holds it). The output layer holds lm_head and the MTP tensors; the vision projector + counts with layer 0, on the first card: tools/mtmd/clip.cpp loads it on the first GPU.""" n = max(m["layers"], 1) - exp, other = m["bytes"]["experts"] / n, m["bytes"]["other"] / n + b = m["bytes"] + exp, other = b["experts"] / n, b["other"] / n f, w, st = kv_parts(kv, ctx) per = {"full": f / max(kv["full_layers"], 1), "swa": w / max(kv["swa_layers"], 1), "linear": st / max(kv["linear_layers"], 1)} kinds = kv.get("kinds") or [] wl = [other + (exp if il >= n_cpu_moe else 0) for il in range(n)] + wl[0] += b["vision"] kl = [per[kinds[il]] * seqs if il < len(kinds) else 0 for il in range(n)] - return wl + [m["bytes"]["lm_head"] + m["bytes"]["mtp"] + m["bytes"]["vision"]], kl + [0] + out = m["serve_bytes"] - m["lookup_bytes"] - b["experts"] - b["other"] - b["vision"] + return wl + [out], kl + [0] def llama_split(layer_bytes, k, room): @@ -1338,17 +1343,32 @@ def _placements(eng, m, kv, ladder, relaxed, want_seqs, want_gpus, cards, hfree, yield _res(eng, place, True, "", lines, ctx=c, seqs=n, tp=k if eng == "vllm" else 1, gpus=k, gpu_bytes=gpu_use, host_bytes=hb, kv_bytes=int(kvused), kv_scale=scale, card_bytes=[gpu_use] * k, flags=_flags(eng, place, c, n, k, gpu_use, use, kvused, m, 0, kv)) - # MoE experts to DRAM (llama.cpp --n-cpu-moe): every free card, fewest CPU layers. + # MoE experts to DRAM (llama.cpp --n-cpu-moe). Every card is held to the smallest budget + # of the cards used, so one more card can leave less room per card: the free cards by + # budget, 1, 2, ... of them (or --gpus), and the plan with the most context, then the + # fewest CPU-expert layers, then the most sequences, then the fewest cards. if eng == "llama.cpp" and cards and m["moe"]: - use = cards[:want_gpus or len(cards)] - if not (cpu["amx"] or cpu["avx512"]): - yield _res(eng, "moe-offload", False, "the CPU has neither AMX nor AVX-512 for expert compute", gpus=len(use)) + ks = [want_gpus] if want_gpus else list(range(1, len(cards) + 1)) + if want_gpus and want_gpus > len(cards): + yield _res(eng, "moe-offload", False, "%d GPUs asked, %d not busy" % (want_gpus, len(cards)), gpus=want_gpus) + elif not (cpu["amx"] or cpu["avx512"]): + yield _res(eng, "moe-offload", False, "the CPU has neither AMX nor AVX-512 for expert compute", gpus=ks[-1]) elif not bw: yield _res(eng, "moe-offload", False, "DRAM bandwidth unknown (no STREAM run cached, SMBIOS unreadable)", - gpus=len(use)) + gpus=ks[-1]) else: - yield _llama_multi(m, kv, ladder, relaxed, want_seqs, use, ov, ovl, range(1, m["layers"] + 1), on_host, - hfree, bw) + tries = [_llama_multi(m, kv, ladder, relaxed, want_seqs, cards[:k], ov, ovl, range(1, m["layers"] + 1), + on_host, hfree, bw) for k in ks] + ok = [r for r in tries if r["fits"]] + if not ok: + yield tries[-1] + else: + best = max(ok, key=lambda r: (r["ctx"], -r["n_cpu_moe"], r["seqs"], -r["gpus"])) + if len(tries) > 1: + best["math"].append("cards tried, by budget: " + "; ".join( + "%d: %d x %d, experts of %d layers in DRAM" % (r["gpus"], r["ctx"], r["seqs"], r["n_cpu_moe"]) + if r["fits"] else "%d: no" % r["gpus"] for r in tries)) + yield best # CPU only: weights, KV and activations all in DRAM (vLLM: its CPU build, the KV # pool sized by VLLM_CPU_KVCACHE_SPACE, as EAT's vLLM chart sets it). if cpu_ok: @@ -1406,6 +1426,9 @@ def _llama_multi(m, kv, ladder, relaxed, want_seqs, use, ov, ovl, n_cpus, on_hos wl, kl = llama_layer_bytes(m, kv, c, n, n_cpu) cards_b = [int(b + ov) for b in llama_cards(ts, [a + b for a, b in zip(wl, kl)])] lines += _llama_card_lines(use, ts, wl, kl, ov) + if m["bytes"]["vision"]: + lines.append(" (%s holds the vision projector, %s: tools/mtmd/clip.cpp loads it on the first GPU)" + % (use[0]["card"], g1(m["bytes"]["vision"]))) kvused = per_seq(kv, c) * n lines.append("KV: %d seqs x %s at %d tokens = %s over the cards" % (n, _mb(per_seq(kv, c)), c, g1(kvused))) hb = int(HOST_RUNTIME["llama.cpp"] + on_host) @@ -1427,7 +1450,7 @@ def _llama_multi(m, kv, ladder, relaxed, want_seqs, use, ov, ovl, n_cpus, on_hos return _res("llama.cpp", place, False, "host side %s exceeds the host budget %s" % (g1(hb), g1(hfree)), lines, gpus=k) return _res("llama.cpp", place, True, "", lines, ctx=c, seqs=n, gpus=k, gpu_bytes=max(cards_b), card_bytes=cards_b, - host_bytes=hb, dram_tok_s=round(toks, 1) if toks else None, kv_bytes=int(kvused), + host_bytes=hb, dram_tok_s=round(toks, 1) if toks else None, kv_bytes=int(kvused), n_cpu_moe=n_cpu, flags=_flags("llama.cpp", place, c, n, k, max(cards_b), use, kvused, m, n_cpu, kv, ts)) diff --git a/docs/fit.md b/docs/fit.md index 8f03dc06..ee9b6849 100644 --- a/docs/fit.md +++ b/docs/fit.md @@ -60,7 +60,9 @@ in this order, and the first that fits wins: layer split, sized card by card (below); 3. MoE experts in DRAM (llama.cpp `--n-cpu-moe N`): only when the GPU placements do not fit, the CPU has AMX or AVX-512, and the DRAM side's bandwidth bound per token stays - above a floor. The fit takes the fewest layers whose experts leave the GPU; + above a floor. The fit takes the fewest layers whose experts leave the GPU, trying + 1, 2, ... of the free cards (largest budget first) and keeping the plan with the most + context, then the fewest such layers, then the most sequences, then the fewest cards; 4. CPU only. Per placement: @@ -91,10 +93,11 @@ On several GPUs llama.cpp places whole layers, so the fit sizes each card from t layers it gets, the way llama.cpp b11749 assigns them with `-ngl 999` (`src/llama-model.cpp`, `load_tensors`). Layer `il` goes to the device `upper_bound(cumsum(ts) / sum(ts), il / (layers + 1))`, and the output layer -(`il = layers`, holding `lm_head`) goes the same way, which is the last card. Each -layer's weights are its share of the class totals, minus its experts when it is one of -the first `--n-cpu-moe` layers. Each layer's KV and recurrent state sit on its card. Every -card also carries its own overhead. +(`il = layers`, holding `lm_head` and a GGUF's MTP tensors) goes the same way, which is +the last card. Each layer's weights are its share of the class totals, minus its experts +when it is one of the first `--n-cpu-moe` layers. The vision projector counts on the first +card: `tools/mtmd/clip.cpp` loads it on the first GPU. Each layer's KV and recurrent state +sit on its card. Every card also carries its own overhead. The fit emits `-ts` as whole layer counts summing to layers + 1, so each card gets exactly the layers it was sized for. It keeps every card within the smallest budget of the cards used, because the device plugin, not the fit, decides which card is device 0 inside the diff --git a/tests/fit_test.py b/tests/fit_test.py index b9845281..c1574b78 100755 --- a/tests/fit_test.py +++ b/tests/fit_test.py @@ -653,7 +653,9 @@ def llama_cards(d, cfg, w, kv_layer): count): layer il on device upper_bound(cumsum(ts) / sum(ts), il / (n_layer + 1)), the output layer (il = n_layer) the same way, the experts of the first --n-cpu-moe layers in host RAM, each layer's KV on its device, and each card's own overhead. Weights per layer - are the class totals over the layers. kv_layer(il, ctx) is one sequence's KV of layer il.""" + are the class totals over the layers; the output layer holds lm_head and the MTP tensors + (a GGUF's), and the vision projector is on the first GPU (tools/mtmd/clip.cpp). + kv_layer(il, ctx) is one sequence's KV of layer il.""" tc = cfg.get("text_config") or cfg n = tc["num_hidden_layers"] f = d["flags"] @@ -665,10 +667,11 @@ def llama_cards(d, cfg, w, kv_layer): c += t cum.append(c / sum(ts)) cards = [0] * len(ts) + cards[0] += cls.get("vision", 0) for il in range(n + 1): dev = bisect.bisect_right(cum, il / (n + 1)) if il == n: - cards[dev] += cls.get("lm_head", 0) + cls.get("mtp", 0) + cls.get("vision", 0) + cards[dev] += cls.get("lm_head", 0) + (cls.get("mtp", 0) if w["format"] == "gguf" else 0) else: cards[dev] += (cls.get("other", 0) / n + (cls.get("experts", 0) / n if il >= ncpu else 0) + kv_layer(il, d["ctx"]) * d["seqs"]) @@ -737,6 +740,51 @@ def test_gemma_layer_split_cards_within_budget(self): self.assertEqual((d["placement"], d["gpus"]), ("gpu-split", 2)) self.check(d, config(g), weights(g), gemma4_kv) + def test_layer_bytes_are_the_bytes_to_serve(self): + # Sized from safetensors for a GGUF conversion: the MTP head is not served + # (model_profile serve_bytes), and the vision projector loads on the first GPU + # (tools/mtmd/clip.cpp takes the first GPU backend), not on the output layer's card. + cfg = config("Qwen__Qwen3.8-Flash-Next") + m = fit.model_profile(cfg, weights("Qwen__Qwen3.8-Flash-Next")) + self.assertGreater(m["bytes"]["mtp"], GIB) + self.assertGreater(m["bytes"]["vision"], GIB / 2) + wl, _ = fit.llama_layer_bytes(m, fit.kv_profile(cfg, 2), 4096, 1, 0) + self.assertAlmostEqual(sum(wl), m["serve_bytes"] - m["lookup_bytes"], delta=1) + self.assertAlmostEqual(wl[-1], m["bytes"]["lm_head"], delta=1) + self.assertAlmostEqual(wl[0] - wl[1], m["bytes"]["vision"], delta=1) + + def test_moe_offload_takes_the_best_number_of_cards(self): + # Every card is held to the smallest budget of the cards used, so two smaller cards + # beside two large ones can leave less room than the two large ones alone. The fit + # tries 1, 2, ... of the free cards by budget and keeps the plan with the fewest + # CPU-expert layers, not every free card. + h = host("xeon6741p-4xb70") + for g, used in zip(h["gpus"], (0, 0, 12, 14)): + g["vram_used"], g["users"] = used * GIB, [] + model = mdir("Intel__Qwen3.8-Flash-Next-W4A16-AutoRound") + cfg = config("Intel__Qwen3.8-Flash-Next-W4A16-AutoRound") + n = (cfg.get("text_config") or cfg)["num_hidden_layers"] + + def ncpu(d): + f = d["flags"] + return int(f[f.index("--n-cpu-moe") + 1]) if "--n-cpu-moe" in f else n + + kw = dict(ctx=262144, seqs=6, device="gpu") + auto = fit.plan_with_variants(h, model, [], **kw) + self.assertTrue(auto["fits"], auto["summary"]) + self.assertEqual(auto["placement"], "moe-offload") + each = [fit.plan_with_variants(h, model, [], gpus=k, **kw) for k in range(1, 5)] + self.assertEqual(ncpu(auto), min(ncpu(d) for d in each if d["fits"])) + self.assertLess(auto["gpus"], 4) + budget = min(c["budget"] for c in fit.gpu_budget(h)[0][:auto["gpus"]]) + self.assertLessEqual(max(auto["card_bytes"]), budget) + + def test_moe_offload_takes_the_cards_asked_for(self): + # --gpus 3 on the Xeon, where two cards are free: refused, not two cards. + d = plan("xeon6741p-4xb70", FLASH_GGUF, gpus=3, device="gpu") + self.assertFalse(d["fits"]) + self.assertIn("3 GPUs asked, 2 not busy", d["reason"]) + def test_too_much_for_any_split_is_refused(self): d = plan("xeon6741p-4xb70", "google__gemma-4-26B-A4B-it-qat-q4_0-gguf", ctx=262144, seqs=12, device="gpu") self.assertFalse(d["fits"])