diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54bacd6d..ebc42427 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -463,3 +463,23 @@ jobs: sudo tar -xzf kubeconform.tar.gz -C /usr/local/bin kubeconform sudo apt-get update && sudo apt-get install -y --no-install-recommends python3-yaml - run: tests/kserve_test.sh + + # --- Freshness: known issues --------------------------------------------------- + # pins/known-issues.toml lists the upstream bugs, behaviour changes and advisories + # k3d-gpu depends on, each with the release that fixes it. Offline: fails when an + # "ours" pin is inside a fixed issue's affected range, when a workaround marker + # outlives its fix, or when markers and entries disagree. `hack/known-issues poll` + # (network) finds fixed-in releases; hack/ledger-freshness lists SUPPORTED ledger + # rows measured on versions older than today's pins. + known-issues: + name: Known issues (fixed-in rules, markers), version schemes, ledger re-measure queue + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - run: python3 tests/pinver_test.py + - run: python3 tests/known_issues_test.py + - run: python3 tests/ledger_freshness_test.py + - run: hack/known-issues check diff --git a/console/audit.py b/console/audit.py index d047da08..3232aa2f 100644 --- a/console/audit.py +++ b/console/audit.py @@ -84,6 +84,7 @@ def endpoint(address, port): return "%s:%s" % (host, port if port is not None else "(port not set)") +# known-issue: eat-9 (the rule is general; EAT#9 is the instance that motivated it) def check_listener(ls): subject = "%s %s" % (endpoint(ls["address"], ls["port"]), ls["name"]) where = "loopback" if is_loopback(ls["address"]) else "off loopback" diff --git a/console/model_opt.py b/console/model_opt.py index b6903bfa..0776fe08 100644 --- a/console/model_opt.py +++ b/console/model_opt.py @@ -1104,6 +1104,7 @@ def spec_decide(doc, choose, engine, device, pick, drafters, target, facts, draf for d in fit: if d["drafter"] == "assistant": if facts and facts["moe"]: + # known-issue: vllm-pr-51493 (the VLLM_USE_V2_MODEL_RUNNER=1 clause goes once a release has the fix) doc["notes"].append("MoE target: Google says the 26B A4B drafter may not speed up at batch 1 without good parallelism (https://ai.google.dev/gemma/docs/mtp/overview); " "if vLLM's V1 runner fails on it, VLLM_USE_V2_MODEL_RUNNER=1 (https://github.com/vllm-project/vllm/pull/51493); K3D_GPU_AGENT_DRAFT=off turns the drafter off") cfg = {"method": "mtp", "model": d["id"], "num_speculative_tokens": 4} diff --git a/docs/first-run.md b/docs/first-run.md index 84e0560c..3fb78fca 100644 --- a/docs/first-run.md +++ b/docs/first-run.md @@ -72,7 +72,7 @@ What differs on WSL2: agent does not request the WSL2 GPU yet. quickstart says so at step 1. - The NPU is not available under WSL2 ([intel/linux-npu-driver#56](https://github.com/intel/linux-npu-driver/issues/56)), - so `recipe npu-embeddings` cannot run here. + so `recipe npu-embeddings` cannot run here. - WSL2 hides the P/E-core split; no P-core pinning applies. - The WSL2 VM gets half the Windows RAM by default. A 32 GB laptop shows about 15.4 GiB and gets the 4B tier. Raise `memory=` in `%UserProfile%\.wslconfig` diff --git a/docs/platforms.md b/docs/platforms.md index c95e972f..ed23ae98 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -376,15 +376,20 @@ B70 runs land: - Expected collision: a second GPU plugin. `deploy-intel-gpu-plugin.yml` applies `deployments/gpu_plugin/base/intel-gpu-plugin.yaml` from intel-device-plugins `main` (`intel_gpu_plugin_ref` defaults to `main`, unpinned) beside the baked DaemonSet. + `eat-up.sh` therefore stops on `compute_platform=xpu` and points at + `K3D_GPU_AGENT_BACKEND=vllm`. - Expected collision: a second balloons policy. The reworked `nri_cpu_balloons` role renders one NUMA-sized `BalloonsPolicy` per node (`node.`). #21 still installs it only on `deploy_nri_balloon_policy == "yes"` (`fresh-install.sh` line 247), so - `eat-up.sh`'s `off` keeps it away; a config that turns it on meets k3d-gpu's policy. + `eat-up.sh`'s `off` keeps it away, and `eat-up.sh` stops on any other value, an + absent key included (`prompt_for_input`, which the stack path and `brownfield.sh model` + both call, turns that on). - Expected gap: the `hostPath` libraries resolve inside the k3d node container, not on the host. The node image would have to carry compatible Level Zero and OpenCL user space at those paths, or the pod would mount empty or mismatched libraries. - Expected gap: Kata (`kata-fc`, `kata-qemu`) needs `/dev/kvm` and nested virtualization inside the k3d node, which k3d-gpu does not pass through today. + `eat-up.sh` stops on `deploy_kata_containers=on`. Gaps for the #20 example instance (`docs/examples/openclaw-instance.yaml`) on k3d-gpu: diff --git a/docs/quickstart.md b/docs/quickstart.md index 4b2d22e3..16869603 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -152,7 +152,7 @@ the LLM server does not request it yet. ([Docker docs](https://docs.docker.com/desktop/features/gpu/)). The Intel path does not use that flag. - **The Intel NPU does not work under WSL2.** Intel's Linux NPU driver has no - WSL2 support ([intel/linux-npu-driver#56](https://github.com/intel/linux-npu-driver/issues/56)). + WSL2 support ([intel/linux-npu-driver#56](https://github.com/intel/linux-npu-driver/issues/56)). ## Remove everything diff --git a/examples/kserve/install.sh b/examples/kserve/install.sh index 34101868..9ecab427 100755 --- a/examples/kserve/install.sh +++ b/examples/kserve/install.sh @@ -44,7 +44,9 @@ helm upgrade --install kserve-resources oci://ghcr.io/kserve/charts/kserve-resou --version "${KSERVE_VERSION}" --namespace kserve \ --values "${HERE}/kserve-values.yaml" --wait --timeout 10m +# known-issue-begin: kserve-4592 kubectl apply --server-side -f "${HERE}/openvino-runtime.yaml" +# known-issue-end: kserve-4592 sed -e "s/supplementalGroups: \[\]/supplementalGroups: [${render_gid}]/" \ -e "s|gpu.intel.com/i915:|gpu.intel.com/${driver}:|" "${HERE}/qwen3-isvc.yaml" \ diff --git a/examples/kserve/openvino-runtime.yaml b/examples/kserve/openvino-runtime.yaml index cd50295d..498be28b 100644 --- a/examples/kserve/openvino-runtime.yaml +++ b/examples/kserve/openvino-runtime.yaml @@ -4,6 +4,7 @@ # release charts, so it ships here. Changes from upstream: the Helm ownership # annotation and label are dropped (Helm does not install it here), and the # placeholder image openvino/model_server:replace is pinned to the GPU build. +# known-issue: kserve-4592 (this file goes once a KServe release's runtimes chart ships kserve-openvino) apiVersion: serving.kserve.io/v1alpha1 kind: ClusterServingRuntime metadata: diff --git a/hack/known-issues b/hack/known-issues new file mode 100755 index 00000000..bc496190 --- /dev/null +++ b/hack/known-issues @@ -0,0 +1,786 @@ +#!/usr/bin/env python3 +"""known-issues: the upstream bugs, changes and advisories k3d-gpu depends on, +with the release that fixes each (pins/known-issues.toml). + + hack/known-issues [--root DIR] check [--json] + Offline and deterministic (no clock, no network). Fails when: + - the registry breaks its schema (pins/known-issues.toml header); + - a `known-issue: ` / `known-issue-begin|end: ` marker names no + entry, sits in a file its entry's `workarounds` does not list, sits in a + ledger or historical file, or a begin has no end; or a listed workaround + file has no marker; + - a site regex matches nothing, or a matched version does not parse; + - an "ours" site's version is inside `affected` while `fixed_in` is set + ("bump to >= "); + - workaround markers remain while every "ours" site is at or past + `fixed_in` and `retained` is empty. + Reports, without failing: "eat-stock" sites in range, "ours" sites in range + of an unfixed issue, and ledger_refs in range. --json prints the evaluation. + + hack/known-issues [--root DIR] poll [--json] [--id ID]... + Network, read-only (gh api with GITHUB_TOKEN; writes no file). For each + entry: finds the fix commit (fix.sha, the pull request's merge commit, or + the commit that closed the issue, from its timeline), walks the fix repo's + stable releases in version order from the lowest site version, and takes + the first whose compare/... reads "behind" or "identical". A + fix of the form {repo, path, contains} takes the first release whose file + contains the string; {ecosystem, package} reads the GitHub advisory + database for the lowest site version and takes the first stable patched + version above it outside every advisory's range, reading each + candidate's own advisories too until one has none. With an `image` + template the image tag of the fixing release must resolve + (scripts/images-lock digest). Prints + {id, state, fixed_in, tag, release_url, published_at, recorded, mismatch} + per entry; exit 1 when an entry could not be read. + +Site and marker classes come from pins/classes (` ...`, first match +wins) when it exists, then from the defaults below: EAT's stock paths are +"eat-stock", bench/ is "ledger", oracle/seed/ and console/matrix.v1.json are +"historical", everything else is "ours". Sites and markers may only be in "ours" +or "eat-stock" files, never under bench/ or oracle/seed/ or in EXPERIMENTS.md. + +Python standard library only (tomllib: Python 3.11+). +""" + +import argparse +import base64 +import fnmatch +import functools +import importlib.machinery +import importlib.util +import json +import os +import pathlib +import re +import subprocess +import sys +import tomllib +import urllib.parse + +HERE = pathlib.Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import pinver # noqa: E402 + +REGISTRY = "pins/known-issues.toml" +CLASSES = "pins/classes" +MARKER_RE = re.compile(r"known-issue(-begin|-end)?:[ \t]*([a-z0-9][a-z0-9.-]*[a-z0-9])") +ID_RE = re.compile(r"^[a-z0-9][a-z0-9.-]*[a-z0-9]$") +UPSTREAM_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+#[1-9][0-9]*$") +REPO_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") +ROW_RE = re.compile(r"^H\d+[a-z]{0,2}$") +GHSA_RE = re.compile(r"^GHSA(-[23456789cfghjmpqrvwx]{4}){3}$") +KINDS = ("bug", "change", "advisory") +ADOPT = ("review", "auto") + +# The registry, the tools and their tests hold marker text as data. +SELF = {REGISTRY, "hack/known-issues", "hack/ledger-freshness", "hack/pinver.py", + "tests/known_issues_test.py", "tests/ledger_freshness_test.py", "tests/pinver_test.py"} +DEFAULT_CLASSES = [ + ("integrations/eat/*", "eat-stock"), ("examples/kuberay/*", "eat-stock"), + ("examples/agents/litellm-gateway/*", "eat-stock"), ("contrib/eat/*", "eat-stock"), + ("bench/*", "ledger"), ("oracle/seed/*", "historical"), ("console/matrix.v1.json", "historical"), +] +NEVER = ("bench/", "oracle/seed/") +NEVER_FILES = ("EXPERIMENTS.md", "console/matrix.v1.json") + +REQUIRED = {"id": str, "title": str, "kind": str, "upstream": str, "component": str, "scheme": str, + "affected": str, "fixed_in": str, "adopt": str, "sites": list, "workarounds": list, "notes": str} +OPTIONAL = {"fix": dict, "retained": dict, "ledger_refs": list, "advisories": list, "image": str, "test": str} + + +class ApiError(Exception): + def __init__(self, status, detail): + super().__init__("HTTP %s: %s" % (status, detail)) + self.status = status + + +# --- loading ------------------------------------------------------------------- + +def load(root, registry=None): + """The [[issue]] entries of the registry, in file order.""" + path = pathlib.Path(registry) if registry else pathlib.Path(root) / REGISTRY + with open(path, "rb") as f: + data = tomllib.load(f) + extra = sorted(set(data) - {"issue"}) + if extra: + raise ValueError("%s: unknown top-level key(s) %s" % (REGISTRY, ", ".join(extra))) + issues = data.get("issue", []) + if not isinstance(issues, list): + raise ValueError("%s: issue must be an array of tables ([[issue]])" % REGISTRY) + return issues + + +def load_classes(root): + """[(glob, class)]: pins/classes lines first (first match wins), then the defaults.""" + out = [] + p = pathlib.Path(root) / CLASSES + if p.is_file(): + for line in p.read_text().splitlines(): + parts = line.split() + if not parts or parts[0].startswith("#") or parts[0] == "historical-ref" or len(parts) < 2: + continue + out.append((parts[0], parts[1])) + return out + DEFAULT_CLASSES + + +def path_class(classes, rel): + for glob, cls in classes: + if fnmatch.fnmatchcase(rel, glob): + return cls + return "ours" + + +def forbidden(classes, rel): + """Why a site or marker may not live in rel, or None.""" + if rel.startswith(NEVER) or rel in NEVER_FILES: + return "a ledger or historical file" + cls = path_class(classes, rel) + if cls not in ("ours", "eat-stock"): + return "a %s file" % cls + return None + + +def ledger_rows(root): + """{row: verdict} of the target tree's EXPERIMENTS.md via tests/ledger_drift_check.py, or None.""" + if not (pathlib.Path(root) / "EXPERIMENTS.md").is_file(): + return None + src = HERE.parent / "tests" / "ledger_drift_check.py" + loader = importlib.machinery.SourceFileLoader("ledger_drift_check", str(src)) + spec = importlib.util.spec_from_loader("ledger_drift_check", loader) + mod = importlib.util.module_from_spec(spec) + loader.exec_module(mod) + return mod.ledger(pathlib.Path(root)) + + +def list_files(root): + """Repo-relative paths: git's tracked and unignored files when root is a work tree, else a walk.""" + root = pathlib.Path(root) + try: + top = subprocess.run(["git", "-C", str(root), "rev-parse", "--show-toplevel"], + capture_output=True, text=True, check=True).stdout.strip() + if pathlib.Path(top).resolve() == root.resolve(): + out = subprocess.run(["git", "-C", str(root), "ls-files", "-z", "--cached", "--others", + "--exclude-standard"], capture_output=True, check=True).stdout + return sorted({p for p in out.decode().split("\0") if p and (root / p).is_file()}) + except (OSError, subprocess.CalledProcessError): + pass + found = [] + for d, dirs, files in os.walk(root): + dirs[:] = sorted(x for x in dirs if x != ".git") + found += [os.path.relpath(os.path.join(d, f), root) for f in files] + return sorted(found) + + +# --- sites and markers --------------------------------------------------------- + +def matches(root, rel, regex, scheme): + """(versions, error) of a {file, regex} pair.""" + p = pathlib.Path(root) / rel + if not p.is_file(): + return None, "%s: no such file" % rel + try: + rx = re.compile(regex, re.M) + except re.error as e: + return None, "regex %r does not compile (%s)" % (regex, e) + if "v" not in rx.groupindex: + return None, "regex %r has no named group v" % regex + found = [] + for m in rx.finditer(p.read_text(errors="replace")): + if m.group("v") not in found: + found.append(m.group("v")) + if not found: + return None, "regex %r matches nothing in %s" % (regex, rel) + for v in found: + try: + pinver.parse(scheme, v) + except ValueError as e: + return None, "%s: %s" % (rel, e) + return found, None + + +def scan_markers(root, files): + out = [] + for rel in files: + if rel in SELF: + continue + try: + data = (pathlib.Path(root) / rel).read_bytes() + except OSError: + continue + if b"known-issue" not in data: + continue + for n, line in enumerate(data.decode("utf-8", errors="replace").splitlines(), 1): + for m in MARKER_RE.finditer(line): + out.append({"file": rel, "line": n, "kind": (m.group(1) or "-marker")[1:], "id": m.group(2)}) + return out + + +# --- validation ---------------------------------------------------------------- + +def _str_list(v): + return isinstance(v, list) and all(isinstance(x, str) and x for x in v) + + +def validate_fix(fix, kind): + if not fix: + return [] + keys = set(fix) + errs = [] + if "repo" in keys and keys <= {"repo", "sha", "pr"} and keys & {"sha", "pr"}: + if not (isinstance(fix["repo"], str) and REPO_RE.match(fix["repo"])): + errs.append("fix.repo must be owner/name") + if "sha" in fix and not (isinstance(fix["sha"], str) and re.match(r"^[0-9a-f]{40}$", fix["sha"])): + errs.append("fix.sha must be a full 40-hex commit") + if "pr" in fix and not (isinstance(fix["pr"], int) and not isinstance(fix["pr"], bool) and fix["pr"] > 0): + errs.append("fix.pr must be a positive integer") + elif keys == {"repo", "path", "contains"}: + if not (isinstance(fix["repo"], str) and REPO_RE.match(fix["repo"])): + errs.append("fix.repo must be owner/name") + if not all(isinstance(fix[k], str) and fix[k] for k in ("path", "contains")): + errs.append("fix.path and fix.contains must be non-empty strings") + elif keys == {"ecosystem", "package"}: + if kind != "advisory": + errs.append("fix {ecosystem, package} is for kind advisory") + if not all(isinstance(fix[k], str) and fix[k] for k in keys): + errs.append("fix.ecosystem and fix.package must be non-empty strings") + else: + errs.append("fix must be {}, {repo, sha and/or pr}, {repo, path, contains} or {ecosystem, package}") + return errs + + +def validate(root, issues, classes, rows): + """Schema errors, as 'pins/known-issues.toml: : ' lines; and the ids that passed.""" + errors, ok, seen = [], set(), set() + for i, e in enumerate(issues): + ident = e.get("id") if isinstance(e, dict) and isinstance(e.get("id"), str) else "issue %d" % (i + 1) + errs = [] + + def err(msg, errs=errs): + errs.append(msg) + + if not isinstance(e, dict): + errors.append("%s: %s: not a table" % (REGISTRY, ident)) + continue + for k in sorted(set(e) - set(REQUIRED) - set(OPTIONAL)): + err("unknown field %s" % k) + for k, t in REQUIRED.items(): + if k not in e: + err("missing field %s" % k) + elif not isinstance(e[k], t): + err("%s must be a %s" % (k, t.__name__)) + for k, t in OPTIONAL.items(): + if k in e and not isinstance(e[k], t): + err("%s must be a %s" % (k, t.__name__)) + if errs: + errors += ["%s: %s: %s" % (REGISTRY, ident, m) for m in errs] + continue + if not ID_RE.match(e["id"]): + err("id must be lowercase letters, digits, '.' and '-'") + if e["id"] in seen: + err("duplicate id %s" % e["id"]) + seen.add(e["id"]) + if e["kind"] not in KINDS: + err("kind %r is not one of %s" % (e["kind"], ", ".join(KINDS))) + if e["adopt"] not in ADOPT: + err("adopt %r is not one of %s" % (e["adopt"], ", ".join(ADOPT))) + if e["upstream"] and not UPSTREAM_RE.match(e["upstream"]): + err("upstream %r is not owner/repo#n" % e["upstream"]) + if not e["title"].strip(): + err("title is empty") + errs += validate_fix(e.get("fix", {}), e["kind"]) + scheme = e["scheme"] + if scheme not in pinver.SCHEMES: + err("scheme %r is not one of %s" % (scheme, ", ".join(pinver.SCHEMES))) + errors += ["%s: %s: %s" % (REGISTRY, ident, m) for m in errs] + continue + try: + pinver.check_range(scheme, e["affected"]) + except ValueError as x: + err("affected: %s" % x) + if e["fixed_in"]: + try: + if pinver.is_prerelease(scheme, e["fixed_in"]): + err("fixed_in %s is a prerelease" % e["fixed_in"]) + if pinver.in_range(scheme, e["fixed_in"], e["affected"]): + err("fixed_in %s is inside affected %s; narrow affected" % (e["fixed_in"], e["affected"])) + except ValueError as x: + err("fixed_in: %s" % x) + for s in e["sites"]: + if not (isinstance(s, dict) and set(s) == {"file", "regex"} and all(isinstance(s[k], str) for k in s)): + err("each site is {file, regex}") + continue + why = forbidden(classes, s["file"]) + if why: + err("site %s is in %s; sites are ours or eat-stock files" % (s["file"], why)) + _, problem = matches(root, s["file"], s["regex"], scheme) + if problem: + err("site %s" % problem) + if not _str_list(e["workarounds"]) or len(set(e["workarounds"])) != len(e["workarounds"]): + err("workarounds must be a list of distinct file paths") + else: + for w in e["workarounds"]: + why = forbidden(classes, w) + if why: + err("workaround %s is in %s" % (w, why)) + elif not (pathlib.Path(root) / w).is_file(): + err("workaround %s: no such file" % w) + retained = e.get("retained", {}) + if retained and not (set(retained) == {"reason", "pr"} and isinstance(retained["reason"], str) + and retained["reason"].strip() and isinstance(retained["pr"], (str, int))): + err("retained must be {} or {reason, pr} with a reason") + for r in e.get("ledger_refs", []): + if not (isinstance(r, dict) and {"row", "path"} <= set(r) <= {"row", "path", "regex"} + and all(isinstance(r[k], str) for k in r)): + err("each ledger_ref is {row, path, regex?}") + continue + if not ROW_RE.match(r["row"]): + err("ledger_ref row %r is not a ledger id" % r["row"]) + elif rows is not None and r["row"] not in rows: + err("ledger_ref row %s is not in the EXPERIMENTS.md hypothesis table" % r["row"]) + if not (pathlib.Path(root) / r["path"]).is_file(): + err("ledger_ref %s: no such file" % r["path"]) + elif "regex" in r: + _, problem = matches(root, r["path"], r["regex"], scheme) + if problem: + err("ledger_ref %s" % problem) + if "advisories" in e: + if e["kind"] != "advisory": + err("advisories is for kind advisory") + elif not _str_list(e["advisories"]) or not all(GHSA_RE.match(a) for a in e["advisories"]): + err("advisories must be GHSA ids") + if "image" in e and "{tag}" not in e["image"] and "{version}" not in e["image"]: + err("image must hold {tag} or {version}") + test = e.get("test", "") + if test and not test.startswith("canary:") and not (pathlib.Path(root) / test).is_file(): + err("test %s: no such file" % test) + errors += ["%s: %s: %s" % (REGISTRY, ident, m) for m in errs] + if not errs: + ok.add(e["id"]) + return errors, ok + + +def check_markers(issues, markers, classes): + errors = [] + by_id = {e["id"]: e for e in issues if isinstance(e, dict) and isinstance(e.get("id"), str)} + open_blocks = {} + for m in markers: + where = "%s:%d" % (m["file"], m["line"]) + tag = "known-issue%s: %s" % ("" if m["kind"] == "marker" else "-" + m["kind"], m["id"]) + why = forbidden(classes, m["file"]) + if why: + errors.append("%s: %s is in %s; markers go in ours or eat-stock files only" % (where, tag, why)) + e = by_id.get(m["id"]) + if e is None: + errors.append("%s: %s names no entry in %s" % (where, tag, REGISTRY)) + elif m["file"] not in e.get("workarounds", []): + errors.append("%s: %s, but %s's workarounds do not list %s" % (where, tag, m["id"], m["file"])) + key = (m["file"], m["id"]) + if m["kind"] == "begin": + open_blocks.setdefault(key, []).append(m["line"]) + elif m["kind"] == "end": + if not open_blocks.get(key): + errors.append("%s: %s has no known-issue-begin: %s before it" % (where, tag, m["id"])) + else: + open_blocks[key].pop() + for (f, i), lines in sorted(open_blocks.items()): + for n in lines: + errors.append("%s:%d: known-issue-begin: %s has no known-issue-end: %s" % (f, n, i, i)) + have = {(m["file"], m["id"]) for m in markers} + for e in by_id.values(): + for w in e.get("workarounds", []) if isinstance(e.get("workarounds"), list) else []: + if isinstance(w, str) and (w, e["id"]) not in have: + errors.append("%s: %s: %s has no known-issue marker for %s" % (REGISTRY, e["id"], w, e["id"])) + return errors + + +# --- check --------------------------------------------------------------------- + +def at_or_past(scheme, v, fixed): + try: + return pinver.in_range(scheme, v, ">=" + fixed) + except ValueError: + return False + + +def evaluate(root, e, classes, markers): + """(record, failures, reports) for one valid entry.""" + scheme, aff, fixed, ident = e["scheme"], e["affected"], e["fixed_in"], e["id"] + failures, reports, sites = [], [], [] + tail = ", fixed in %s" % fixed if fixed else ", no fixed release known" + for s in e["sites"]: + versions, _ = matches(root, s["file"], s["regex"], scheme) + cls = path_class(classes, s["file"]) + inside = [v for v in versions if pinver.in_range(scheme, v, aff)] + sites.append({"file": s["file"], "class": cls, "versions": versions, "in_affected": bool(inside)}) + for v in inside: + what = "%s %s is inside %s's affected range %s%s" % (e["component"], v, ident, aff, tail) + if cls == "ours" and fixed: + failures.append("%s: %s; bump %s to >= %s" % (s["file"], what, s["file"], fixed)) + else: + reports.append("%s %s: %s" % (cls, s["file"], what)) + refs = [] + for r in e.get("ledger_refs", []): + versions = matches(root, r["path"], r["regex"], scheme)[0] if "regex" in r else [] + inside = [v for v in versions if pinver.in_range(scheme, v, aff)] + refs.append({"row": r["row"], "path": r["path"], "versions": versions, "in_affected": bool(inside)}) + for v in inside: + reports.append("ledger %s %s: %s %s is inside %s's affected range %s%s (informational; the ledger " + "changes only by amendment)" % (r["row"], r["path"], e["component"], v, ident, aff, tail)) + mine = [m for m in markers if m["id"] == ident] + governing = [s for s in sites if s["class"] == "ours"] or sites + past = bool(fixed) and bool(governing) and all( + at_or_past(scheme, v, fixed) for s in governing for v in s["versions"]) + if any(s["class"] == "ours" and s["in_affected"] for s in sites) and fixed: + status = "needs-bump" + elif not fixed: + status = "open" + elif past and mine and not e.get("retained"): + status = "stale-workaround" + files = sorted({m["file"] for m in mine}) + failures.append("%s: workaround markers remain in %s though every ours site is at or past %s; remove " + "them (known-issue-begin/end blocks) or set retained" % (ident, ", ".join(files), fixed)) + elif mine and e.get("retained"): + status = "retained" + else: + status = "fixed" + record = {"id": ident, "kind": e["kind"], "component": e["component"], "scheme": scheme, "affected": aff, + "fixed_in": fixed, "upstream": e["upstream"], "adopt": e["adopt"], "status": status, "sites": sites, + "ledger_refs": refs, "workarounds": e["workarounds"], + "markers": [{k: m[k] for k in ("file", "line", "kind")} for m in mine]} + return record, failures, reports + + +def check(root, registry=None): + root = pathlib.Path(root) + try: + issues = load(root, registry) + except (OSError, ValueError, tomllib.TOMLDecodeError) as x: + return {"ok": False, "errors": ["%s: %s" % (REGISTRY, x)], "reports": [], "issues": []} + classes = load_classes(root) + errors, valid = validate(root, issues, classes, ledger_rows(root)) + markers = scan_markers(root, list_files(root)) + errors += check_markers(issues, markers, classes) + reports, records = [], [] + for e in issues: + if isinstance(e, dict) and e.get("id") in valid: + record, failures, rep = evaluate(root, e, classes, markers) + records.append(record) + errors += failures + reports += rep + return {"ok": not errors, "errors": errors, "reports": reports, "issues": records} + + +def print_check(result): + sites = sum(len(i["sites"]) for i in result["issues"]) + marks = sum(len(i["markers"]) for i in result["issues"]) + print("known-issues: %d issue(s), %d site(s), %d marker(s)" % (len(result["issues"]), sites, marks)) + for r in result["reports"]: + print("report %s" % r) + for x in result["errors"]: + print("FAIL %s" % x) + if result["ok"]: + print("known-issues: ok") + else: + print("known-issues: %d problem(s)" % len(result["errors"])) + + +# --- poll ---------------------------------------------------------------------- + +def gh_api(path): + p = subprocess.run(["gh", "api", "-H", "Accept: application/vnd.github+json", path], + capture_output=True, text=True) + if p.returncode != 0: + m = re.search(r"HTTP (\d{3})", p.stderr) + raise ApiError(int(m.group(1)) if m else 0, "%s: %s" % (path, p.stderr.strip()[:300])) + return json.loads(p.stdout) if p.stdout.strip() else None + + +def images_lock_digest(ref): + p = subprocess.run([sys.executable, str(HERE.parent / "scripts" / "images-lock"), "digest", ref], + capture_output=True, text=True) + if p.returncode != 0 or not p.stdout.strip().startswith("sha256:"): + raise ValueError("%s does not resolve: %s" % (ref, (p.stderr or p.stdout).strip()[:300])) + return p.stdout.strip() + + +def norm(tag): + return tag[1:] if re.match(r"^v\d", tag) else tag + + +def site_floor(root, e): + """The lowest site version (a pinver.Version), or None.""" + found = [] + for s in e["sites"]: + versions, _ = matches(root, s["file"], s["regex"], e["scheme"]) + found += [pinver.parse(e["scheme"], v) for v in versions or []] + found = [v for v in found if pinver.comparable(e["scheme"], v, found[0])] if found else [] + return min(found, key=functools.cmp_to_key(lambda a, b: pinver.compare(e["scheme"], a, b))) if found else None + + +def stable_releases(api, repo, scheme, floor): + """[(Version, tag, release)] ascending: published, not prerelease, parseable, of floor's flavour.""" + out = [] + for page in range(1, 11): + batch = api("repos/%s/releases?per_page=100&page=%d" % (repo, page)) or [] + for r in batch: + if r.get("draft") or r.get("prerelease"): + continue + try: + v = pinver.parse(scheme, r["tag_name"]) + except ValueError: + continue + if v.pre or (floor is not None and not pinver.comparable(scheme, v, floor)): + continue + out.append((v, r["tag_name"], r)) + if len(batch) < 100: + break + if floor is None and out: + out = [t for t in out if pinver.comparable(scheme, t[0], out[0][0])] + out.sort(key=functools.cmp_to_key(lambda a, b: pinver.compare(scheme, a[0], b[0]))) + return out + + +def first_containing(tags, floor, scheme, contains, budget=60): + """Index of the first release that contains the fix, walking from the floor's release, or None.""" + if not tags: + return None + start = 0 + if floor is not None: + start = next((i for i, t in enumerate(tags) if pinver.compare(scheme, t[0], floor) >= 0), len(tags) - 1) + calls = [0] + + def has(i): + calls[0] += 1 + if calls[0] > budget: + raise ValueError("more than %d release checks; narrow the sites or the fix" % budget) + return contains(tags[i][1]) + + if has(start): + i = start + while i > 0 and has(i - 1): + i -= 1 + return i + for i in range(start + 1, len(tags)): + if has(i): + return i + return None + + +def pr_commit(api, repo, n): + pr = api("repos/%s/pulls/%s" % (repo, n)) or {} + if pr.get("merged") or pr.get("merged_at"): + return "merged", repo, pr.get("merge_commit_sha") + return ("open" if pr.get("state") == "open" else "closed-unmerged"), None, None + + +def fix_commit(api, e): + """(state, repo, sha or None) of an entry's fix.""" + fix = e.get("fix") or {} + if fix.get("sha"): + return "merged", fix["repo"], fix["sha"] + if fix.get("pr"): + return pr_commit(api, fix["repo"], fix["pr"]) + if not e["upstream"]: + return "untracked", None, None + repo, n = e["upstream"].split("#") + issue = api("repos/%s/issues/%s" % (repo, n)) or {} + if issue.get("pull_request"): + return pr_commit(api, repo, n) + if issue.get("state") == "open": + return "open", None, None + if issue.get("state_reason") == "not_planned": + return "closed-not-planned", None, None + closing = None + for page in range(1, 11): + events = api("repos/%s/issues/%s/timeline?per_page=100&page=%d" % (repo, n, page)) or [] + for ev in events: + if ev.get("event") == "closed" and ev.get("commit_id"): + m = re.search(r"/repos/([^/]+/[^/]+)/commits/", ev.get("commit_url") or "") + closing = (m.group(1) if m else repo, ev["commit_id"]) + if len(events) < 100: + break + if closing: + return "closed", closing[0], closing[1] + return "closed-no-commit", None, None + + +def gh_range(text): + """A GitHub advisory range ('>= 1.65.5, < 1.85.0') as a pinver spec.""" + parts = [] + for p in text.split(","): + m = re.match(r"^\s*(<=|>=|<|>|=)\s*(\S+)\s*$", p) + if not m: + raise ValueError("advisory range %r" % text) + parts.append(("==" if m.group(1) == "=" else m.group(1)) + m.group(2)) + return ",".join(parts) + + +def poll_advisory(api, root, e, out, rounds=10): + """The first stable patched version above the lowest site version that no advisory affects: + the advisories of the site version, then of each candidate, until a candidate has none.""" + fix, scheme = e["fix"], e["scheme"] + floor = site_floor(root, e) + if floor is None: + raise ValueError("no site version to query the advisory database with") + ranges, patched, affected = {}, set(), set() + + def read(version): + """Add the advisories that affect version; True when there is one.""" + q = urllib.parse.quote("%s@%s" % (fix["package"], version)) + hit = False + for a in api("advisories?ecosystem=%s&affects=%s&per_page=100" % (fix["ecosystem"], q)) or []: + spec = [] + for vul in a.get("vulnerabilities") or []: + pkg = vul.get("package") or {} + if (pkg.get("name") or "").lower() != fix["package"].lower(): + continue + spec.append(gh_range(vul.get("vulnerable_version_range") or "")) + if vul.get("first_patched_version"): + patched.add(vul["first_patched_version"]) + if spec: + ranges.setdefault(a["ghsa_id"], " || ".join(spec)) + hit = True + return hit + + def clear(p): + try: + return (p not in affected and not pinver.is_prerelease(scheme, p) + and pinver.compare(scheme, p, floor) > 0 + and not any(pinver.in_range(scheme, p, r) for r in ranges.values())) + except ValueError: + return False + + read(floor.raw) + for _ in range(rounds): + cands = [p for p in patched if clear(p)] + best = min(cands, key=functools.cmp_to_key(lambda a, b: pinver.compare(scheme, a, b))) if cands else "" + if not best or not read(best): + break + affected.add(best) + else: + raise ValueError("every patched version had advisories of its own after %d reads" % rounds) + out.update(state="advisory", fixed_in=best, advisories=sorted(ranges), + new_advisories=sorted(set(ranges) - set(e.get("advisories", [])))) + if best and fix["ecosystem"] == "pip": + out["release_url"] = "https://pypi.org/project/%s/%s/" % (fix["package"], best) + + +def poll_entry(api, digest, root, e): + out = {"id": e["id"], "state": "", "fixed_in": "", "tag": "", "release_url": "", "published_at": "", + "recorded": e.get("fixed_in", ""), "mismatch": False} + fix = e.get("fix") or {} + try: + if fix.get("ecosystem"): + poll_advisory(api, root, e, out) + else: + if fix.get("path"): + repo = fix["repo"] + + def contains(ref=None): + path = "repos/%s/contents/%s" % (repo, fix["path"]) + ("?ref=%s" % ref if ref else "") + try: + body = api(path) or {} + except ApiError as x: + if x.status == 404: + return False + raise + if body.get("encoding") != "base64": + raise ValueError("%s: no base64 content (file too large for the contents API?)" % path) + return fix["contains"] in base64.b64decode(body["content"]).decode("utf-8", errors="replace") + else: + state, repo, sha = fix_commit(api, e) + if not sha: + out["state"] = state + return finish(out) + + def contains(ref=None): + try: + status = (api("repos/%s/compare/%s...%s" % (repo, ref, sha)) or {}).get("status") + except ApiError as x: + if x.status == 404: + return False + raise + return status in ("behind", "identical") + floor = site_floor(pathlib.Path(root), e) + tags = stable_releases(api, repo, e["scheme"], floor) + i = first_containing(tags, floor, e["scheme"], contains) + if i is None: + out["state"] = "not-shipped" if fix.get("path") and not contains() else "merged-unreleased" + else: + _, tag, rel = tags[i] + out.update(state="released", fixed_in=norm(tag), tag=tag, release_url=rel.get("html_url", ""), + published_at=rel.get("published_at", "")) + if e.get("image"): + out["image"] = e["image"].format(tag=tag, version=norm(tag)) + out["image_digest"] = digest(out["image"]) + except (ApiError, ValueError, KeyError, TypeError) as x: + out["error"] = str(x) + return out + return finish(out, e["scheme"]) + + +def finish(out, scheme=None): + if out["state"] == "untracked" or "error" in out: + return out + a, b = out["fixed_in"], out["recorded"] + same = a == b + if a and b and not same and scheme: + try: + same = pinver.compare(scheme, a, b) == 0 + except ValueError: + pass + out["mismatch"] = not same + return out + + +def poll(issues, root, api=gh_api, digest=images_lock_digest, ids=None): + return [poll_entry(api, digest, root, e) for e in issues + if isinstance(e, dict) and (not ids or e.get("id") in ids)] + + +def print_poll(rows): + for r in rows: + line = "%s: %s" % (r["id"], r.get("error") and "error: " + r["error"] or r["state"]) + if r["fixed_in"] or r["recorded"]: + line += " fixed_in=%s (recorded %s)%s" % (r["fixed_in"] or "-", r["recorded"] or "-", + " MISMATCH" if r["mismatch"] else "") + if r.get("new_advisories"): + line += " new advisories: %s" % ", ".join(r["new_advisories"]) + if r["release_url"]: + line += " %s" % r["release_url"] + print(line) + + +def main(argv): + ap = argparse.ArgumentParser(prog="known-issues", description="Known upstream issues and their fixed-in releases.") + ap.add_argument("--root", default=str(HERE.parent), help="repo root (default: this repo)") + ap.add_argument("--registry", help="registry file (default: ROOT/%s)" % REGISTRY) + sub = ap.add_subparsers(dest="cmd", required=True) + c = sub.add_parser("check", help="offline: schema, markers, sites, fixed-in rules") + c.add_argument("--json", action="store_true") + p = sub.add_parser("poll", help="network, read-only: find each entry's first fixed release") + p.add_argument("--json", action="store_true") + p.add_argument("--id", action="append", default=[], help="poll only this entry (repeatable)") + args = ap.parse_args(argv) + if args.cmd == "check": + result = check(args.root, args.registry) + if args.json: + print(json.dumps(result, indent=1, sort_keys=True)) + else: + print_check(result) + return 0 if result["ok"] else 1 + try: + issues = load(args.root, args.registry) + except (OSError, ValueError, tomllib.TOMLDecodeError) as x: + print("known-issues: %s" % x, file=sys.stderr) + return 2 + rows = poll(issues, pathlib.Path(args.root), ids=set(args.id)) + if args.json: + print(json.dumps(rows, indent=1, sort_keys=True)) + else: + print_poll(rows) + return 1 if any("error" in r for r in rows) else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/hack/ledger-freshness b/hack/ledger-freshness new file mode 100755 index 00000000..11ef66e0 --- /dev/null +++ b/hack/ledger-freshness @@ -0,0 +1,425 @@ +#!/usr/bin/env python3 +"""ledger-freshness: which SUPPORTED ledger rows were measured on component +versions older than today's pins. Read-only: it never writes EXPERIMENTS.md, +console/ledger-rules.json or any other file. + + hack/ledger-freshness [--root DIR] --json | --md + +For each SUPPORTED row of the Hypotheses table (parsed by ledger() from +tests/ledger_drift_check.py), it follows the run anchors of the Probe and +Verdict cells (links to a heading that says run or result; else a heading +" run|result ...") to the run sections, and reads their records: fenced +blocks that open with "# Reproduce" or "# provenance", and "**Reproduce:**" +paragraphs. From those it extracts the measured versions of vllm-openai-cpu/xpu, +openvino/model_server (a version, or a digest that images.lock maps to a tag), +llama.cpp (bNNNNN), rancher/k3s, the cryptoandcoffee/k3d-gpu node image, and +the Hugging Face revisions on the `models:` line. + +Current versions are today's "ours" pins: images.lock entries with an "ours" +source (pins/classes, as hack/known-issues reads it) and scripts/k3d-gpu's +VERSION for the node image. A row is stale when a measured version is a minor +or major behind the current one (hack/pinver.py update types; a llama.cpp build +change counts as a minor); priority-stale when a measured version is inside a +known issue's affected range (pins/known-issues.toml). + +The queue lists rows in console/ledger-rules.json with non-empty classes first +(they drive optimizer defaults), then rows docs claims cite +(tests/claims_check.sh --cited), then the rest; priority-stale rows first within +each tier. A re-measure is a new pre-registered ledger row; the suggested id is +only a free one. Rows whose records the parser cannot read are listed under +"not machine-readable". + +--json: {schema, measured: {row: {component: version}}, current: {component: +version}, queue: [{row, tier, priority, measured, current, reason, known_issues, +suggested_followup_id}], not_machine_readable: [{row, reason}]} +--md: the same as a Markdown section for the freshness dashboard. + +Python standard library only. +""" + +import argparse +import functools +import importlib.machinery +import importlib.util +import json +import pathlib +import re +import subprocess +import sys + +HERE = pathlib.Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import pinver # noqa: E402 + +SCHEMA = "k3d-gpu.ledger-freshness/v1" +VERSION_RE = re.compile(r'^VERSION="(\d+\.\d+\.\d+)"', re.M) +RUN_RE = re.compile(r"\b(runs?|results?)\b", re.I) +FENCE_RE = re.compile(r"^\s*(`{3,}|~{3,})") +HEADING_RE = re.compile(r"^(#{2,6})\s+(.*?)\s*#*\s*$") +ANCHOR_RE = re.compile(r"\]\(#([^)\s]+)\)") +FIELD_RE = re.compile(r"^([a-z][a-z0-9 /()-]*?):\s+(.*)$") +HF_RE = re.compile(r"(?[A-Za-z0-9][\w.-]*/[\w.-]+)\s+@?\s*(?P[0-9a-f]{7,40})\b") +OVMS_DIGEST_RE = re.compile(r"openvino/model_server@sha256:(?P[0-9a-f]{8,64})") +CAL = r"\d{4}\.\d+(?:\.\d+)?" + +# component: (scheme, images.lock names, tag -> version regex, record regexes with group v) +COMPONENTS = { + "vllm-openai-cpu": ("zerover", ("docker.io/vllm/vllm-openai-cpu",), r"^v?(\d+\.\d+\.\d+)$", + (r"vllm/vllm-openai-cpu:v(?P\d+\.\d+\.\d+)",)), + "vllm-openai-xpu": ("zerover", ("docker.io/vllm/vllm-openai-xpu",), r"^v?(\d+\.\d+\.\d+)$", + (r"vllm/vllm-openai-xpu:v(?P\d+\.\d+\.\d+)",)), + "openvino/model_server": ("calver", ("docker.io/openvino/model_server",), r"^(" + CAL + r")", + (r"openvino/model_server:(?P" + CAL + r")", + r"openvino/model_server@sha256:[0-9a-f.…]+`?\s*\((?P" + CAL + r")\)", + r"\bOVMS\s+(?P" + CAL + r")\b")), + "llama.cpp": ("llama", ("ghcr.io/ggml-org/llama.cpp",), r"(?:^|-)(b\d+)$", + (r"llama\.cpp:(?:[a-z]+-)*(?Pb\d{3,})\b", r"\bllama\.cpp\s+(?:server-)?(?Pb\d{3,})\b")), + "rancher/k3s": ("k8s", ("docker.io/rancher/k3s",), r"^(v\d+\.\d+\.\d+[-+]k3s\d+)$", + (r"rancher/k3s:(?Pv\d+\.\d+\.\d+[-+]k3s\d+)", r"\bk3s (?Pv\d+\.\d+\.\d+\+k3s\d+)")), + "cryptoandcoffee/k3d-gpu": ("semver", (), None, (r"cryptoandcoffee/k3d-gpu:(?P\d+\.\d+\.\d+)\b",)), +} + + +def load_path(name, path): + loader = importlib.machinery.SourceFileLoader(name, str(path)) + spec = importlib.util.spec_from_loader(name, loader) + mod = importlib.util.module_from_spec(spec) + loader.exec_module(mod) + return mod + + +drift = load_path("ledger_drift_check", HERE.parent / "tests" / "ledger_drift_check.py") +known = load_path("known_issues", HERE / "known-issues") + + +def row_key(row): + m = re.match(r"^H(\d+)([a-z]*)$", row) + return (int(m.group(1)), m.group(2)) if m else (1 << 30, row) + + +def cmp_key(scheme): + return functools.cmp_to_key(lambda a, b: pinver.compare(scheme, a, b)) + + +# --- the ledger ---------------------------------------------------------------- + +def table_cells(text): + """{row: (probe cell, verdict cell)} of the Hypotheses table.""" + out, inside, header = {}, False, None + for line in text.splitlines(): + if line.startswith("## "): + inside = line.startswith("## Hypotheses") + continue + if not (inside and line.startswith("|")): + continue + cells = [c.strip() for c in re.sub(r"\\\|", "", line).strip().strip("|").split("|")] + if header is None and cells and cells[0] == "#": + header = [c.lower() for c in cells] + continue + m = re.match(r"^(H\d+[a-z]{0,2})$", cells[0]) if cells else None + if m and header: + probe = cells[header.index("probe")] if "probe" in header and header.index("probe") < len(cells) else "" + out[m.group(1)] = (probe, cells[-1]) + return out + + +def slugify(text, seen): + s = re.sub(r"[^\w\- ]", "", text.strip().lower()).replace(" ", "-") + n = seen.get(s, 0) + seen[s] = n + 1 + return s if n == 0 else "%s-%d" % (s, n) + + +def closes(line, fence): + """A closing fence (CommonMark): the opening character, at least as many, nothing after.""" + t = line.strip() + return t.startswith(fence) and not t[len(fence):].strip(fence[0]).strip() + + +def sections(lines): + """[(level, title, slug, first body line, end line)] of the headings outside fences.""" + heads, seen, fence = [], {}, None + for i, line in enumerate(lines): + if fence: + if closes(line, fence): + fence = None + continue + f = FENCE_RE.match(line) + if f: + fence = f.group(1) + continue + h = HEADING_RE.match(line) + if h: + heads.append([len(h.group(1)), h.group(2), slugify(h.group(2), seen), i + 1, len(lines)]) + for j, h in enumerate(heads): + for k in heads[j + 1:]: + if k[0] <= h[0]: + h[4] = k[3] - 1 + break + return heads + + +def records(body): + """(text of the Reproduce/provenance records, models field text, problem) of a section body. + problem: a record block that holds a Markdown heading, the sign of a fence left open above it.""" + texts, models, problem, i = [], [], None, 0 + while i < len(body): + f = FENCE_RE.match(body[i]) + if f: + j = i + 1 + while j < len(body) and not closes(body[j], f.group(1)): + j += 1 + block = body[i + 1:j] + first = next((b.strip() for b in block if b.strip()), "") + if first.startswith(("# Reproduce", "# provenance")): + swallowed = next((b for b in block if HEADING_RE.match(b)), None) + if swallowed: + problem = "a record block runs into the heading %r (a fence left open?)" % swallowed.strip()[:80] + texts.append("\n".join(block)) + field = None + for b in block: + m = FIELD_RE.match(b) + if m: + field = m.group(1) + if field == "models": + models.append(m.group(2)) + elif field == "models" and b.startswith((" ", "\t")): + models.append(b.strip()) + i = j + 1 + continue + if "**Reproduce:**" in body[i]: + j = i + while j < len(body) and body[j].strip() and not FENCE_RE.match(body[j]): + j += 1 + texts.append("\n".join(body[i:j])) + i = j + continue + i += 1 + return "\n".join(texts), "\n".join(models), problem + + +def lock_tags(lock): + """{(name, digest): tag} of the tagged images.lock entries.""" + return {(e["name"], e["digest"]): e["tag"] for e in lock.get("images", []) if e.get("tag")} + + +def extract(text, models, lock): + found = {} + for comp, (scheme, names, _, regexes) in COMPONENTS.items(): + vals = [m.group("v") for rx in regexes for m in re.finditer(rx, text)] + if comp == "openvino/model_server" and not vals: + tags = lock_tags(lock) + for m in OVMS_DIGEST_RE.finditer(text): + for (name, digest), tag in tags.items(): + if name in names and digest.startswith("sha256:" + m.group("d")): + t = re.match(COMPONENTS[comp][2], tag) + if t: + vals.append(t.group(1)) + good = [] + for v in vals: + try: + pinver.parse(scheme, v) + good.append(v) + except ValueError: + pass + if good: + group = [v for v in good if pinver.comparable(scheme, v, good[0])] + found[comp] = min(group, key=cmp_key(scheme)) + for m in HF_RE.finditer(models): + found.setdefault("hf:" + m.group("repo"), m.group("v")) + return found + + +def measured_rows(root, lock): + text = (root / "EXPERIMENTS.md").read_text() + lines = text.splitlines() + verdicts = drift.ledger(root) + cells = table_cells(text) + heads = sections(lines) + by_slug = {h[2]: h for h in heads} + measured, unreadable = {}, [] + for row in sorted((r for r, v in verdicts.items() if v == "SUPPORTED"), key=row_key): + probe, verdict = cells.get(row, ("", "")) + runs = [by_slug[a] for a in ANCHOR_RE.findall(probe + " " + verdict) + if a in by_slug and RUN_RE.search(by_slug[a][1])] + if not runs: + pat = re.compile(r"^%s (run|result)s?\b" % re.escape(row)) + runs = [h for h in heads if pat.match(h[1])] + if not runs: + unreadable.append({"row": row, "reason": "no run section linked from the Probe column"}) + continue + got, saw_record, problem = {}, False, None + for h in runs: + rec, models, bad = records(lines[h[3]:h[4]]) + saw_record, problem = saw_record or bool(rec), problem or bad + for comp, v in extract(rec, models, lock).items(): + got.setdefault(comp, v) + if problem: + unreadable.append({"row": row, "reason": problem}) + elif not saw_record: + unreadable.append({"row": row, "reason": "run section has no '# Reproduce' or '# provenance' block " + "and no **Reproduce:** paragraph"}) + elif not got: + unreadable.append({"row": row, "reason": "records name no component version this parser reads"}) + else: + measured[row] = got + return measured, unreadable + + +# --- current pins -------------------------------------------------------------- + +def current_versions(root, lock): + classes = known.load_classes(root) + tags = lock_tags(lock) + found = {} + for e in lock.get("images", []): + srcs = [s.split()[0] for s in e.get("sources", []) if s.split()] + if not any(known.forbidden(classes, s) is None and known.path_class(classes, s) == "ours" for s in srcs): + continue + tag = e.get("tag") or tags.get((e["name"], e["digest"])) + for comp, (scheme, names, tag_rx, _) in COMPONENTS.items(): + if e["name"] in names and tag and tag_rx: + m = re.search(tag_rx, tag) + if m: + found.setdefault(comp, []).append(m.group(1)) + script = root / "scripts" / "k3d-gpu" + if script.is_file(): + m = VERSION_RE.search(script.read_text(errors="replace")) + if m: + found["cryptoandcoffee/k3d-gpu"] = [m.group(1)] + out = {} + for comp, vals in found.items(): + scheme = COMPONENTS[comp][0] + group = [v for v in vals if pinver.comparable(scheme, v, vals[0])] + out[comp] = max(group, key=cmp_key(scheme)) + return out + + +# --- the queue ----------------------------------------------------------------- + +def cited_rows(root): + p = subprocess.run(["bash", str(HERE.parent / "tests" / "claims_check.sh"), "--cited", str(root)], + capture_output=True, text=True) + if p.returncode != 0: + print("ledger-freshness: claims_check.sh --cited failed; tier 2 is empty: %s" % p.stdout.strip()[-300:], + file=sys.stderr) + return set() + return set(p.stdout.split()) + + +def rule_rows(root): + p = root / "console" / "ledger-rules.json" + if not p.is_file(): + return set() + return {r["id"] for r in json.loads(p.read_text()).get("rows", []) if r.get("classes")} + + +def followup_id(row, taken): + m = re.match(r"^H(\d+)([a-z]*)$", row) + cands = [row + "r"] if len(m.group(2)) < 2 else [] + cands += ["H%sr" % m.group(1)] + ["H%sr%s" % (m.group(1), c) for c in "abcdefghijklmnopqstuvwxyz"] + return next(c for c in cands if c not in taken) + + +def queue(root, measured, current, issues): + rules, cited, taken = rule_rows(root), cited_rows(root), set(drift.ledger(root)) + items = [] + for row, comps in measured.items(): + reasons, prio, ids, m_out, c_out = [], [], [], {}, {} + for comp, mv in comps.items(): + if comp.startswith("hf:"): + continue + scheme, cur = COMPONENTS[comp][0], current.get(comp) + if cur and pinver.comparable(scheme, mv, cur) and pinver.compare(scheme, mv, cur) < 0: + t = pinver.update_type(scheme, mv, cur) + if t in ("minor", "major"): + reasons.append("%s %s -> %s (%s)" % (comp, mv, cur, t)) + m_out[comp], c_out[comp] = mv, cur + for e in issues: + if not isinstance(e, dict) or e.get("component") != comp: + continue + try: + inside = pinver.in_range(e["scheme"], mv, e["affected"]) + except (KeyError, ValueError): + continue + if inside: + since = "" + if e.get("fixed_in"): + since = ", %s in %s" % ("changed" if e.get("kind") == "change" else "fixed", e["fixed_in"]) + prio.append("priority-stale: %s %s is inside %s (%s%s)" % (comp, mv, e["id"], e["affected"], since)) + ids.append(e["id"]) + m_out[comp] = mv + if cur: + c_out[comp] = cur + if not (reasons or prio): + continue + tier = 1 if row in rules else 2 if row in cited else 3 + follow = followup_id(row, taken) + taken.add(follow) + items.append({"row": row, "tier": tier, "priority": bool(prio), "measured": m_out, "current": c_out, + "reason": "; ".join(prio + reasons), "known_issues": sorted(set(ids)), + "suggested_followup_id": follow}) + items.sort(key=lambda q: (q["tier"], not q["priority"], row_key(q["row"]))) + return items + + +def build(root): + root = pathlib.Path(root) + lock_path = root / "images.lock" + lock = json.loads(lock_path.read_text()) if lock_path.is_file() else {"images": []} + reg = root / known.REGISTRY + issues = known.load(root) if reg.is_file() else [] + measured, unreadable = measured_rows(root, lock) + current = current_versions(root, lock) + return {"schema": SCHEMA, "measured": measured, "current": current, + "queue": queue(root, measured, current, issues), "not_machine_readable": unreadable} + + +TIERS = {1: "ledger-rules (drives optimizer defaults)", 2: "cited by a docs claim", 3: "other"} + + +def cell(text): + return str(text).replace("|", "\\|") + + +def markdown(data): + out = ["## Ledger re-measure queue", "", + "Read-only. SUPPORTED rows of EXPERIMENTS.md whose recorded versions are a minor or major behind " + "today's pins, or inside a known issue's affected range. A re-measure is a new pre-registered row; " + "nothing here edits the ledger.", ""] + if data["queue"]: + out += ["| # | Row | Why first | Reason | Suggested row |", "|---|-----|-----------|--------|---------------|"] + for n, q in enumerate(data["queue"], 1): + out.append("| %d | %s | %s | %s | %s |" % (n, q["row"], TIERS[q["tier"]], cell(q["reason"]), + q["suggested_followup_id"])) + else: + out.append("No SUPPORTED row is behind today's pins.") + out += ["", "### Measured versions (SUPPORTED rows)", "", "| Row | Measured | In the queue |", "|---|---|---|"] + queued = {q["row"] for q in data["queue"]} + for row in sorted(data["measured"], key=row_key): + comps = "; ".join("%s %s" % kv for kv in sorted(data["measured"][row].items())) + out.append("| %s | %s | %s |" % (row, cell(comps), "yes" if row in queued else "no")) + out += ["", "Current pins: " + "; ".join("%s %s" % kv for kv in sorted(data["current"].items())), ""] + if data["not_machine_readable"]: + out += ["### Not machine-readable", ""] + out += ["- %s: %s" % (r["row"], r["reason"]) for r in data["not_machine_readable"]] + return "\n".join(out).rstrip() + "\n" + + +def main(argv): + ap = argparse.ArgumentParser(prog="ledger-freshness", description="Read-only ledger re-measure queue.") + ap.add_argument("--root", default=str(HERE.parent), help="repo root (default: this repo)") + mode = ap.add_mutually_exclusive_group(required=True) + mode.add_argument("--json", action="store_true") + mode.add_argument("--md", action="store_true") + args = ap.parse_args(argv) + data = build(args.root) + if args.json: + print(json.dumps(data, indent=1, sort_keys=True)) + else: + sys.stdout.write(markdown(data)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/hack/pinver.py b/hack/pinver.py new file mode 100755 index 00000000..b89506dc --- /dev/null +++ b/hack/pinver.py @@ -0,0 +1,477 @@ +#!/usr/bin/env python3 +"""pinver: version schemes, ranges and update types for the pins k3d-gpu tracks. + +A library (hack/known-issues, hack/ledger-freshness and the freshness tooling +import it) and a small CLI. Python standard library only. + +Schemes: + semver 1.2.3, v1.2.3, 1.2 (padded); a prerelease (-rc.1, rc1) sorts + before its release; build metadata (+x) is ignored. Breaking: major. + zerover semver, but while the major is 0 a minor is breaking (vLLM 0.x). + k8s v1.37.1+k3s1 or v1.37.1-k3s1, -rcN prereleases; the distro build + (k3s1, k3s2) orders after the patch. Breaking: minor and major. + llama llama.cpp: build tags bNNNNN (server-b11960, full-b11960 read as + b11960), compared numerically, any change a non-breaking minor; + and vX.Y.Z releases under zerover rules. A build and a release + are incomparable. + calver 2026.4.1 (2026.4 = 2026.4.0); year = major. Breaking: major. + pep440 Python versions: epoch, release, aN/bN/rcN, .postN, .devN, +local. + A range without a local label ignores the candidate's. Breaking: major. + docker-suffix numeric core with a prefix and suffix that must match: + 3.14.8-slim, contrib-v1.40.0. Tags of another flavour are + incomparable (latest() skips them); a range reads the core + only. Breaking: major. + eat Intel EAT release tags: release-1.1.0, and release-1.0.3-hotfix + (after release-1.0.3), one stream. Breaking: major. + apt Debian versions (dpkg order: epoch, upstream, revision, ~ before + the end). Breaking: major of the upstream's numeric prefix. + hf-sha a Hugging Face revision: no ordering; ranges are * and ==/!= only. + +API (stable; later tooling imports it): + parse(scheme, v) -> Version compare(scheme, a, b) -> -1 | 0 | 1 + comparable(scheme, a, b) -> bool is_prerelease(scheme, v) -> bool + in_range(scheme, v, spec) -> bool check_range(scheme, spec) (raises ValueError) + update_type(scheme, old, new) -> "digest" | "patch" | "minor" | "major" + breaking(scheme, old, new) -> bool classify(scheme, old, new) -> {type, breaking} + latest(scheme, current, candidates) -> newest stable comparable candidate + above current, or None + +update_type reports where two versions differ ("digest" when they are the same +version); breaking applies the scheme's rule, so a zerover 0.24 -> 0.25 is a +"minor" that is breaking. Range spec: "*", or clauses joined by "||", each a +comma-separated list of <, <=, >, >=, ==, != comparators ("<0.25.0", +">=0.24.0,<0.25.0", "<1.95.1 || >=1.96.0,<1.96.2"). Bad input raises ValueError. + +CLI: + pinver.py parse SCHEME V JSON of the parsed version + pinver.py compare SCHEME A B -1, 0 or 1 + pinver.py in-range SCHEME V SPEC exit 0 inside, 1 outside + pinver.py update-type SCHEME OLD NEW "minor breaking", "patch", ... + pinver.py latest SCHEME CURRENT CAND... the newest stable candidate, or nothing +Exit 2 on bad input. +""" + +import functools +import json +import re +import sys + +SCHEMES = ("semver", "zerover", "k8s", "llama", "calver", "pep440", "docker-suffix", "eat", "apt", "hf-sha") +TYPES = ("digest", "patch", "minor", "major") + + +class Version: + """A parsed version. key orders versions of one flavour; release is the + numeric (major, minor, patch, ...) tuple update_type reads.""" + + __slots__ = ("scheme", "raw", "flavour", "key", "release", "pre", "local") + + def __init__(self, scheme, raw, flavour, key, release, pre, local=None): + self.scheme, self.raw, self.flavour, self.key = scheme, raw, flavour, key + self.release, self.pre, self.local = release, pre, local + + def as_dict(self): + return {"scheme": self.scheme, "raw": self.raw, "flavour": list(self.flavour) if self.flavour else None, + "release": list(self.release), "prerelease": self.pre} + + def __repr__(self): + return "Version(%s %r)" % (self.scheme, self.raw) + + +def _bad(scheme, v, why="not a version"): + return ValueError("%s: %r is %s" % (scheme, v, why)) + + +def _pad(nums, n=3): + nums = tuple(nums) + return nums + (0,) * (n - len(nums)) if len(nums) < n else nums + + +# --- semver / zerover ----------------------------------------------------------- + +SEMVER_RE = re.compile( + r"^v?(\d+)(?:\.(\d+))?(?:\.(\d+))?" + r"(?:-([0-9A-Za-z][0-9A-Za-z.-]*)|((?:rc|alpha|beta|a|b|dev|pre)\.?\d+[0-9A-Za-z.]*))?" + r"(?:\+[0-9A-Za-z][0-9A-Za-z.-]*)?$") + + +def _pre_ids(text): + out = [] + for part in re.split(r"[.-]", text): + if not part: + raise ValueError("empty prerelease identifier") + out.append((0, int(part), "") if part.isdigit() else (1, 0, part)) + return tuple(out) + + +def _semver(scheme, v): + m = SEMVER_RE.match(v) + if not m: + raise _bad(scheme, v) + rel = _pad(int(x) for x in m.group(1, 2, 3) if x is not None) + pre_text = m.group(4) or m.group(5) + try: + pre_key = (0,) + _pre_ids(pre_text) if pre_text else (1,) + except ValueError: + raise _bad(scheme, v) from None + return Version(scheme, v, ("",), (rel, pre_key), rel, bool(pre_text)) + + +# --- k8s ----------------------------------------------------------------------- + +K8S_RE = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)(?:-(alpha|beta|rc)\.?(\d+))?(?:[+-]([a-z][a-z0-9]*?)(\d+))?$") +K8S_PRE = {"alpha": 0, "beta": 1, "rc": 2} + + +def _k8s(scheme, v): + m = K8S_RE.match(v) + if not m: + raise _bad(scheme, v) + rel = tuple(int(x) for x in m.group(1, 2, 3)) + pre = (0, K8S_PRE[m.group(4)], int(m.group(5))) if m.group(4) else (1, 0, 0) + distro = m.group(6) or "" + build = int(m.group(7)) if m.group(7) else 0 + return Version(scheme, v, (distro,), (rel, pre, build), rel, bool(m.group(4))) + + +# --- llama.cpp ----------------------------------------------------------------- + +LLAMA_BUILD_RE = re.compile(r"^(?:[a-z][a-z0-9]*(?:-[a-z0-9]+)*-)?b(\d+)$") + + +def _llama(scheme, v): + m = LLAMA_BUILD_RE.match(v) + if m: + n = int(m.group(1)) + return Version(scheme, v, ("build",), (n,), (n,), False) + if not re.match(r"^v\d+\.\d+\.\d+$", v): + raise _bad(scheme, v) + s = _semver(scheme, v) + return Version(scheme, v, ("release",), s.key, s.release, s.pre) + + +# --- calver -------------------------------------------------------------------- + +CALVER_RE = re.compile(r"^(\d{4})\.(\d+)(?:\.(\d+))?$") + + +def _calver(scheme, v): + m = CALVER_RE.match(v) + if not m: + raise _bad(scheme, v) + rel = _pad(int(x) for x in m.group(1, 2, 3) if x is not None) + return Version(scheme, v, ("",), (rel,), rel, False) + + +# --- pep440 -------------------------------------------------------------------- + +PEP440_RE = re.compile( + r"^v?(?:(\d+)!)?(\d+(?:\.\d+)*)" + r"(?:[-_.]?(a|b|rc|c|alpha|beta|pre|preview)[-_.]?(\d+)?)?" + r"(?:-(\d+)|[-_.]?(post|rev|r)[-_.]?(\d+)?)?" + r"(?:[-_.]?(dev)[-_.]?(\d+)?)?" + r"(?:\+([a-z0-9]+(?:[-_.][a-z0-9]+)*))?$", re.I) +PEP440_PRE = {"a": 0, "alpha": 0, "b": 1, "beta": 1, "c": 2, "rc": 2, "pre": 2, "preview": 2} + + +def _pep440(scheme, v): + m = PEP440_RE.match(v.strip()) + if not m: + raise _bad(scheme, v) + epoch = int(m.group(1) or 0) + release = tuple(int(x) for x in m.group(2).split(".")) + trimmed = release + while len(trimmed) > 1 and trimmed[-1] == 0: + trimmed = trimmed[:-1] + pre_l, pre_n = m.group(3), m.group(4) + post_n = m.group(5) or (m.group(7) or "0" if m.group(6) else None) + dev = m.group(8) is not None + dev_n = int(m.group(9) or 0) + if pre_l: + pre = (PEP440_PRE[pre_l.lower()], int(pre_n or 0)) + elif post_n is None and dev: + pre = (-1, 0) # 1.0.dev1 sorts before 1.0a1 + else: + pre = (9, 0) + post = (int(post_n),) if post_n is not None else (-1,) + devk = (0, dev_n) if dev else (1, 0) + local = None + lockey = (0,) + if m.group(10): + local = m.group(10).lower() + segs = tuple((1, int(s), "") if s.isdigit() else (0, 0, s) for s in re.split(r"[-_.]", local)) + lockey = (1, segs) + key = (epoch, trimmed, pre, post, devk, lockey) + return Version(scheme, v, ("",), key, _pad(release), bool(pre_l) or dev, local) + + +# --- docker-suffix ------------------------------------------------------------- + +DOCKER_RE = re.compile(r"^(\D*)(\d+(?:\.\d+)*)(.*)$") +DOCKER_PRE_RE = re.compile(r"^[-.]?(?:rc|alpha|beta|a|b|dev)\d", re.I) + + +def _docker(scheme, v): + m = DOCKER_RE.match(v) + if not m or not v: + raise _bad(scheme, v) + prefix, core, suffix = m.groups() + nums = tuple(int(x) for x in core.split(".")) + rel = _pad(nums) + return Version(scheme, v, (prefix, suffix), (_pad(nums, 8),), rel, bool(DOCKER_PRE_RE.match(suffix))) + + +# --- eat ----------------------------------------------------------------------- + +EAT_RE = re.compile(r"^release-(\d+)\.(\d+)\.(\d+)(-hotfix)?$") + + +def _eat(scheme, v): + m = EAT_RE.match(v) + if not m: + raise _bad(scheme, v) + rel = tuple(int(x) for x in m.group(1, 2, 3)) + return Version(scheme, v, ("",), (rel, bool(m.group(4))), rel, False) + + +# --- apt (dpkg) ---------------------------------------------------------------- + +APT_RE = re.compile(r"^(?:(\d+):)?([0-9][A-Za-z0-9.+~:-]*)$") + + +def _dpkg_order(c): + if c == "~": + return -1 + if c.isalpha(): + return ord(c) + return ord(c) + 256 + + +def _verrevcmp(a, b): + """dpkg's verrevcmp: alternating non-digit and digit runs.""" + i = j = 0 + while i < len(a) or j < len(b): + first_diff = 0 + while (i < len(a) and not a[i].isdigit()) or (j < len(b) and not b[j].isdigit()): + ac = _dpkg_order(a[i]) if i < len(a) and not a[i].isdigit() else 0 + bc = _dpkg_order(b[j]) if j < len(b) and not b[j].isdigit() else 0 + if ac != bc: + return -1 if ac < bc else 1 + i += 1 + j += 1 + while i < len(a) and a[i] == "0": + i += 1 + while j < len(b) and b[j] == "0": + j += 1 + while i < len(a) and a[i].isdigit() and j < len(b) and b[j].isdigit(): + if not first_diff: + first_diff = (a[i] > b[j]) - (a[i] < b[j]) + i += 1 + j += 1 + if i < len(a) and a[i].isdigit(): + return 1 + if j < len(b) and b[j].isdigit(): + return -1 + if first_diff: + return first_diff + return 0 + + +_DPKG_KEY = functools.cmp_to_key(_verrevcmp) + + +def _apt(scheme, v): + m = APT_RE.match(v) + if not m: + raise _bad(scheme, v) + epoch = int(m.group(1) or 0) + rest = m.group(2) + upstream, revision = (rest.rsplit("-", 1) if "-" in rest else (rest, "0")) + if not upstream or not upstream[0].isdigit() or not revision: + raise _bad(scheme, v) + nums = re.match(r"\d+(?:\.\d+)*", upstream).group(0) + rel = _pad(int(x) for x in nums.split(".")) + key = (epoch, _DPKG_KEY(upstream), _DPKG_KEY(revision)) + return Version(scheme, v, ("",), key, rel, "~" in upstream) + + +# --- hf-sha -------------------------------------------------------------------- + +def _hf(scheme, v): + if not re.match(r"^[0-9a-f]{7,40}$", v): + raise _bad(scheme, v, "not a hex revision") + return Version(scheme, v, ("",), None, (), False) + + +PARSERS = {"semver": _semver, "zerover": _semver, "k8s": _k8s, "llama": _llama, "calver": _calver, + "pep440": _pep440, "docker-suffix": _docker, "eat": _eat, "apt": _apt, "hf-sha": _hf} + + +def parse(scheme, v): + if scheme not in PARSERS: + raise ValueError("unknown scheme %r (want one of %s)" % (scheme, ", ".join(SCHEMES))) + if not isinstance(v, str): + raise _bad(scheme, v, "not a string") + return PARSERS[scheme](scheme, v.strip()) + + +def _parsed(scheme, v): + return v if isinstance(v, Version) else parse(scheme, v) + + +def comparable(scheme, a, b): + try: + a, b = _parsed(scheme, a), _parsed(scheme, b) + except ValueError: + return False + return scheme != "hf-sha" and a.flavour == b.flavour + + +def compare(scheme, a, b): + pa, pb = _parsed(scheme, a), _parsed(scheme, b) + if scheme == "hf-sha": + raise ValueError("hf-sha: revisions have no order (%r, %r)" % (pa.raw, pb.raw)) + if pa.flavour != pb.flavour: + raise ValueError("%s: %r and %r are different flavours and do not compare" % (scheme, pa.raw, pb.raw)) + return (pa.key > pb.key) - (pa.key < pb.key) + + +def is_prerelease(scheme, v): + return _parsed(scheme, v).pre + + +# --- ranges -------------------------------------------------------------------- + +CMP_RE = re.compile(r"^(<=|>=|==|!=|<|>)\s*(\S+)$") +OPS = {"<": lambda c: c < 0, "<=": lambda c: c <= 0, ">": lambda c: c > 0, + ">=": lambda c: c >= 0, "==": lambda c: c == 0, "!=": lambda c: c != 0} + + +def _clauses(scheme, spec): + """[[(op, Version), ...], ...] of a spec; [] for "*".""" + if not isinstance(spec, str) or not spec.strip(): + raise ValueError("%s: empty range" % scheme) + if spec.strip() == "*": + return [] + out = [] + for clause in spec.split("||"): + comps = [] + for part in clause.split(","): + m = CMP_RE.match(part.strip()) + if not m: + raise ValueError("%s: bad comparator %r in range %r" % (scheme, part.strip(), spec)) + op = m.group(1) + if scheme == "hf-sha" and op not in ("==", "!="): + raise ValueError("hf-sha: only ==, != and * ranges (got %r)" % spec) + comps.append((op, parse(scheme, m.group(2)))) + out.append(comps) + return out + + +def check_range(scheme, spec): + _clauses(scheme, spec) + + +def _range_cmp(scheme, v, bound): + if scheme == "hf-sha": + same = v.raw.startswith(bound.raw) or bound.raw.startswith(v.raw) + return 0 if same else 1 + if scheme == "docker-suffix": + return (v.key > bound.key) - (v.key < bound.key) + if scheme == "pep440" and v.local is not None and bound.local is None: + k = v.key[:-1] + ((0,),) + return (k > bound.key) - (k < bound.key) + return compare(scheme, v, bound) + + +def in_range(scheme, v, spec): + pv = _parsed(scheme, v) + clauses = _clauses(scheme, spec) + if not clauses: + return True + return any(all(OPS[op](_range_cmp(scheme, pv, bound)) for op, bound in clause) for clause in clauses) + + +# --- update types -------------------------------------------------------------- + +def update_type(scheme, old, new): + po, pn = _parsed(scheme, old), _parsed(scheme, new) + if scheme == "hf-sha": + return "digest" + if compare(scheme, po, pn) == 0: + return "digest" + if scheme == "llama" and po.flavour == ("build",): + return "minor" + ro, rn = _pad(po.release), _pad(pn.release) + if ro[0] != rn[0]: + return "major" + if ro[1] != rn[1]: + return "minor" + return "patch" + + +def breaking(scheme, old, new): + po, pn = _parsed(scheme, old), _parsed(scheme, new) + t = update_type(scheme, po, pn) + if t == "major": + return True + if t != "minor": + return False + if scheme == "k8s": + return True + zero = scheme == "zerover" or (scheme == "llama" and po.flavour == ("release",)) + return zero and po.release[0] == 0 + + +def classify(scheme, old, new): + return {"type": update_type(scheme, old, new), "breaking": breaking(scheme, old, new)} + + +def latest(scheme, current, candidates): + cur = _parsed(scheme, current) + best = None + for c in candidates: + try: + pc = parse(scheme, c) + except ValueError: + continue + if pc.pre or not comparable(scheme, cur, pc) or compare(scheme, pc, cur) <= 0: + continue + if best is None or compare(scheme, pc, best) > 0: + best = pc + return best.raw if best else None + + +# --- CLI ----------------------------------------------------------------------- + +def main(argv): + usage = __doc__.split("CLI:", 1)[1].split("Exit 2", 1)[0] + if len(argv) < 2 or argv[0] in ("-h", "--help"): + print("usage:" + usage, file=sys.stderr) + return 2 + cmd, scheme, args = argv[0], argv[1], argv[2:] + try: + if cmd == "parse" and len(args) == 1: + print(json.dumps(parse(scheme, args[0]).as_dict(), sort_keys=True)) + elif cmd == "compare" and len(args) == 2: + print(compare(scheme, *args)) + elif cmd == "in-range" and len(args) == 2: + return 0 if in_range(scheme, *args) else 1 + elif cmd == "update-type" and len(args) == 2: + c = classify(scheme, *args) + print(c["type"] + (" breaking" if c["breaking"] else "")) + elif cmd == "latest" and args: + got = latest(scheme, args[0], args[1:]) + if got: + print(got) + else: + print("usage:" + usage, file=sys.stderr) + return 2 + except ValueError as e: + print("pinver: %s" % e, file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/images.unpinned b/images.unpinned index a6785cf0..98180692 100644 --- a/images.unpinned +++ b/images.unpinned @@ -29,3 +29,4 @@ tests/image_test.sh * node tests/model_share_test.sh localhost:5000/agent checks a registry port is not read as a tag tests/image_pins_test.sh * the pin gate's own unpinned inputs tests/eat_test.sh * stubs of EAT's gateway chart files, the input gateway-charts.py pins +tests/ledger_freshness_test.py cryptoandcoffee/k3d-gpu:0.1.70 a fixture ledger row's node image the re-measure queue parses, never pulled diff --git a/integrations/eat/README.md b/integrations/eat/README.md index da214a23..28c35356 100644 --- a/integrations/eat/README.md +++ b/integrations/eat/README.md @@ -257,6 +257,12 @@ except where an item says otherwise, and item 11 is new there. In above. `release-1.1.0` reworks the role (`install_nri.yaml` is gone; it renders one `BalloonsPolicy` per node) and still runs it only on `deploy_nri_balloon_policy=yes` (fresh-install.sh:247), which stays off. + `prompt_for_input` turns an absent key on for `compute_platform` c + (parse-user-prompts.sh:55-66). The stack path calls it (fresh-install.sh:39), + so an absent key deploys EAT's policy there. `brownfield.sh model` calls it too, + and `deploy-inference-models.yml` then sets vLLM's `cpu_balloon_annotation` to + EAT's `vllm-balloon-tp` over k3d-gpu's `performance`. `eat-up.sh` stops before + touching the cluster on any value but `off` or `no`, absent included. 11. **The guardrail ignores the config on the brownfield path** (new in `release-1.1.0`). `deploy-agentic-stack.sh:195` sets `DEPLOY_GUARDRAILS="${DEPLOY_GUARDRAILS:-on}"` when it is sourced, and @@ -270,6 +276,15 @@ except where an item says otherwise, and item 11 is new there. In `eat-up.sh` stops before touching the cluster when the config gives `deploy_guardrails` no value (a config written for `release-1.0.3-hotfix` has none) and asks for `off` or `on`. +12. **Release-1.1.0 settings that collide with a k3d-gpu cluster.** `eat-up.sh` + stops before touching the cluster, reading each key as `read_config_file` + does, on `compute_platform=xpu` (fresh-install.sh:219 applies + intel-device-plugins' `gpu_plugin` from `main`, unpinned, beside the plugin + k3d-gpu bakes; vLLM on an Intel GPU is `K3D_GPU_AGENT_BACKEND=vllm k3d-gpu + agent up`), `deploy_kata_containers=on` (Kata needs `/dev/kvm`, which a k3d + node lacks) and `rack_scale_mode=on` (it builds its own cluster from + `rack-profile.yaml`: netplan, BGP, the NVIDIA GPU operator). None of these has + run here; each is read from source. Not a blocker on the previous pin: **AMX/AVX-512 are not required.** The CPU vLLM image (`v0.24.0`) with EAT's exact Qwen3 settings (`VLLM_CPU_SGL_KERNEL=1`, diff --git a/integrations/eat/eat-up.sh b/integrations/eat/eat-up.sh index 5c343a38..4e5ed978 100755 --- a/integrations/eat/eat-up.sh +++ b/integrations/eat/eat-up.sh @@ -66,6 +66,30 @@ esac # choice is asked for here rather than made for the user. [ -n "$(cfg_value "${EAT_CONFIG}" deploy_guardrails)" ] \ || err "${EAT_CONFIG} does not set deploy_guardrails, so ${EAT_TAG} would deploy its Qwen3Guard guardrail, which has not run on a k3d-gpu cluster; set deploy_guardrails=off (as the shipped agentic-config.cfg does) or deploy_guardrails=on" +# Settings with which ${EAT_TAG} puts a second copy of what a k3d-gpu cluster already +# runs next to it, or needs what a k3d node lacks (core/lib/cluster/deployment/fresh-install.sh): +# - its NRI balloons policy on yes (:247). prompt_for_input turns an absent key on for +# compute_platform c (parse-user-prompts.sh:55-66). The stack path calls it (:39); so +# does brownfield.sh model mode, after which deploy-inference-models.yml gives vLLM +# EAT's balloon (vllm-balloon-tp) in place of k3d-gpu's performance balloon. +# - intel-device-plugins' gpu_plugin from its main branch, unpinned, on xpu (:219, +# playbooks/deploy-intel-gpu-plugin.yml:12-14). +# - Kata Containers, which needs /dev/kvm on the node (playbooks/deploy-kata-containers.yml). +# - rack-scale mode, which builds the cluster from rack-profile.yaml: netplan, BGP and the +# NVIDIA GPU operator (:195-210). +case "$(cfg_value "${EAT_CONFIG}" deploy_nri_balloon_policy)" in + off|no) ;; + *) err "${EAT_CONFIG}: deploy_nri_balloon_policy must be off: ${EAT_TAG} deploys its own NRI balloons policy when it is on or absent, beside k3d-gpu's, which must remain the only one, and in model mode it moves vLLM to a balloon type k3d-gpu's policy does not define" ;; +esac +case "$(cfg_value "${EAT_CONFIG}" compute_platform)" in + x|xpu) err "compute_platform=xpu: ${EAT_TAG} then applies intel-device-plugins' gpu_plugin from its main branch, unpinned, beside the GPU plugin k3d-gpu bakes; for vLLM on an Intel GPU run K3D_GPU_AGENT_BACKEND=vllm k3d-gpu agent up" ;; +esac +case "$(cfg_value "${EAT_CONFIG}" deploy_kata_containers)" in + on|yes) err "deploy_kata_containers=on: Kata needs /dev/kvm on the node, and a k3d node is a container without it; set deploy_kata_containers=off" ;; +esac +case "$(cfg_value "${EAT_CONFIG}" rack_scale_mode)" in + on|yes) err "rack_scale_mode=on: ${EAT_TAG} then builds a cluster from rack-profile.yaml (netplan, BGP, the NVIDIA GPU operator), and a k3d-gpu cluster is brownfield; set rack_scale_mode=off" ;; +esac # The KV cache from the fit of EAT_FIT_MODEL, unless EAT_VLLM_KV_GIB is set. if [ -z "${EAT_VLLM_KV_GIB:-}" ] && [ -n "${EAT_FIT_MODEL:-}" ]; then command -v k3d-gpu >/dev/null 2>&1 || err "EAT_FIT_MODEL needs k3d-gpu on PATH (or set EAT_VLLM_KV_GIB)" diff --git a/pins/known-issues.toml b/pins/known-issues.toml new file mode 100644 index 00000000..c0c587a4 --- /dev/null +++ b/pins/known-issues.toml @@ -0,0 +1,219 @@ +# Known upstream issues k3d-gpu depends on: bugs, behaviour changes and security +# advisories, each with the release that fixes it. hack/known-issues reads this +# file (Python tomllib); `hack/known-issues check` runs in CI, offline. +# +# Fields of each [[issue]]: +# id lowercase id; markers name it +# title one line +# kind bug | change | advisory +# upstream "owner/repo#n" (an issue or a pull request), or "" +# fix {} when unknown, or one of: +# {repo, sha and/or pr} the commit or pull request that fixes it +# {repo, path, contains} fixed in the first release whose file +# `path` contains `contains` +# {ecosystem, package} GitHub advisory database (kind advisory) +# component what the sites pin (vllm-openai-cpu, kserve, litellm, ...) +# scheme a hack/pinver.py scheme for the site versions and the fields below +# affected a pinver range of affected versions ("*" until the fix is known) +# fixed_in the first release with the fix; "" until known. Must lie outside +# `affected` (narrow `affected` when setting it). +# adopt review | auto. auto is owner consent to take a patch or minor fix +# without review; a major always needs review. +# sites [{file, regex}]: where k3d-gpu pins the component. Each regex +# (Python re, MULTILINE) has a named group `v` and must match at +# least once. Files are "ours" or "eat-stock" only (pins/classes). +# workarounds files carrying `known-issue: ` markers (or a +# `known-issue-begin: ` .. `known-issue-end: ` block) for +# code or docs that exist because of the issue. Never in bench/, +# oracle/seed/, console/matrix.v1.json or EXPERIMENTS.md. +# retained {} or {reason, pr}: the workarounds stay after the fix, and why +# ledger_refs [{row, path, regex?}]: ledger rows whose recorded versions the +# issue touches. Informational: reported, never failed, never edited. +# advisories kind advisory: the GHSA ids this entry covers +# image optional image reference template ({tag} = release tag, +# {version} = tag without a leading v); poll checks it resolves +# test optional: a test path or canary: +# notes free text +# +# A check fails when an "ours" site's version is inside `affected` while `fixed_in` +# is set (bump it), or when workarounds remain after every "ours" site reached +# `fixed_in` and `retained` is empty (remove them, or say why they stay). +# `hack/known-issues poll` (network, read-only) finds fixed_in for open entries. + +[[issue]] +id = "vllm-45844" +title = "vLLM CPU attention sizes its split-KV scratchpad too small (Gemma 4 global layers stall, EngineDeadError)" +kind = "bug" +upstream = "vllm-project/vllm#45844" +fix = {repo = "vllm-project/vllm", sha = "26eb87204d58d9d8a25c378fbaec9da23db3fb85", pr = 45844} +component = "vllm-openai-cpu" +scheme = "zerover" +affected = "<0.25.0" +fixed_in = "0.25.0" +adopt = "review" +image = "vllm/vllm-openai-cpu:{tag}" +sites = [ + {file = "scripts/k3d-gpu", regex = '^AMX_TEST_IMAGE="\$\{K3D_GPU_AMX_TEST_IMAGE:-vllm/vllm-openai-cpu:v(?P\d+\.\d+\.\d+)@sha256:'}, + {file = "console/moe_predict.py", regex = '"vllm/vllm-openai-cpu:v(?P\d+\.\d+\.\d+)@sha256:'}, + {file = "integrations/eat/vllm-values.yaml", regex = 'repository: vllm/vllm-openai-cpu\n\s+tag: "v(?P\d+\.\d+\.\d+)@sha256:'}, +] +workarounds = [] +ledger_refs = [ + {row = "H54a", path = "bench/h54/h54.py", regex = '"vllm024": "vllm/vllm-openai-cpu:v(?P\d+\.\d+\.\d+)@'}, + {row = "H53", path = "bench/eatflow/live.sh", regex = 'B1_TAG=\$\{LIVE_EAT_B1_TAG:-v(?P\d+\.\d+\.\d+)@'}, +] +notes = """The B1g stall (EXPERIMENTS.md, H54 note N2). Merged 2026-07-04 as 26eb8720; GitHub's \ +compare puts it in v0.25.0 (behind 106, ahead 0) and not in v0.24.0 (diverged). EAT \ +release-1.0.3-hotfix pins v0.24.0; EAT release-1.1.0 moves to v0.28.0. The ledger arms stay \ +on v0.24.0: changing them takes a dated amendment.""" + +[[issue]] +id = "vllm-50801" +title = "vLLM no longer reads VLLM_CPU_SGL_KERNEL (CPU kernel dispatch refined)" +kind = "change" +upstream = "vllm-project/vllm#50801" +fix = {repo = "vllm-project/vllm", sha = "c8602c79062440074a018c1d5f875a5571eb6881", pr = 50801} +component = "vllm-openai-cpu" +scheme = "zerover" +affected = "<0.28.0" +fixed_in = "0.28.0" +adopt = "review" +sites = [ + {file = "integrations/eat/vllm-values.yaml", regex = 'repository: vllm/vllm-openai-cpu\n\s+tag: "v(?P\d+\.\d+\.\d+)@sha256:'}, +] +workarounds = [] +notes = """EAT's xeon-values.yaml sets VLLM_CPU_SGL_KERNEL=1 (integrations/eat/README.md); from \ +v0.28.0 on it is a no-op, so an EAT image at or past v0.28.0 runs without the SGL kernels it \ +asks for. k3d-gpu's own vLLM CPU path does not set it.""" + +[[issue]] +id = "vllm-pr-51493" +title = "vLLM's V1 model runner fails Gemma 4 MoE MTP drafting (pre_projection shape mismatch); VLLM_USE_V2_MODEL_RUNNER=1 works around it" +kind = "bug" +upstream = "vllm-project/vllm#51493" +fix = {repo = "vllm-project/vllm", pr = 51493} +component = "vllm" +scheme = "zerover" +affected = "*" +fixed_in = "" +adopt = "review" +sites = [ + {file = "console/model_opt.py", regex = 'o\.get\("vllm_version"\) or "(?P\d+\.\d+\.\d+)"'}, +] +workarounds = ["console/model_opt.py"] +notes = """The optimizer's MoE drafter note points at VLLM_USE_V2_MODEL_RUNNER=1 until the pull \ +request is merged and released.""" + +[[issue]] +id = "linux-npu-driver-56" +title = "Intel's Linux NPU driver has no WSL2 support" +kind = "bug" +upstream = "intel/linux-npu-driver#56" +fix = {} +component = "linux-npu-driver" +scheme = "semver" +affected = "*" +fixed_in = "" +adopt = "review" +sites = [ + {file = "scripts/k3d-gpu", regex = '^HS_NPU_REL=v(?P\d+\.\d+\.\d+)$'}, +] +workarounds = ["scripts/k3d-gpu", "docs/first-run.md", "docs/quickstart.md"] +notes = """quickstart's WSL2 guidance and the WSL2 docs say the NPU is unavailable there.""" + +[[issue]] +id = "kserve-4592" +title = "KServe's kserve-openvino ClusterServingRuntime is in the source tree but in no release's runtimes chart" +kind = "bug" +upstream = "kserve/kserve#4592" +fix = {repo = "kserve/kserve", path = "charts/kserve-runtime-configs/files/runtimes/resources.yaml", contains = "name: kserve-openvino"} +component = "kserve" +scheme = "semver" +affected = "*" +fixed_in = "" +adopt = "review" +image = "ghcr.io/kserve/charts/kserve-resources:{tag}" +sites = [ + {file = "examples/kserve/install.sh", regex = '^KSERVE_VERSION=v(?P\d+\.\d+\.\d+)$'}, +] +workarounds = ["examples/kserve/openvino-runtime.yaml", "examples/kserve/install.sh"] +notes = """kserve#4592 (merged 2026-01-16) added config/runtimes/kserve-openvino.yaml, and every \ +release since contains that commit, but neither config/runtimes/kustomization.yaml nor the \ +kserve-runtime-configs chart lists the runtime (v0.21.0 and master, read 2026-10-29). Fixed in \ +the first release whose runtimes chart names kserve-openvino; until then the example vendors \ +the runtime.""" + +[[issue]] +id = "eat-9" +title = "EAT's vLLM chart default (apisix.enabled: false) serves with no authentication" +kind = "bug" +upstream = "intel/enterprise-agent-toolkit#9" +fix = {} +component = "enterprise-agent-toolkit" +scheme = "eat" +affected = "*" +fixed_in = "" +adopt = "review" +sites = [ + {file = "integrations/eat/eat-up.sh", regex = '^EAT_TAG="(?Prelease-[^"]+)"$'}, +] +workarounds = ["console/audit.py"] +notes = """`k3d-gpu audit` fails any unauthenticated listener off loopback, the EAT#9 class. The \ +rule is general; when EAT ships the fix, keep it with `retained`.""" + +[[issue]] +id = "litellm-1.83.14-advisories" +title = "LiteLLM 1.83.14 is affected by published security advisories" +kind = "advisory" +upstream = "" +fix = {ecosystem = "pip", package = "litellm"} +component = "litellm" +scheme = "pep440" +affected = "<1.100.4 || >=1.101.0,<1.101.3 || >=1.102.0,<1.102.2 || >=1.103.0,<1.103.1 || >=1.104.0rc1,<1.104.0rc2" +fixed_in = "1.100.4" +adopt = "review" +advisories = [ + "GHSA-4xpc-pv4p-pm3w", + "GHSA-4jcj-7x88-m979", + "GHSA-7488-6r32-c95q", + "GHSA-3cv6-jpf6-8222", + "GHSA-hhww-mrg2-969h", + "GHSA-g5ff-637f-6q2m", + "GHSA-7hp6-4w63-5g45", +] +sites = [ + {file = "integrations/eat/gateway-values.yaml", regex = 'repository: docker\.litellm\.ai/berriai/litellm\n\s+tag: "v(?P\d+\.\d+\.\d+)-'}, + {file = "examples/agents/litellm-gateway/requirements.txt", regex = '^litellm==(?P\S+)$'}, + {file = "examples/agents/litellm-gateway/constraints.txt", regex = '^litellm==(?P\S+)$'}, +] +workarounds = [] +notes = """Read from the GitHub advisory database 2026-10-29: fixed in 1.84.0 (GHSA-4xpc, -4jcj, \ +-7488), 1.85.0 (GHSA-hhww), 1.88.6 with backports up to 1.96.2 (GHSA-3cv6), 1.95.0 (GHSA-g5ff) \ +and 1.100.4 with backports 1.101.3, 1.102.2 and 1.103.1 (GHSA-7hp6, high, published 2026-10-27; \ +it affects 1.91.0 through 1.100.3, so not 1.83.14 itself, but 1.95.1 and 1.96.2). 1.100.4 is the \ +first release outside every range, and the advisory database lists none for it. Both sites are \ +EAT's stock gateway, so the check reports and does not fail.""" + +[[issue]] +id = "vllm-0.32-num-spec-tokens" +title = "vLLM v0.32.0 compares num_speculative_tokens before defaulting it (gemma4_mtp drafter with no length exits with a TypeError)" +kind = "bug" +upstream = "" +fix = {} +component = "vllm" +scheme = "zerover" +affected = ">=0.32.0" +fixed_in = "" +adopt = "review" +sites = [ + {file = "console/model_opt.py", regex = 'o\.get\("vllm_version"\) or "(?P\d+\.\d+\.\d+)"'}, +] +workarounds = [] +ledger_refs = [ + {row = "H54q", path = "bench/h54/h54.py"}, +] +notes = """File upstream: config/speculative.py rewrites a gemma4_assistant drafter to gemma4_mtp \ +with n_predict 1, then reaches the MTP branch's num_speculative_tokens > 1 check before the line \ +that defaults num_speculative_tokens to n_predict ("TypeError: '>' not supported between \ +instances of 'NoneType' and 'int'"). H54 amendment R2 names the length (1). The optimizer always \ +names num_speculative_tokens, so its plans do not hit it.""" diff --git a/scripts/k3d-gpu b/scripts/k3d-gpu index b3bb2260..545b1018 100755 --- a/scripts/k3d-gpu +++ b/scripts/k3d-gpu @@ -10208,7 +10208,8 @@ qs_step() { echo "${GRN}==>${RST} [$1/${QS_STEPS}] $(progress_bar "$(($1 - 1))" # paravirtualization for NVIDIA only (https://docs.docker.com/desktop/features/gpu/); # the Intel path is /dev/dxg plus /usr/lib/wsl, as in OpenVINO's container recipe # (https://github.com/openvinotoolkit/docker_ci/blob/master/docs/accelerators.md). -# The Intel NPU has no WSL2 driver (https://github.com/intel/linux-npu-driver/issues/56). +# The Intel NPU has no WSL2 driver (https://github.com/intel/linux-npu-driver/issues/56; +# known-issue: linux-npu-driver-56). # agent_device requests gpu.intel.com/xe, i915, a Gaudi or the CPU, never # gpu.intel.com/dxg, so under WSL2 quickstart's LLM runs on the CPU. # Host changes are printed as suggestions, never run. @@ -10229,7 +10230,9 @@ qs_wsl_guidance() { fi echo " Docker: Docker Desktop with WSL integration enabled for this distro, or Docker Engine inside it." echo " Docker Desktop's --gpus flag is NVIDIA-only; the Intel GPU goes through /dev/dxg instead." + # known-issue-begin: linux-npu-driver-56 echo " The Intel NPU is not reachable from WSL2 (intel/linux-npu-driver#56)." + # known-issue-end: linux-npu-driver-56 } # Pid of the port-forward quickstart started, while it runs. A pid whose diff --git a/tests/claims_check.sh b/tests/claims_check.sh index 67ea642e..0db9f460 100755 --- a/tests/claims_check.sh +++ b/tests/claims_check.sh @@ -21,10 +21,16 @@ # docs/provenance.md is the source of truth for cited numbers and is not scanned # either. A file the guard cannot read is an error. # -# Usage: tests/claims_check.sh [repo-root] +# --cited prints, instead of checking, the ids of the ledger rows that claims cite +# (a [ledger:Hn] reference on a claim line, or on either line of a claim split over +# two lines), sorted, one per line, and exits 0. hack/ledger-freshness reads it. +# +# Usage: tests/claims_check.sh [--cited] [repo-root] set -uo pipefail +CITED=0 +if [ "${1:-}" = "--cited" ]; then CITED=1; shift; fi ROOT=${1:-"$(cd "$(dirname "$0")/.." && pwd)"} LEDGER="${ROOT}/EXPERIMENTS.md" PROVENANCE="${ROOT}/docs/provenance.md" @@ -87,7 +93,7 @@ report="" for f in "${files[@]}"; do rel=${f#"${ROOT}/"} [ -r "${f}" ] || read_error "${f}" - out=$(awk -v rel="${rel}" -v ledger="${ledger}" -v prov="${provenance}" ' + out=$(awk -v rel="${rel}" -v ledger="${ledger}" -v prov="${provenance}" -v cited="${CITED}" ' function load(s, m, n, i, kv, p) { n = split(s, kv, ";") for (i = 1; i <= n; i++) if ((p = index(kv[i], "=")) > 0) m[substr(kv[i], 1, p - 1)] = substr(kv[i], p + 1) @@ -154,13 +160,14 @@ for f in "${files[@]}"; do claim = is_claim(low) - backed = linked; notes = ""; rest = s + backed = linked; notes = ""; rest = s; rows = "" while (match(rest, /\[(ledger|provenance):[A-Za-z0-9]+\]/)) { ref = substr(rest, RSTART, RLENGTH) rest = substr(rest, RSTART + RLENGTH) kind = substr(ref, 2, index(ref, ":") - 2) id = substr(ref, index(ref, ":") + 1); sub(/\]$/, "", id) if (kind == "ledger") { + rows = rows " " id if (!(id in L)) printf "%s:%d: %s not in EXPERIMENTS.md hypothesis table\n", rel, NR, ref else if (L[id] == "SUPPORTED") backed = 1 else notes = notes sprintf("; %s is %s", ref, L[id]) @@ -179,14 +186,22 @@ for f in "${files[@]}"; do } if ((claim || split_claim) && !backed) printf "%s:%d: unbacked claim (no ledger or provenance reference that backs it%s): %s\n", rel, NR, notes, $0 + if (cited && (claim || split_claim)) { + n = split(rows (split_claim ? prevrows : ""), ids, " ") + for (i = 1; i <= n; i++) if (ids[i] != "") printf "cite %s\n", ids[i] + } if (low ~ /^[ \t]*$/) prev = "" - else { prev = low; prevclaim = claim || split_claim; prevbacked = backed } + else { prev = low; prevclaim = claim || split_claim; prevbacked = backed; prevrows = rows } }' "${f}") || read_error "${f}" report+=${out}$'\n' done report=$(sed '/^$/d' <<< "${report}") +if [ "${CITED}" = 1 ]; then + sed -n 's/^cite //p' <<< "${report}" | sort -u + exit 0 +fi if [ -z "${report}" ]; then echo "claims_check: ${#files[@]} file(s) clean" exit 0 diff --git a/tests/claims_check_test.sh b/tests/claims_check_test.sh index 7290299d..2c2df5dd 100755 --- a/tests/claims_check_test.sh +++ b/tests/claims_check_test.sh @@ -228,6 +228,32 @@ mkdir -p "${intel}/docs/intel" printf '%s\n' 'The iGPU serves 42 tok/s.' > "${intel}/docs/intel/report.md" check "an unbacked claim in docs/intel/ fails" "${intel}" fail 'docs/intel/report.md:1' +# --- --cited: the ledger rows docs claims cite (hack/ledger-freshness reads it) ---- +# A row counts when its [ledger:Hn] reference sits on a claim line, or on the +# previous line of a claim split over two lines; a reference without a claim, one +# in a fenced block or one in an HTML comment does not. +cited=$(repo 'The CPU serves 42 tok/s [ledger:H2]. +See [ledger:H3] for context. +' 'Measured in [ledger:H1]: the iGPU decodes 42 +tok/s on this host. + +```text +50 tok/s [ledger:H3] +```') +out=$("${GUARD}" --cited "${cited}" 2>&1); rc=$? +if [ "${rc}" -eq 0 ] && [ "${out}" = $'H1\nH2' ]; then + PASS=$((PASS + 1)); echo "ok --cited lists the rows claim lines cite, sorted, once each" +else + FAILS=$((FAILS + 1)); echo "FAIL --cited lists the rows claim lines cite, sorted, once each" + echo " | rc=${rc} output: ${out//$'\n'/ | }" +fi +out=$("${GUARD}" --cited "$(repo 'An unbacked 42 tok/s claim.')" 2>&1); rc=$? +if [ "${rc}" -eq 0 ] && [ -z "${out}" ]; then + PASS=$((PASS + 1)); echo "ok --cited prints nothing and exits 0 when no claim cites a row" +else + FAILS=$((FAILS + 1)); echo "FAIL --cited prints nothing and exits 0 when no claim cites a row (rc=${rc}: ${out})" +fi + # --- the real repo ----------------------------------------------------------- check "this repo's own docs pass" "$(cd "$(dirname "$0")/.." && pwd)" 0 diff --git a/tests/eat_test.sh b/tests/eat_test.sh index e841e5e3..0e6b1872 100755 --- a/tests/eat_test.sh +++ b/tests/eat_test.sh @@ -10,7 +10,10 @@ # python-runtime-sandbox) and for one that runs it under /bin/sh -c # - the check fails when the sandbox prints anything but 385 # 3. eat-up.sh refuses deploy_openshell=on, unverified on k3d-gpu, and a config with no -# deploy_guardrails value, on which release-1.1.0 deploys its guardrail. +# deploy_guardrails value, on which release-1.1.0 deploys its guardrail. It also +# refuses a deploy_nri_balloon_policy other than off (a second balloons policy), +# compute_platform=xpu (an unpinned second GPU plugin), deploy_kata_containers=on +# (no /dev/kvm in a k3d node) and rack_scale_mode=on (it builds its own cluster). # 5. The GenAI Gateway installs from copies of its charts with exact dependency # versions and every image name:tag@sha256 (gateway-charts.py, brownfield.sh). # 6. eat-opt.py puts the model optimizer's choices on EAT's own modelConfigs entry, @@ -342,6 +345,74 @@ guard_case "deploy_guardrails=on is the user's choice" pass 'deploy_guardrails=o guard_case "spaces and quotes" pass ' deploy_guardrails = "off" ' guard_case "set, then emptied: last wins" refuse 'deploy_guardrails=off' 'deploy_guardrails=' +# 3d. eat-up.sh stops before the cluster on release-1.1.0 settings that put a second copy +# of what a k3d-gpu cluster runs next to it, or that need what a k3d node lacks. Each +# case starts from the shipped config without the key's line; the key is read as +# read_config_file reads it (on -> yes, off -> no, last assignment wins). +# - deploy_nri_balloon_policy: fresh-install.sh:247 deploys EAT's NRI balloons policy +# on yes. prompt_for_input turns an absent key on for compute_platform c +# (parse-user-prompts.sh:55-66), and both the stack path (fresh-install.sh:39) and +# brownfield.sh model mode call it: only off or no pass. +# - compute_platform x or xpu: fresh-install.sh:219 applies intel-device-plugins' +# gpu_plugin from its main branch (deploy-intel-gpu-plugin.yml:12-14). +# - deploy_kata_containers on or yes: Kata needs /dev/kvm (deploy-kata-containers.yml). +# - rack_scale_mode on or yes: builds the cluster from rack-profile.yaml (netplan, BGP, +# the NVIDIA GPU operator; fresh-install.sh:195-210). +# key_case +key_case() { + local name=$1 want=$2 refused=$3 key=$4 out rc msg cfg + shift 4 + N=$((N + 1)) + cfg="${WORK}/key-${N}.cfg" + grep -v "^${key}=" "${EAT}/agentic-config.cfg" > "${cfg}" + [ "$#" -eq 0 ] || printf '%s\n' "$@" >> "${cfg}" + out=$(EAT_CONFIG="${cfg}" EAT_KUBECONFIG=/nonexistent "${EAT}/eat-up.sh" 2>&1) && rc=0 || rc=$? + if [ "${want}" = refuse ]; then msg=${refused}; else msg=${PASSED}; fi + if [ "${rc}" -ne 0 ] && grep -qF "${msg}" <<< "${out}" && { [ "${want}" = pass ] || ! grep -qF "${PASSED}" <<< "${out}"; }; then + ok "eat-up ${key} ${want}: ${name}" + else + fail "eat-up ${key} ${want}: ${name} (rc=${rc}): ${out}" + fi +} +NRI_REFUSED="deploy_nri_balloon_policy must be off" +key_case "absent (prompt_for_input turns it on)" refuse "${NRI_REFUSED}" deploy_nri_balloon_policy +key_case "on" refuse "${NRI_REFUSED}" deploy_nri_balloon_policy 'deploy_nri_balloon_policy=on' +key_case "yes" refuse "${NRI_REFUSED}" deploy_nri_balloon_policy 'deploy_nri_balloon_policy=yes' +key_case "empty" refuse "${NRI_REFUSED}" deploy_nri_balloon_policy 'deploy_nri_balloon_policy=' +key_case "off then on, last wins" refuse "${NRI_REFUSED}" deploy_nri_balloon_policy \ + 'deploy_nri_balloon_policy=off' 'deploy_nri_balloon_policy=on' +key_case "off" pass "" deploy_nri_balloon_policy 'deploy_nri_balloon_policy=off' +key_case "no" pass "" deploy_nri_balloon_policy 'deploy_nri_balloon_policy=no' +key_case "spaces and quotes" pass "" deploy_nri_balloon_policy ' deploy_nri_balloon_policy = "off" ' +cfg="${WORK}/nri-absent.cfg" +grep -v '^deploy_nri_balloon_policy=' "${EAT}/agentic-config.cfg" > "${cfg}" +out=$(EAT_CONFIG="${cfg}" EAT_KUBECONFIG=/nonexistent "${EAT}/eat-up.sh" 2>&1) || true +if grep -qF 'deploys its own NRI balloons policy when it is on or absent' <<< "${out}"; then + ok "eat-up: the balloons refusal says an absent key deploys EAT's policy on the stack path too" +else fail "eat-up: the balloons refusal ties an absent key to model mode only: ${out}"; fi +XPU_REFUSED="compute_platform=xpu" +key_case "xpu" refuse "${XPU_REFUSED}" compute_platform 'compute_platform=xpu' +key_case "x" refuse "${XPU_REFUSED}" compute_platform 'compute_platform=x' +key_case "quoted xpu" refuse "${XPU_REFUSED}" compute_platform 'compute_platform = "xpu"' +key_case "cpu" pass "" compute_platform 'compute_platform=cpu' +key_case "absent (EAT's default cpu)" pass "" compute_platform +KATA_REFUSED="deploy_kata_containers=on" +key_case "on" refuse "${KATA_REFUSED}" deploy_kata_containers 'deploy_kata_containers=on' +key_case "yes" refuse "${KATA_REFUSED}" deploy_kata_containers 'deploy_kata_containers=yes' +key_case "off then on, last wins" refuse "${KATA_REFUSED}" deploy_kata_containers \ + 'deploy_kata_containers=off' 'deploy_kata_containers=on' +key_case "off" pass "" deploy_kata_containers 'deploy_kata_containers=off' +RACK_REFUSED="rack_scale_mode=on" +key_case "on" refuse "${RACK_REFUSED}" rack_scale_mode 'rack_scale_mode=on' +key_case "yes" refuse "${RACK_REFUSED}" rack_scale_mode 'rack_scale_mode=yes' +key_case "off" pass "" rack_scale_mode 'rack_scale_mode=off' +cfg="${WORK}/xpu-reason.cfg" +cp "${EAT}/agentic-config.cfg" "${cfg}" +echo 'compute_platform=xpu' >> "${cfg}" +out=$(EAT_CONFIG="${cfg}" EAT_KUBECONFIG=/nonexistent "${EAT}/eat-up.sh" 2>&1) || true +if grep -qF 'K3D_GPU_AGENT_BACKEND=vllm' <<< "${out}"; then ok "eat-up: the xpu refusal points at vLLM on an Intel GPU through k3d-gpu agent up" +else fail "eat-up: the xpu refusal names no alternative: ${out}"; fi + # 5. The GenAI Gateway path installs pinned charts and images. EAT's # deploy-genai-gateway.yml syncs helm_charts_base_genai{,_trace} to the node, runs # `helm dependency update` on genai-gateway (Chart.yaml ranges postgresql >=13.3.0, diff --git a/tests/known_issues_test.py b/tests/known_issues_test.py new file mode 100755 index 00000000..3ed6deb7 --- /dev/null +++ b/tests/known_issues_test.py @@ -0,0 +1,581 @@ +#!/usr/bin/env python3 +"""Tests for hack/known-issues. Each case writes a throwaway repo tree (registry, +site files, markers, a small ledger) and runs `check` or `poll` on it; poll runs +against a stubbed GitHub API. The last cases run check on this repo. + +Usage: python3 tests/known_issues_test.py +""" + +import base64 +import hashlib +import importlib.machinery +import importlib.util +import io +import json +import contextlib +import pathlib +import re +import shutil +import subprocess +import sys +import tempfile +import textwrap +import tomllib +import unittest + +ROOT = pathlib.Path(__file__).resolve().parent.parent +TOOL = ROOT / "hack" / "known-issues" + + +def load_tool(): + loader = importlib.machinery.SourceFileLoader("known_issues", str(TOOL)) + spec = importlib.util.spec_from_loader("known_issues", loader) + mod = importlib.util.module_from_spec(spec) + loader.exec_module(mod) + return mod + + +ki = load_tool() + +LEDGER = """# Experiments + +## Hypotheses + +| # | Hypothesis | Prediction | Probe | Verdict | +|---|------------|------------|-------|---------| +| H53 | stock arm | p | probe | PENDING | +| H54a | engine | p | probe | INCONCLUSIVE | + +## Falsification log +""" + +VLLM = """ +[[issue]] +id = "vllm-fix" +title = "a CPU attention bug" +kind = "bug" +upstream = "vllm-project/vllm#45844" +fix = {repo = "vllm-project/vllm", sha = "26eb87204d58d9d8a25c378fbaec9da23db3fb85"} +component = "vllm-openai-cpu" +scheme = "zerover" +affected = "<0.25.0" +fixed_in = "0.25.0" +adopt = "review" +image = "vllm/vllm-openai-cpu:{tag}" +sites = [ + {file = "scripts/tool.sh", regex = '^CPU_IMAGE="vllm/vllm-openai-cpu:v(?P[0-9.]+)@'}, + {file = "integrations/eat/values.yaml", regex = 'tag: "v(?P[0-9.]+)@'}, +] +workarounds = [] +ledger_refs = [{row = "H53", path = "bench/live.sh", regex = 'B1_TAG=v(?P[0-9.]+)'}] +notes = "" +""" + +OPEN = """ +[[issue]] +id = "open-bug" +title = "an open upstream bug" +kind = "bug" +upstream = "acme/engine#7" +fix = {repo = "acme/engine", pr = 7} +component = "engine" +scheme = "semver" +affected = "*" +fixed_in = "" +adopt = "review" +sites = [{file = "console/opt.py", regex = 'ENGINE = "(?P[0-9.]+)"'}] +workarounds = ["console/opt.py"] +notes = "" +""" + +FILES = { + "EXPERIMENTS.md": LEDGER, + "scripts/tool.sh": 'CPU_IMAGE="vllm/vllm-openai-cpu:v0.32.0@sha256:' + "a" * 64 + '"\n', + "integrations/eat/values.yaml": 'image:\n tag: "v0.24.0@sha256:' + "b" * 64 + '"\n', + "bench/live.sh": "B1_TAG=v0.24.0\n", + "console/opt.py": textwrap.dedent("""\ + ENGINE = "1.4.0" + # known-issue-begin: open-bug + NOTE = "set ENGINE_FALLBACK=1 until the fix ships" + # known-issue-end: open-bug + """), +} + + +def make_tree(registry, files=None, extra=None): + d = pathlib.Path(tempfile.mkdtemp(prefix="known_issues.")) + tree = dict(FILES if files is None else files) + if 'id = "open-bug"' not in registry: + tree.pop("console/opt.py", None) # its markers name open-bug + tree.update(extra or {}) + tree["pins/known-issues.toml"] = registry + for rel, text in tree.items(): + p = d / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(text) + return d + + +def run_check(root): + out = io.StringIO() + with contextlib.redirect_stdout(out): + rc = ki.main(["--root", str(root), "check"]) + return rc, out.getvalue() + + +def run_check_json(root): + out = io.StringIO() + with contextlib.redirect_stdout(out): + rc = ki.main(["--root", str(root), "check", "--json"]) + return rc, json.loads(out.getvalue()) + + +class Check(unittest.TestCase): + def setUp(self): + self.dirs = [] + + def tearDown(self): + for d in self.dirs: + shutil.rmtree(d, ignore_errors=True) + + def tree(self, registry, files=None, extra=None): + d = make_tree(registry, files, extra) + self.dirs.append(d) + return d + + def assertFails(self, root, needle): + rc, out = run_check(root) + self.assertEqual(rc, 1, out) + self.assertIn(needle, out) + return out + + def test_fixture_passes(self): + rc, out = run_check(self.tree(VLLM + OPEN)) + self.assertEqual(rc, 0, out) + + def test_ours_site_in_range_with_fixed_in_fails_with_bump_message(self): + files = dict(FILES) + files["scripts/tool.sh"] = files["scripts/tool.sh"].replace("v0.32.0", "v0.24.0") + self.assertFails(self.tree(VLLM + OPEN, files), "bump scripts/tool.sh to >= 0.25.0") + + def test_in_range_site_with_fixed_in_empty_passes(self): + # open-bug: affected "*" covers console/opt.py's 1.4.0, fixed_in is empty. + rc, data = run_check_json(self.tree(OPEN)) + self.assertEqual(rc, 0) + site = data["issues"][0]["sites"][0] + self.assertEqual((site["versions"], site["in_affected"]), (["1.4.0"], True)) + + def test_eat_stock_in_range_only_reports(self): + rc, data = run_check_json(self.tree(VLLM)) + self.assertEqual(rc, 0) + self.assertTrue(any("integrations/eat/values.yaml" in r and "eat-stock" in r for r in data["reports"]), + data["reports"]) + + def test_ledger_ref_in_range_only_reports(self): + rc, data = run_check_json(self.tree(VLLM)) + self.assertEqual(rc, 0) + self.assertTrue(any("H53" in r and "bench/live.sh" in r for r in data["reports"]), data["reports"]) + ref = data["issues"][0]["ledger_refs"][0] + self.assertEqual((ref["row"], ref["versions"], ref["in_affected"]), ("H53", ["0.24.0"], True)) + + def stale(self, retained=""): + reg = OPEN.replace('fixed_in = ""', 'fixed_in = "1.5.0"').replace('affected = "*"', 'affected = "<1.5.0"') + reg += retained + files = dict(FILES) + files["console/opt.py"] = files["console/opt.py"].replace('"1.4.0"', '"1.5.0"') + return self.tree(reg, files) + + def test_stale_workaround_without_retained_fails(self): + self.assertFails(self.stale(), "open-bug: workaround markers remain") + + def test_stale_workaround_with_retained_passes(self): + rc, out = run_check(self.stale('retained = {reason = "the note helps older engines", pr = "#12"}\n')) + self.assertEqual(rc, 0, out) + + def test_workaround_not_stale_while_an_ours_site_is_behind(self): + reg = OPEN.replace('fixed_in = ""', 'fixed_in = "1.5.0"').replace('affected = "*"', 'affected = "<1.5.0"') + # console/opt.py stays at 1.4.0: the bump message fires, the stale rule does not. + out = self.assertFails(self.tree(reg), "bump console/opt.py to >= 1.5.0") + self.assertNotIn("workaround markers remain", out) + + def test_site_regex_without_match_fails(self): + reg = VLLM.replace("CPU_IMAGE=", "GPU_IMAGE=") + self.assertFails(self.tree(reg), "matches nothing in scripts/tool.sh") + + def test_missing_site_file_fails(self): + reg = VLLM.replace('file = "scripts/tool.sh"', 'file = "scripts/gone.sh"') + self.assertFails(self.tree(reg), "scripts/gone.sh") + + def test_unparseable_site_version_fails(self): + files = dict(FILES) + files["scripts/tool.sh"] = files["scripts/tool.sh"].replace("v0.32.0", "v0.32.x") + reg = VLLM.replace("[0-9.]+)@'}", "[0-9.x]+)@'}", 1) + self.assertFails(self.tree(reg, files), "0.32.x") + + def test_marker_in_unlisted_file_fails(self): + self.assertFails(self.tree(OPEN, extra={"docs/guide.md": "Use the fallback. \n"}), + "docs/guide.md") + + def test_listed_workaround_without_marker_fails(self): + files = dict(FILES) + files["console/opt.py"] = 'ENGINE = "1.4.0"\n' + self.assertFails(self.tree(OPEN, files), "console/opt.py has no known-issue marker for open-bug") + + def test_marker_for_unknown_issue_fails(self): + self.assertFails(self.tree(OPEN, extra={"scripts/x.sh": "# known-issue: no-such-issue\n"}), + "no-such-issue") + + def test_marker_in_ledger_file_fails(self): + reg = OPEN.replace('workarounds = ["console/opt.py"]', 'workarounds = ["console/opt.py", "bench/live.sh"]') + files = dict(FILES) + files["bench/live.sh"] += "# known-issue: open-bug\n" + self.assertFails(self.tree(reg, files), "bench/live.sh") + + def test_unpaired_block_fails(self): + files = dict(FILES) + files["console/opt.py"] = 'ENGINE = "1.4.0"\n# known-issue-begin: open-bug\nNOTE = 1\n' + self.assertFails(self.tree(OPEN, files), "known-issue-begin: open-bug") + + def test_fixed_in_inside_affected_fails(self): + self.assertFails(self.tree(VLLM.replace('affected = "<0.25.0"', 'affected = "<=0.25.0"')), + "fixed_in 0.25.0 is inside affected") + + def test_unknown_key_fails(self): + self.assertFails(self.tree(OPEN.replace('notes = ""', 'notes = ""\nfixed_by = "x"')), "fixed_by") + + def test_missing_field_fails(self): + self.assertFails(self.tree(OPEN.replace('adopt = "review"\n', "")), "adopt") + + def test_bad_enum_and_scheme_fail(self): + self.assertFails(self.tree(OPEN.replace('kind = "bug"', 'kind = "wish"')), "kind") + self.assertFails(self.tree(OPEN.replace('scheme = "semver"', 'scheme = "romver"')), "romver") + + def test_bad_upstream_fails(self): + self.assertFails(self.tree(OPEN.replace('"acme/engine#7"', '"acme/engine/7"')), "upstream") + + def test_duplicate_id_fails(self): + self.assertFails(self.tree(OPEN + OPEN), "duplicate id open-bug") + + def test_ledger_ref_row_must_exist(self): + self.assertFails(self.tree(VLLM.replace('row = "H53"', 'row = "H99"')), "H99") + + def test_site_in_ledger_file_fails(self): + reg = VLLM.replace('file = "scripts/tool.sh", regex = \'^CPU_IMAGE="vllm/vllm-openai-cpu:v(?P[0-9.]+)@\'', + 'file = "bench/live.sh", regex = \'B1_TAG=v(?P[0-9.]+)\'') + self.assertFails(self.tree(reg), "bench/live.sh") + + def test_pins_classes_file_decides_the_class(self): + # pins/classes (first match wins) makes scripts/ eat-stock: the in-range site only reports. + files = dict(FILES) + files["scripts/tool.sh"] = files["scripts/tool.sh"].replace("v0.32.0", "v0.24.0") + rc, out = run_check(self.tree(VLLM, files, {"pins/classes": "scripts/** eat-stock test\n"})) + self.assertEqual(rc, 0, out) + + def test_json_shape(self): + rc, data = run_check_json(self.tree(VLLM + OPEN)) + self.assertEqual(rc, 0) + self.assertEqual(set(data), {"ok", "errors", "reports", "issues"}) + self.assertEqual([i["id"] for i in data["issues"]], ["vllm-fix", "open-bug"]) + self.assertEqual(data["issues"][0]["status"], "fixed") + self.assertEqual(data["issues"][1]["status"], "open") + self.assertEqual(data["issues"][1]["markers"][0]["file"], "console/opt.py") + + +# --- poll ---------------------------------------------------------------------- + +SHA = "26eb87204d58d9d8a25c378fbaec9da23db3fb85" + + +def releases(*tags, pre=()): + return [{"tag_name": t, "prerelease": t in pre, "draft": False, + "html_url": "https://github.com/x/releases/tag/" + t, "published_at": "2026-07-0%dT00:00:00Z" % (i + 1)} + for i, t in enumerate(tags)] + + +class Api: + """GitHub API stub: routes path -> JSON; 404 for anything else. Records calls.""" + + def __init__(self, routes): + self.routes, self.calls = routes, [] + + def __call__(self, path): + self.calls.append(path) + if path in self.routes: + return self.routes[path] + raise ki.ApiError(404, path) + + +def vllm_routes(compare): + r = {"repos/vllm-project/vllm/releases?per_page=100&page=1": + releases("v0.26.0", "v0.25.0", "v0.25.0rc1", "v0.24.0", "v0.23.0", pre=("v0.25.0rc1",)), + "repos/vllm-project/vllm/releases?per_page=100&page=2": []} + for tag, status in compare.items(): + r["repos/vllm-project/vllm/compare/%s...%s" % (tag, SHA)] = {"status": status} + return r + + +class Poll(unittest.TestCase): + def setUp(self): + self.dirs = [] + + def tearDown(self): + for d in self.dirs: + shutil.rmtree(d, ignore_errors=True) + + def poll(self, registry, api, files=None, digests=None): + d = make_tree(registry, files) + self.dirs.append(d) + before = {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in d.rglob("*") if p.is_file()} + resolved = [] + + def digest(ref): + resolved.append(ref) + return (digests or {}).get(ref, "sha256:" + "c" * 64) + + out = ki.poll(ki.load(d), d, api, digest) + after = {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in d.rglob("*") if p.is_file()} + self.assertEqual(before, after, "poll wrote to the tree") + return {o["id"]: o for o in out}, resolved + + SEQ = {"v0.23.0": "ahead", "v0.24.0": "diverged", "v0.25.0": "behind", "v0.26.0": "behind"} + + def test_first_containing_release_walking_up(self): + # Lowest site is 0.24.0 (diverged): walk up, v0.25.0 is behind -> contains the fix. + api = Api(vllm_routes(self.SEQ)) + got, resolved = self.poll(VLLM.replace('fixed_in = "0.25.0"', 'fixed_in = ""').replace('"<0.25.0"', '"*"'), api) + o = got["vllm-fix"] + self.assertEqual((o["fixed_in"], o["tag"], o["state"]), ("0.25.0", "v0.25.0", "released")) + self.assertEqual(o["release_url"], "https://github.com/x/releases/tag/v0.25.0") + self.assertEqual(resolved, ["vllm/vllm-openai-cpu:v0.25.0"]) + self.assertFalse(any("rc1" in c for c in api.calls)) + + def test_first_containing_release_walking_down(self): + # Every site at 0.26.0 (behind): walk down past v0.25.0 (behind) to v0.24.0 (diverged). + files = dict(FILES) + files["integrations/eat/values.yaml"] = files["integrations/eat/values.yaml"].replace("v0.24.0", "v0.26.0") + files["scripts/tool.sh"] = files["scripts/tool.sh"].replace("v0.32.0", "v0.26.0") + got, _ = self.poll(VLLM, Api(vllm_routes(self.SEQ)), files) + o = got["vllm-fix"] + self.assertEqual((o["fixed_in"], o["recorded"], o["mismatch"]), ("0.25.0", "0.25.0", False)) + + def test_recorded_mismatch_is_flagged(self): + seq = dict(self.SEQ, **{"v0.25.0": "diverged"}) + got, _ = self.poll(VLLM, Api(vllm_routes(seq))) + o = got["vllm-fix"] + self.assertEqual((o["fixed_in"], o["mismatch"]), ("0.26.0", True)) + + def test_merged_but_unreleased(self): + seq = {t: "diverged" for t in self.SEQ} + got, resolved = self.poll(VLLM, Api(vllm_routes(seq))) + self.assertEqual((got["vllm-fix"]["state"], got["vllm-fix"]["fixed_in"]), ("merged-unreleased", "")) + self.assertEqual(resolved, []) + + def test_open_pr_gives_empty(self): + api = Api({"repos/acme/engine/pulls/7": {"state": "open", "merged": False, "merge_commit_sha": None}}) + got, _ = self.poll(OPEN, api) + self.assertEqual((got["open-bug"]["state"], got["open-bug"]["fixed_in"]), ("open", "")) + + def test_merged_pr_uses_its_merge_commit(self): + routes = {"repos/acme/engine/pulls/7": {"state": "closed", "merged": True, "merge_commit_sha": "f" * 40}, + "repos/acme/engine/releases?per_page=100&page=1": releases("v1.6.0", "v1.5.0", "v1.4.0"), + "repos/acme/engine/releases?per_page=100&page=2": [], + "repos/acme/engine/compare/v1.4.0..." + "f" * 40: {"status": "ahead"}, + "repos/acme/engine/compare/v1.5.0..." + "f" * 40: {"status": "identical"}} + got, _ = self.poll(OPEN, Api(routes)) + self.assertEqual((got["open-bug"]["fixed_in"], got["open-bug"]["state"]), ("1.5.0", "released")) + + def test_closed_issue_reads_the_closing_commit_from_the_timeline(self): + reg = OPEN.replace('fix = {repo = "acme/engine", pr = 7}', "fix = {}") + routes = {"repos/acme/engine/issues/7": {"state": "closed", "state_reason": "completed"}, + "repos/acme/engine/issues/7/timeline?per_page=100&page=1": [ + {"event": "labeled"}, + {"event": "closed", "commit_id": "e" * 40, + "commit_url": "https://api.github.com/repos/acme/engine/commits/" + "e" * 40}], + "repos/acme/engine/issues/7/timeline?per_page=100&page=2": [], + "repos/acme/engine/releases?per_page=100&page=1": releases("v1.5.0", "v1.4.0"), + "repos/acme/engine/releases?per_page=100&page=2": [], + "repos/acme/engine/compare/v1.4.0..." + "e" * 40: {"status": "behind"}, + "repos/acme/engine/compare/v1.3.0..." + "e" * 40: {"status": "ahead"}} + got, _ = self.poll(reg, Api(routes)) + self.assertEqual(got["open-bug"]["fixed_in"], "1.4.0") + + def test_open_issue_gives_empty(self): + reg = OPEN.replace('fix = {repo = "acme/engine", pr = 7}', "fix = {}") + got, _ = self.poll(reg, Api({"repos/acme/engine/issues/7": {"state": "open"}})) + self.assertEqual((got["open-bug"]["state"], got["open-bug"]["fixed_in"]), ("open", "")) + + def test_content_probe(self): + reg = OPEN.replace('fix = {repo = "acme/engine", pr = 7}', + 'fix = {repo = "acme/engine", path = "charts/runtimes.yaml", contains = "name: ovms"}') + + def content(text): + return {"content": base64.b64encode(text.encode()).decode(), "encoding": "base64"} + + routes = {"repos/acme/engine/releases?per_page=100&page=1": releases("v1.4.0", "v1.3.0"), + "repos/acme/engine/releases?per_page=100&page=2": [], + "repos/acme/engine/contents/charts/runtimes.yaml?ref=v1.4.0": content("name: sklearn\n"), + "repos/acme/engine/contents/charts/runtimes.yaml": content("name: sklearn\n")} + got, _ = self.poll(reg, Api(routes)) + self.assertEqual((got["open-bug"]["state"], got["open-bug"]["fixed_in"]), ("not-shipped", "")) + routes["repos/acme/engine/contents/charts/runtimes.yaml"] = content("name: ovms\n") + got, _ = self.poll(reg, Api(routes)) + self.assertEqual((got["open-bug"]["state"], got["open-bug"]["fixed_in"]), ("merged-unreleased", "")) + routes["repos/acme/engine/releases?per_page=100&page=1"] = releases("v1.5.0", "v1.4.0", "v1.3.0") + routes["repos/acme/engine/contents/charts/runtimes.yaml?ref=v1.5.0"] = content("name: ovms\n") + got, _ = self.poll(reg, Api(routes)) + self.assertEqual((got["open-bug"]["state"], got["open-bug"]["fixed_in"]), ("released", "1.5.0")) + + ADVISORY_ENTRY = textwrap.dedent(""" + [[issue]] + id = "lib-advisories" + title = "advisories" + kind = "advisory" + upstream = "" + fix = {ecosystem = "pip", package = "lib"} + component = "lib" + scheme = "pep440" + affected = "<1.5.0" + fixed_in = "1.5.0" + adopt = "review" + advisories = ["GHSA-aaaa-aaaa-aaaa"] + sites = [{file = "req.txt", regex = '^lib==(?P\\S+)$'}] + workarounds = [] + notes = "" + """) + + @staticmethod + def adv(ghsa, *vulns): + return {"ghsa_id": ghsa, "html_url": "https://github.com/advisories/" + ghsa, + "vulnerabilities": [{"package": {"ecosystem": "pip", "name": "lib"}, + "vulnerable_version_range": r, "first_patched_version": p} for r, p in vulns]} + + def test_advisories(self): + reg, adv = self.ADVISORY_ENTRY, self.adv + routes = {"advisories?ecosystem=pip&affects=lib%401.2.0&per_page=100": [ + adv("GHSA-aaaa-aaaa-aaaa", ("< 1.5.0", "1.5.0")), + adv("GHSA-bbbb-bbbb-bbbb", ("< 1.6.2", "1.6.2"), (">= 1.7.0, < 1.7.3", "1.7.3")), + ], "advisories?ecosystem=pip&affects=lib%401.6.2&per_page=100": []} + got, _ = self.poll(reg, Api(routes), files={"req.txt": "lib==1.2.0\n"}) + o = got["lib-advisories"] + self.assertEqual((o["state"], o["fixed_in"], o["mismatch"]), ("advisory", "1.6.2", True)) + self.assertEqual(o["new_advisories"], ["GHSA-bbbb-bbbb-bbbb"]) + + def test_advisories_of_the_candidate_count_and_versions_below_the_floor_do_not(self): + # The floor's advisories put the fix at 1.5.0 (1.1.5 is a backport below the floor, not a + # fix for it). 1.5.0 has an advisory of its own that 1.2.0 does not, so the fix is 1.6.0. + reg = self.ADVISORY_ENTRY + routes = {"advisories?ecosystem=pip&affects=lib%401.2.0&per_page=100": [ + self.adv("GHSA-aaaa-aaaa-aaaa", ("< 1.1.5", "1.1.5"), (">= 1.2.0, < 1.5.0", "1.5.0")), + ], "advisories?ecosystem=pip&affects=lib%401.5.0&per_page=100": [ + self.adv("GHSA-cccc-cccc-cccc", (">= 1.4.0, < 1.6.0", "1.6.0")), + ], "advisories?ecosystem=pip&affects=lib%401.6.0&per_page=100": []} + got, _ = self.poll(reg, Api(routes), files={"req.txt": "lib==1.2.0\n"}) + o = got["lib-advisories"] + self.assertEqual((o["state"], o["fixed_in"], o["mismatch"]), ("advisory", "1.6.0", True)) + self.assertEqual(o["advisories"], ["GHSA-aaaa-aaaa-aaaa", "GHSA-cccc-cccc-cccc"]) + self.assertEqual(o["new_advisories"], ["GHSA-cccc-cccc-cccc"]) + + def test_a_prerelease_patch_is_not_a_fix(self): + routes = {"advisories?ecosystem=pip&affects=lib%401.2.0&per_page=100": [ + self.adv("GHSA-aaaa-aaaa-aaaa", ("< 1.5.0rc2", "1.5.0rc2")), + ]} + got, _ = self.poll(self.ADVISORY_ENTRY, Api(routes), files={"req.txt": "lib==1.2.0\n"}) + o = got["lib-advisories"] + self.assertNotIn("error", o) + self.assertEqual((o["state"], o["fixed_in"]), ("advisory", "")) + + def test_eat_entry_sees_a_plain_release_after_a_hotfix(self): + # The registry's eat-9 entry, on EAT's real tag stream: release-1.0.3-hotfix is followed by + # release-1.1.0 and release-1.2.0, and the fix lands in release-1.2.0. + with open(ROOT / "pins" / "known-issues.toml", "rb") as f: + entry = next(e for e in tomllib.load(f)["issue"] if e["id"] == "eat-9") + reg = textwrap.dedent(""" + [[issue]] + id = "eat-9" + title = "t" + kind = "bug" + upstream = "acme/toolkit#9" + fix = {} + component = "enterprise-agent-toolkit" + scheme = "%s" + affected = "*" + fixed_in = "" + adopt = "review" + sites = [{file = "eat-up.sh", regex = '%s'}] + workarounds = [] + notes = "" + """) % (entry["scheme"], entry["sites"][0]["regex"]) + sha = "d" * 40 + routes = {"repos/acme/toolkit/issues/9": {"state": "closed", "state_reason": "completed"}, + "repos/acme/toolkit/issues/9/timeline?per_page=100&page=1": [ + {"event": "closed", "commit_id": sha, + "commit_url": "https://api.github.com/repos/acme/toolkit/commits/" + sha}], + "repos/acme/toolkit/releases?per_page=100&page=1": releases( + "release-1.2.0", "release-1.1.0", "release-1.0.3-hotfix", "release-1.0.2-hotfix", + "release-1.0.0", pre=("release-1.0.3-hotfix",)), + "repos/acme/toolkit/compare/release-1.1.0..." + sha: {"status": "diverged"}, + "repos/acme/toolkit/compare/release-1.2.0..." + sha: {"status": "behind"}} + got, _ = self.poll(reg, Api(routes), files={"eat-up.sh": 'EAT_TAG="release-1.0.3-hotfix"\n'}) + o = got["eat-9"] + self.assertEqual((o["state"], o["fixed_in"]), ("released", "release-1.2.0"), o) + + def test_untracked_entry(self): + reg = OPEN.replace('upstream = "acme/engine#7"', 'upstream = ""').replace( + 'fix = {repo = "acme/engine", pr = 7}', "fix = {}") + got, _ = self.poll(reg, Api({})) + self.assertEqual((got["open-bug"]["state"], got["open-bug"]["fixed_in"]), ("untracked", "")) + + def test_api_error_is_reported_per_entry(self): + got, _ = self.poll(VLLM + OPEN, Api({"repos/acme/engine/pulls/7": {"state": "open", "merged": False}})) + self.assertIn("error", got["vllm-fix"]) + self.assertEqual(got["open-bug"]["state"], "open") + + +# --- this repo ----------------------------------------------------------------- + +class RealRepo(unittest.TestCase): + def test_check_passes_on_this_repo(self): + p = subprocess.run([sys.executable, str(TOOL), "--root", str(ROOT), "check"], capture_output=True, text=True) + self.assertEqual(p.returncode, 0, p.stdout + p.stderr) + + def test_registry_seeds_the_known_items(self): + with open(ROOT / "pins" / "known-issues.toml", "rb") as f: + ids = {e["id"]: e for e in tomllib.load(f)["issue"]} + for want in ("vllm-45844", "vllm-50801", "vllm-pr-51493", "linux-npu-driver-56", "kserve-4592", + "eat-9", "litellm-1.83.14-advisories", "vllm-0.32-num-spec-tokens"): + self.assertIn(want, ids) + self.assertEqual((ids["vllm-45844"]["affected"], ids["vllm-45844"]["fixed_in"]), ("<0.25.0", "0.25.0")) + self.assertEqual(ids["kserve-4592"]["fixed_in"], "") + self.assertEqual(ids["vllm-pr-51493"]["fixed_in"], "") + + def test_moving_the_ours_vllm_cpu_pin_to_0_24_fails_with_the_bump_message(self): + """Copy the registry and every file it names into a fixture; move AMX_TEST_IMAGE to v0.24.0.""" + d = pathlib.Path(tempfile.mkdtemp(prefix="known_issues_real.")) + try: + with open(ROOT / "pins" / "known-issues.toml", "rb") as f: + reg = tomllib.load(f) + paths = {"pins/known-issues.toml", "EXPERIMENTS.md"} + for e in reg["issue"]: + paths |= {s["file"] for s in e["sites"]} | set(e["workarounds"]) + paths |= {r["path"] for r in e.get("ledger_refs", [])} + if (ROOT / "pins" / "classes").is_file(): + paths.add("pins/classes") + for rel in paths: + (d / rel).parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(ROOT / rel, d / rel) + rc, out = run_check(d) + self.assertEqual(rc, 0, out) + k = d / "scripts" / "k3d-gpu" + text = k.read_text() + moved = re.sub(r"(K3D_GPU_AMX_TEST_IMAGE:-vllm/vllm-openai-cpu:)v\d+\.\d+\.\d+@", r"\1v0.24.0@", text) + self.assertNotEqual(text, moved) + k.write_text(moved) + rc, out = run_check(d) + self.assertEqual(rc, 1, out) + self.assertIn("bump scripts/k3d-gpu to >= 0.25.0", out) + finally: + shutil.rmtree(d, ignore_errors=True) + + +if __name__ == "__main__": + unittest.main(verbosity=1) diff --git a/tests/ledger_freshness_test.py b/tests/ledger_freshness_test.py new file mode 100755 index 00000000..2e8a560a --- /dev/null +++ b/tests/ledger_freshness_test.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python3 +"""Tests for hack/ledger-freshness. Each case writes a throwaway repo (ledger, run +sections, images.lock, ledger-rules, a cited doc, a known-issues registry) and +reads the re-measure queue; the last cases run it on this repo and check that +EXPERIMENTS.md and console/ledger-rules.json are byte-identical afterwards. + +Usage: python3 tests/ledger_freshness_test.py +""" + +import hashlib +import importlib.machinery +import importlib.util +import json +import pathlib +import shutil +import subprocess +import sys +import tempfile +import unittest + +ROOT = pathlib.Path(__file__).resolve().parent.parent +TOOL = ROOT / "hack" / "ledger-freshness" + +LEDGER = """# Experiments + +## Hypotheses + +| # | Hypothesis | Prediction | Probe | Verdict | +|---|------------|------------|-------|---------| +| H34 | ovms draft | p | Pre-registered. Result ([H34 result](#h34-result-host-2026-10-17)). | SUPPORTED | +| H57dm | llama mtp | p | Run 2026-10-21 ([H57d run](#h57d-run-host-2026-10-21)). | SUPPORTED | +| H60 | ovms older | p | Run ([H60 run](#h60-run-host-2026-10-22)) \\| piped cell. | SUPPORTED | +| H61 | vllm cpu | p | Pre-registration ([H61 pre-registration](#h61-pre-registration)); run ([H61 run](#h61-run-host-2026-10-23)). | SUPPORTED | +| H62 | no record | p | probe only | SUPPORTED | +| H63 | pending | p | ([H63 run](#h63-run-host)) | PENDING | +| H64 | refuted | p | ([H64 run](#h64-run-host)) | REFUTED | + +## Reproduce + +### H34 result (host, 2026-10-17) + +**Reproduce:** commit `bb382c2`, node image `cryptoandcoffee/k3d-gpu:0.1.55`, OVMS +2026.4.1 (`openvino/model_server@sha256:a928903f...76fc58`), models from `~/m`. + +### H57d run (host, 2026-10-21) + +**Reproduce:** + +```text +# provenance +k3d-gpu commit: 458bca0e, node image + cryptoandcoffee/k3d-gpu:0.1.63@sha256:b6fc +runtimes: ghcr.io/ggml-org/llama.cpp:server-b11749@sha256:9fba; + openvino/model_server@sha256:a928903fe429f1e3 (2026.4.1) +models: google/gemma-4-26B-A4B-it-qat-q4_0-gguf @ d1c082be (gemma.gguf sha256 3eca3b8f...); + OpenVINO/gemma-4-26b-a4b-it-int4-ov @ 4e5885c3 +records: bench/h57d/results/ +``` + +### H60 run (host, 2026-10-22) + +```text +# Reproduce +runtime: openvino/model_server:2026.3-gpu@sha256:aaaa +``` + +### H61 pre-registration + +Arms on vllm/vllm-openai-cpu:v0.20.0 (not a run; ignored). + +### H61 run (host, 2026-10-23) + +```text +# Reproduce +runtime: vllm/vllm-openai-cpu:v0.24.0@sha256:b0e6 +node image: cryptoandcoffee/k3d-gpu:0.1.70 +``` + +### H64 run (host) + +```text +# Reproduce +runtime: openvino/model_server:2025.1 +``` +""" + +LOCK = {"images": [ + {"name": "docker.io/openvino/model_server", "tag": "2026.4.1-gpu", "digest": "sha256:a928903fe429f1e3" + "0" * 48, + "sources": ["scripts/k3d-gpu NPU_UMD_IMAGE"]}, + {"name": "docker.io/openvino/model_server", "digest": "sha256:a928903fe429f1e3" + "0" * 48, + "sources": ["scripts/k3d-gpu AGENT_OVMS_IMAGE"]}, + {"name": "ghcr.io/ggml-org/llama.cpp", "tag": "server-b11960", "digest": "sha256:" + "1" * 64, + "sources": ["scripts/k3d-gpu K3D_GPU_LLAMA_CPU_IMAGE"]}, + {"name": "docker.io/vllm/vllm-openai-cpu", "tag": "v0.32.0", "digest": "sha256:" + "2" * 64, + "sources": ["scripts/k3d-gpu K3D_GPU_AMX_TEST_IMAGE"]}, + {"name": "docker.io/vllm/vllm-openai-cpu", "tag": "v0.24.0", "digest": "sha256:" + "3" * 64, + "sources": ["integrations/eat/vllm-values.yaml"]}, +]} + +RULES = {"rows": [{"id": "H57dm", "verdict": "SUPPORTED", "classes": ["i9-13900t"]}, + {"id": "H34", "verdict": "SUPPORTED", "classes": []}]} + +KNOWN = """ +[[issue]] +id = "vllm-fix" +title = "a CPU attention bug" +kind = "bug" +upstream = "" +component = "vllm-openai-cpu" +scheme = "zerover" +affected = "<0.25.0 || >=0.30.0,<0.30.1" +fixed_in = "0.25.0" +adopt = "review" +sites = [{file = "scripts/k3d-gpu", regex = 'vllm-openai-cpu:v(?P[0-9.]+)@'}] +workarounds = [] +notes = "" +""" + +FILES = { + "EXPERIMENTS.md": LEDGER, + "images.lock": json.dumps(LOCK), + "console/ledger-rules.json": json.dumps(RULES), + "docs/perf.md": "OVMS decodes 12 tok/s here [ledger:H60].\n", + "README.md": "k3d-gpu\n", + "scripts/k3d-gpu": 'VERSION="0.1.79"\nAMX="vllm/vllm-openai-cpu:v0.32.0@sha256:x"\n', + "pins/known-issues.toml": KNOWN, +} + + +def run(root, *args): + p = subprocess.run([sys.executable, str(TOOL), "--root", str(root), *args], capture_output=True, text=True) + return p.returncode, p.stdout, p.stderr + + +class Fixture(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.root = pathlib.Path(tempfile.mkdtemp(prefix="ledger_freshness.")) + for rel, text in FILES.items(): + p = cls.root / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(text) + rc, out, err = run(cls.root, "--json") + assert rc == 0, out + err + cls.data = json.loads(out) + cls.queue = {q["row"]: q for q in cls.data["queue"]} + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.root, ignore_errors=True) + + def test_measured_versions(self): + m = self.data["measured"] + self.assertEqual(m["H34"]["openvino/model_server"], "2026.4.1") + self.assertEqual(m["H34"]["cryptoandcoffee/k3d-gpu"], "0.1.55") + self.assertEqual(m["H57dm"]["llama.cpp"], "b11749") + self.assertEqual(m["H57dm"]["openvino/model_server"], "2026.4.1") + self.assertEqual(m["H57dm"]["hf:google/gemma-4-26B-A4B-it-qat-q4_0-gguf"], "d1c082be") + self.assertEqual(m["H57dm"]["hf:OpenVINO/gemma-4-26b-a4b-it-int4-ov"], "4e5885c3") + self.assertEqual(m["H60"]["openvino/model_server"], "2026.3") + self.assertEqual(m["H61"]["vllm-openai-cpu"], "0.24.0") + + def test_only_supported_rows(self): + self.assertNotIn("H63", self.data["measured"]) + self.assertNotIn("H64", self.data["measured"]) + + def test_pre_registration_anchor_is_not_a_run(self): + self.assertNotIn("0.20.0", json.dumps(self.data["measured"]["H61"])) + + def test_current_versions_from_ours_pins(self): + c = self.data["current"] + self.assertEqual(c["openvino/model_server"], "2026.4.1") + self.assertEqual(c["llama.cpp"], "b11960") + self.assertEqual(c["vllm-openai-cpu"], "0.32.0") # the eat-stock v0.24.0 is not ours + self.assertEqual(c["cryptoandcoffee/k3d-gpu"], "0.1.79") + + def test_ovms_2026_3_against_2026_4_1_is_stale(self): + q = self.queue["H60"] + self.assertEqual((q["measured"]["openvino/model_server"], q["current"]["openvino/model_server"]), + ("2026.3", "2026.4.1")) + self.assertIn("minor", q["reason"]) + self.assertFalse(q["priority"]) + + def test_same_minor_is_not_stale(self): + # H34: OVMS 2026.4.1 = current; node image 0.1.55 -> 0.1.79 is a patch. + self.assertNotIn("H34", self.queue) + + def test_llama_build_behind_is_stale(self): + self.assertIn("llama.cpp b11749 -> b11960", self.queue["H57dm"]["reason"]) + + def test_measured_inside_a_known_issue_is_priority_stale(self): + q = self.queue["H61"] + self.assertTrue(q["priority"]) + self.assertEqual(q["known_issues"], ["vllm-fix"]) + self.assertIn("priority-stale", q["reason"]) + + def test_unparseable_row_is_listed(self): + rows = {r["row"]: r["reason"] for r in self.data["not_machine_readable"]} + self.assertIn("H62", rows) + self.assertNotIn("H34", rows) + + def test_queue_order(self): + # ledger-rules rows with classes, then rows docs claims cite, then the rest. + self.assertEqual([q["row"] for q in self.data["queue"]], ["H57dm", "H60", "H61"]) + self.assertEqual([q["tier"] for q in self.data["queue"]], [1, 2, 3]) + + def test_suggested_followup_ids_are_free(self): + rows = set(self.data["measured"]) | {"H63", "H64", "H62"} + for q in self.data["queue"]: + self.assertRegex(q["suggested_followup_id"], r"^H\d+[a-z]{1,2}$") + self.assertNotIn(q["suggested_followup_id"], rows) + self.assertEqual(self.queue["H60"]["suggested_followup_id"], "H60r") + + def test_two_queued_rows_of_one_family_get_different_followup_ids(self): + # H57dm and H57de both suggest H57r first; the second must take the next free id. + loader = importlib.machinery.SourceFileLoader("ledger_freshness", str(TOOL)) + spec = importlib.util.spec_from_loader("ledger_freshness", loader) + mod = importlib.util.module_from_spec(spec) + loader.exec_module(mod) + measured = {"H57dm": {"llama.cpp": "b11749"}, "H57de": {"llama.cpp": "b11749"}} + got = [q["suggested_followup_id"] for q in mod.queue(self.root, measured, {"llama.cpp": "b11960"}, [])] + self.assertEqual(len(got), 2) + self.assertEqual(len(set(got)), 2, got) + + def test_markdown(self): + rc, out, err = run(self.root, "--md") + self.assertEqual(rc, 0, err) + for needle in ("| 1 | H57dm |", "b11749", "| H34 |", "openvino/model_server 2026.4.1", "H62", "priority-stale"): + self.assertIn(needle, out) + # A pipe inside a cell (a range union) is escaped, so every queue row keeps 5 cells. + self.assertIn("<0.25.0 \\|\\| >=0.30.0", out) + for line in out.splitlines(): + if line.startswith("| 3 |"): + self.assertEqual(len(line.replace("\\|", "").split("|")), 7, line) + + def test_needs_an_output_mode(self): + rc, _, _ = run(self.root) + self.assertEqual(rc, 2) + + +def digest(paths): + return {p: hashlib.sha256((ROOT / p).read_bytes()).hexdigest() for p in paths} + + +class RealRepo(unittest.TestCase): + PROTECTED = ("EXPERIMENTS.md", "console/ledger-rules.json") + + def test_runs_read_only_and_lists_the_supported_engine_rows(self): + before = digest(self.PROTECTED) + rc, out, err = run(ROOT, "--json") + self.assertEqual(rc, 0, err) + rc2, md, err2 = run(ROOT, "--md") + self.assertEqual(rc2, 0, err2) + self.assertEqual(before, digest(self.PROTECTED)) + data = json.loads(out) + # The ledger's recorded versions, read by shape: a pin bump substitutes version strings in tests/. + self.assertRegex(data["measured"]["H34"]["openvino/model_server"], r"^\d{4}\.\d+(\.\d+)?$") + self.assertRegex(data["measured"]["H57dm"]["llama.cpp"], r"^b\d+$") + self.assertIn("| H34 |", md) + self.assertIn("| H57dm |", md) + + +if __name__ == "__main__": + unittest.main(verbosity=1) diff --git a/tests/pinver_test.py b/tests/pinver_test.py new file mode 100755 index 00000000..32d2b16d --- /dev/null +++ b/tests/pinver_test.py @@ -0,0 +1,324 @@ +#!/usr/bin/env python3 +"""Tests for hack/pinver.py: every versioning scheme, ranges, update types, +prerelease exclusion and the CLI. + +Usage: python3 tests/pinver_test.py +""" + +import pathlib +import subprocess +import sys +import unittest + +ROOT = pathlib.Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "hack")) +import pinver # noqa: E402 + + +class Semver(unittest.TestCase): + def test_v_prefix_is_optional(self): + self.assertEqual(pinver.compare("semver", "v1.2.3", "1.2.3"), 0) + + def test_numeric_order(self): + self.assertEqual(pinver.compare("semver", "1.9.0", "1.10.0"), -1) + self.assertEqual(pinver.compare("semver", "2.0.0", "1.99.99"), 1) + + def test_prerelease_sorts_before_release(self): + self.assertEqual(pinver.compare("semver", "1.2.3-rc.1", "1.2.3"), -1) + self.assertEqual(pinver.compare("semver", "1.2.3-rc.2", "1.2.3-rc.10"), -1) + self.assertEqual(pinver.compare("semver", "1.2.3-alpha", "1.2.3-alpha.1"), -1) + + def test_prerelease_flag(self): + self.assertTrue(pinver.is_prerelease("semver", "v1.2.3-rc1")) + self.assertFalse(pinver.is_prerelease("semver", "v1.2.3")) + + def test_build_metadata_ignored_in_order(self): + self.assertEqual(pinver.compare("semver", "1.2.3+a", "1.2.3+b"), 0) + + def test_update_types(self): + self.assertEqual(pinver.update_type("semver", "1.2.3", "1.2.4"), "patch") + self.assertEqual(pinver.update_type("semver", "1.2.3", "1.3.0"), "minor") + self.assertEqual(pinver.update_type("semver", "1.2.3", "2.0.0"), "major") + self.assertEqual(pinver.update_type("semver", "v1.2.3", "1.2.3"), "digest") + + def test_breaking_is_major_only(self): + self.assertFalse(pinver.breaking("semver", "1.2.3", "1.3.0")) + self.assertTrue(pinver.breaking("semver", "1.2.3", "2.0.0")) + # Under plain semver a 0.x minor is not breaking; zerover says it is. + self.assertFalse(pinver.breaking("semver", "0.24.0", "0.25.0")) + + def test_garbage_is_rejected(self): + for bad in ("", "latest", "1.2.x", "v"): + with self.assertRaises(ValueError, msg=bad): + pinver.parse("semver", bad) + + +class Zerover(unittest.TestCase): + def test_zerover_minor_is_minor_but_breaking(self): + self.assertEqual(pinver.update_type("zerover", "v0.24.0", "v0.25.0"), "minor") + self.assertTrue(pinver.breaking("zerover", "v0.24.0", "v0.25.0")) + self.assertEqual(pinver.classify("zerover", "v0.24.0", "v0.25.0"), + {"type": "minor", "breaking": True}) + + def test_zerover_patch_is_not_breaking(self): + self.assertEqual(pinver.update_type("zerover", "v0.25.0", "v0.25.1"), "patch") + self.assertFalse(pinver.breaking("zerover", "v0.25.0", "v0.25.1")) + + def test_zerover_past_one_is_semver(self): + self.assertFalse(pinver.breaking("zerover", "1.2.0", "1.3.0")) + self.assertTrue(pinver.breaking("zerover", "0.32.0", "1.0.0")) + + def test_vllm_fixed_in(self): + self.assertTrue(pinver.in_range("zerover", "v0.24.0", "<0.25.0")) + self.assertFalse(pinver.in_range("zerover", "v0.25.0", "<0.25.0")) + self.assertFalse(pinver.in_range("zerover", "v0.32.0", "<0.25.0")) + + +class K8s(unittest.TestCase): + def test_plus_and_dash_k3s_forms(self): + self.assertEqual(pinver.compare("k8s", "v1.37.1+k3s1", "v1.37.1-k3s1"), 0) + self.assertEqual(pinver.compare("k8s", "v1.37.1+k3s1", "v1.37.1+k3s2"), -1) + self.assertEqual(pinver.compare("k8s", "v1.37.1+k3s9", "v1.37.2+k3s1"), -1) + + def test_k8s_minor_is_breaking(self): + self.assertEqual(pinver.update_type("k8s", "v1.37.1+k3s1", "v1.38.0+k3s1"), "minor") + self.assertTrue(pinver.breaking("k8s", "v1.37.1+k3s1", "v1.38.0+k3s1")) + + def test_k8s_patch_and_distro_build(self): + self.assertEqual(pinver.update_type("k8s", "v1.37.1+k3s1", "v1.37.2+k3s1"), "patch") + self.assertEqual(pinver.update_type("k8s", "v1.37.1+k3s1", "v1.37.1+k3s2"), "patch") + self.assertFalse(pinver.breaking("k8s", "v1.37.1+k3s1", "v1.37.2+k3s1")) + + def test_k8s_rc_is_prerelease(self): + self.assertTrue(pinver.is_prerelease("k8s", "v1.38.0-rc1+k3s1")) + self.assertEqual(pinver.compare("k8s", "v1.38.0-rc1+k3s1", "v1.38.0+k3s1"), -1) + + +class Llama(unittest.TestCase): + def test_build_numbers_compare_numerically(self): + self.assertEqual(pinver.compare("llama", "b9999", "b11749"), -1) + self.assertEqual(pinver.compare("llama", "server-b11960", "b11960"), 0) + self.assertEqual(pinver.compare("llama", "b11749", "b11960"), -1) + + def test_build_bump_is_minor_and_not_breaking(self): + self.assertEqual(pinver.update_type("llama", "b11749", "b11960"), "minor") + self.assertFalse(pinver.breaking("llama", "b11749", "b11960")) + + def test_releases_are_zerover(self): + self.assertEqual(pinver.compare("llama", "v0.7.0", "v0.8.0"), -1) + self.assertTrue(pinver.breaking("llama", "v0.7.0", "v0.8.0")) + + def test_build_and_release_are_incomparable(self): + self.assertFalse(pinver.comparable("llama", "b11960", "v0.8.0")) + with self.assertRaises(ValueError): + pinver.compare("llama", "b11960", "v0.8.0") + + def test_build_range(self): + self.assertTrue(pinver.in_range("llama", "b11749", "=2.9.0,<2.10")) + + def test_prerelease_flag(self): + for v in ("1.0a1", "1.0rc1", "1.0.dev3"): + self.assertTrue(pinver.is_prerelease("pep440", v), v) + for v in ("1.0", "1.0.post1", "1.0+cpu"): + self.assertFalse(pinver.is_prerelease("pep440", v), v) + + def test_update_types(self): + self.assertEqual(pinver.update_type("pep440", "1.83.14", "1.84.0"), "minor") + self.assertEqual(pinver.update_type("pep440", "1.83.14", "1.83.14.post1"), "patch") + self.assertEqual(pinver.update_type("pep440", "1.83.14", "2.0"), "major") + self.assertEqual(pinver.update_type("pep440", "1.83.14", "1.83.14"), "digest") + + +class DockerSuffix(unittest.TestCase): + def test_core_order_within_one_flavour(self): + self.assertEqual(pinver.compare("docker-suffix", "3.14.8-slim", "3.14.10-slim"), -1) + self.assertEqual(pinver.compare("docker-suffix", "contrib-v1.40.0", "contrib-v1.41.0"), -1) + + def test_different_suffix_or_prefix_is_incomparable(self): + self.assertFalse(pinver.comparable("docker-suffix", "3.14.8-slim", "3.14.9-alpine")) + self.assertFalse(pinver.comparable("docker-suffix", "contrib-v1.40.0", "v1.41.0")) + with self.assertRaises(ValueError): + pinver.update_type("docker-suffix", "3.14.8-slim", "3.14.9-alpine") + + def test_latest_ignores_a_different_suffix(self): + got = pinver.latest("docker-suffix", "3.14.8-slim", + ["3.14.9-slim", "3.15.0-alpine", "3.15.0-slim-bookworm", "3.14.7-slim"]) + self.assertEqual(got, "3.14.9-slim") + + def test_update_types_follow_the_core(self): + self.assertEqual(pinver.update_type("docker-suffix", "3.14.8-slim", "3.15.0-slim"), "minor") + self.assertEqual(pinver.update_type("docker-suffix", "release-1.0.3-hotfix", "release-1.0.4-hotfix"), "patch") + + def test_range_reads_the_core(self): + self.assertTrue(pinver.in_range("docker-suffix", "v1.83.14-stable.patch.2", "<1.84.0")) + self.assertFalse(pinver.in_range("docker-suffix", "v1.88.6-stable", "<1.84.0")) + + +class Eat(unittest.TestCase): + """Intel EAT's release tags: one stream of release-X.Y.Z and release-X.Y.Z-hotfix.""" + + def test_hotfix_and_plain_releases_are_one_stream(self): + self.assertTrue(pinver.comparable("eat", "release-1.0.3-hotfix", "release-1.1.0")) + self.assertEqual(pinver.compare("eat", "release-1.0.3-hotfix", "release-1.1.0"), -1) + self.assertEqual(pinver.compare("eat", "release-1.0.2-hotfix", "release-1.0.3-hotfix"), -1) + self.assertEqual(pinver.compare("eat", "release-1.0.3", "release-1.0.3-hotfix"), -1) + + def test_latest_moves_from_a_hotfix_to_the_next_release(self): + got = pinver.latest("eat", "release-1.0.3-hotfix", + ["release-1.0.2-hotfix", "release-1.1.0", "release-1.0.0", "1.2.0", "latest"]) + self.assertEqual(got, "release-1.1.0") + + def test_update_types_and_ranges(self): + self.assertEqual(pinver.update_type("eat", "release-1.0.3-hotfix", "release-1.1.0"), "minor") + self.assertFalse(pinver.breaking("eat", "release-1.0.3-hotfix", "release-1.1.0")) + self.assertTrue(pinver.breaking("eat", "release-1.1.0", "release-2.0.0")) + self.assertTrue(pinver.in_range("eat", "release-1.0.3-hotfix", "1.2.3")) + + def test_bad_specs(self): + for bad in ("", "~1.2", "<", "<1.2.3,,>1", "1.2.3"): + with self.assertRaises(ValueError, msg=bad): + pinver.in_range("semver", "1.2.3", bad) + + def test_check_range_reports_syntax_only(self): + pinver.check_range("zerover", "<0.25.0 || >=0.26.0,<0.26.1") + with self.assertRaises(ValueError): + pinver.check_range("zerover", "<0.25.x") + + +class Latest(unittest.TestCase): + def test_prerelease_excluded(self): + got = pinver.latest("zerover", "v0.24.0", ["v0.25.0", "v0.26.0rc1", "v0.26.0-rc.1", "v0.24.1"]) + self.assertEqual(got, "v0.25.0") + + def test_never_backwards(self): + self.assertIsNone(pinver.latest("zerover", "v0.32.0", ["v0.25.0", "v0.31.0"])) + + def test_unparseable_candidates_skipped(self): + self.assertEqual(pinver.latest("calver", "2026.4.1", ["latest", "2026.5.0", "nightly"]), "2026.5.0") + + +class UnknownScheme(unittest.TestCase): + def test_unknown_scheme(self): + with self.assertRaises(ValueError): + pinver.parse("romver", "1.0") + + +class Cli(unittest.TestCase): + def run_cli(self, *args): + return subprocess.run([sys.executable, str(ROOT / "hack" / "pinver.py"), *args], + capture_output=True, text=True) + + def test_compare(self): + p = self.run_cli("compare", "zerover", "v0.24.0", "0.25.0") + self.assertEqual((p.returncode, p.stdout.strip()), (0, "-1")) + + def test_in_range_exit_code(self): + self.assertEqual(self.run_cli("in-range", "zerover", "v0.24.0", "<0.25.0").returncode, 0) + self.assertEqual(self.run_cli("in-range", "zerover", "v0.32.0", "<0.25.0").returncode, 1) + + def test_update_type(self): + p = self.run_cli("update-type", "zerover", "v0.24.0", "v0.25.0") + self.assertEqual(p.stdout.strip(), "minor breaking") + + def test_bad_input_exit_2(self): + p = self.run_cli("compare", "semver", "x", "1.0.0") + self.assertEqual(p.returncode, 2) + self.assertIn("semver", p.stderr) + + +if __name__ == "__main__": + unittest.main(verbosity=1)