diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90f4bde3..636c5dcc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -462,3 +462,23 @@ jobs: sudo tar -xzf kubeconform.tar.gz -C /usr/local/bin kubeconform sudo apt-get update && sudo apt-get install -y --no-install-recommends python3-yaml - run: tests/kserve_test.sh + + # --- Freshness: known issues --------------------------------------------------- + # pins/known-issues.toml lists the upstream bugs, behaviour changes and advisories + # k3d-gpu depends on, each with the release that fixes it. Offline: fails when an + # "ours" pin is inside a fixed issue's affected range, when a workaround marker + # outlives its fix, or when markers and entries disagree. `hack/known-issues poll` + # (network) finds fixed-in releases; hack/ledger-freshness lists SUPPORTED ledger + # rows measured on versions older than today's pins. + known-issues: + name: Known issues (fixed-in rules, markers), version schemes, ledger re-measure queue + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - run: python3 tests/pinver_test.py + - run: python3 tests/known_issues_test.py + - run: python3 tests/ledger_freshness_test.py + - run: hack/known-issues check diff --git a/console/audit.py b/console/audit.py index d047da08..3232aa2f 100644 --- a/console/audit.py +++ b/console/audit.py @@ -84,6 +84,7 @@ def endpoint(address, port): return "%s:%s" % (host, port if port is not None else "(port not set)") +# known-issue: eat-9 (the rule is general; EAT#9 is the instance that motivated it) def check_listener(ls): subject = "%s %s" % (endpoint(ls["address"], ls["port"]), ls["name"]) where = "loopback" if is_loopback(ls["address"]) else "off loopback" diff --git a/console/model_opt.py b/console/model_opt.py index b6903bfa..0776fe08 100644 --- a/console/model_opt.py +++ b/console/model_opt.py @@ -1104,6 +1104,7 @@ def spec_decide(doc, choose, engine, device, pick, drafters, target, facts, draf for d in fit: if d["drafter"] == "assistant": if facts and facts["moe"]: + # known-issue: vllm-pr-51493 (the VLLM_USE_V2_MODEL_RUNNER=1 clause goes once a release has the fix) doc["notes"].append("MoE target: Google says the 26B A4B drafter may not speed up at batch 1 without good parallelism (https://ai.google.dev/gemma/docs/mtp/overview); " "if vLLM's V1 runner fails on it, VLLM_USE_V2_MODEL_RUNNER=1 (https://github.com/vllm-project/vllm/pull/51493); K3D_GPU_AGENT_DRAFT=off turns the drafter off") cfg = {"method": "mtp", "model": d["id"], "num_speculative_tokens": 4} diff --git a/docs/first-run.md b/docs/first-run.md index 84e0560c..3fb78fca 100644 --- a/docs/first-run.md +++ b/docs/first-run.md @@ -72,7 +72,7 @@ What differs on WSL2: agent does not request the WSL2 GPU yet. quickstart says so at step 1. - The NPU is not available under WSL2 ([intel/linux-npu-driver#56](https://github.com/intel/linux-npu-driver/issues/56)), - so `recipe npu-embeddings` cannot run here. + so `recipe npu-embeddings` cannot run here. - WSL2 hides the P/E-core split; no P-core pinning applies. - The WSL2 VM gets half the Windows RAM by default. A 32 GB laptop shows about 15.4 GiB and gets the 4B tier. Raise `memory=` in `%UserProfile%\.wslconfig` diff --git a/docs/quickstart.md b/docs/quickstart.md index 4b2d22e3..16869603 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -152,7 +152,7 @@ the LLM server does not request it yet. ([Docker docs](https://docs.docker.com/desktop/features/gpu/)). The Intel path does not use that flag. - **The Intel NPU does not work under WSL2.** Intel's Linux NPU driver has no - WSL2 support ([intel/linux-npu-driver#56](https://github.com/intel/linux-npu-driver/issues/56)). + WSL2 support ([intel/linux-npu-driver#56](https://github.com/intel/linux-npu-driver/issues/56)). ## Remove everything diff --git a/examples/kserve/install.sh b/examples/kserve/install.sh index 34101868..9ecab427 100755 --- a/examples/kserve/install.sh +++ b/examples/kserve/install.sh @@ -44,7 +44,9 @@ helm upgrade --install kserve-resources oci://ghcr.io/kserve/charts/kserve-resou --version "${KSERVE_VERSION}" --namespace kserve \ --values "${HERE}/kserve-values.yaml" --wait --timeout 10m +# known-issue-begin: kserve-4592 kubectl apply --server-side -f "${HERE}/openvino-runtime.yaml" +# known-issue-end: kserve-4592 sed -e "s/supplementalGroups: \[\]/supplementalGroups: [${render_gid}]/" \ -e "s|gpu.intel.com/i915:|gpu.intel.com/${driver}:|" "${HERE}/qwen3-isvc.yaml" \ diff --git a/examples/kserve/openvino-runtime.yaml b/examples/kserve/openvino-runtime.yaml index cd50295d..498be28b 100644 --- a/examples/kserve/openvino-runtime.yaml +++ b/examples/kserve/openvino-runtime.yaml @@ -4,6 +4,7 @@ # release charts, so it ships here. Changes from upstream: the Helm ownership # annotation and label are dropped (Helm does not install it here), and the # placeholder image openvino/model_server:replace is pinned to the GPU build. +# known-issue: kserve-4592 (this file goes once a KServe release's runtimes chart ships kserve-openvino) apiVersion: serving.kserve.io/v1alpha1 kind: ClusterServingRuntime metadata: diff --git a/hack/known-issues b/hack/known-issues new file mode 100755 index 00000000..bc496190 --- /dev/null +++ b/hack/known-issues @@ -0,0 +1,786 @@ +#!/usr/bin/env python3 +"""known-issues: the upstream bugs, changes and advisories k3d-gpu depends on, +with the release that fixes each (pins/known-issues.toml). + + hack/known-issues [--root DIR] check [--json] + Offline and deterministic (no clock, no network). Fails when: + - the registry breaks its schema (pins/known-issues.toml header); + - a `known-issue: ` / `known-issue-begin|end: ` marker names no + entry, sits in a file its entry's `workarounds` does not list, sits in a + ledger or historical file, or a begin has no end; or a listed workaround + file has no marker; + - a site regex matches nothing, or a matched version does not parse; + - an "ours" site's version is inside `affected` while `fixed_in` is set + ("bump to >= "); + - workaround markers remain while every "ours" site is at or past + `fixed_in` and `retained` is empty. + Reports, without failing: "eat-stock" sites in range, "ours" sites in range + of an unfixed issue, and ledger_refs in range. --json prints the evaluation. + + hack/known-issues [--root DIR] poll [--json] [--id ID]... + Network, read-only (gh api with GITHUB_TOKEN; writes no file). For each + entry: finds the fix commit (fix.sha, the pull request's merge commit, or + the commit that closed the issue, from its timeline), walks the fix repo's + stable releases in version order from the lowest site version, and takes + the first whose compare/... reads "behind" or "identical". A + fix of the form {repo, path, contains} takes the first release whose file + contains the string; {ecosystem, package} reads the GitHub advisory + database for the lowest site version and takes the first stable patched + version above it outside every advisory's range, reading each + candidate's own advisories too until one has none. With an `image` + template the image tag of the fixing release must resolve + (scripts/images-lock digest). Prints + {id, state, fixed_in, tag, release_url, published_at, recorded, mismatch} + per entry; exit 1 when an entry could not be read. + +Site and marker classes come from pins/classes (` ...`, first match +wins) when it exists, then from the defaults below: EAT's stock paths are +"eat-stock", bench/ is "ledger", oracle/seed/ and console/matrix.v1.json are +"historical", everything else is "ours". Sites and markers may only be in "ours" +or "eat-stock" files, never under bench/ or oracle/seed/ or in EXPERIMENTS.md. + +Python standard library only (tomllib: Python 3.11+). +""" + +import argparse +import base64 +import fnmatch +import functools +import importlib.machinery +import importlib.util +import json +import os +import pathlib +import re +import subprocess +import sys +import tomllib +import urllib.parse + +HERE = pathlib.Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import pinver # noqa: E402 + +REGISTRY = "pins/known-issues.toml" +CLASSES = "pins/classes" +MARKER_RE = re.compile(r"known-issue(-begin|-end)?:[ \t]*([a-z0-9][a-z0-9.-]*[a-z0-9])") +ID_RE = re.compile(r"^[a-z0-9][a-z0-9.-]*[a-z0-9]$") +UPSTREAM_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+#[1-9][0-9]*$") +REPO_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") +ROW_RE = re.compile(r"^H\d+[a-z]{0,2}$") +GHSA_RE = re.compile(r"^GHSA(-[23456789cfghjmpqrvwx]{4}){3}$") +KINDS = ("bug", "change", "advisory") +ADOPT = ("review", "auto") + +# The registry, the tools and their tests hold marker text as data. +SELF = {REGISTRY, "hack/known-issues", "hack/ledger-freshness", "hack/pinver.py", + "tests/known_issues_test.py", "tests/ledger_freshness_test.py", "tests/pinver_test.py"} +DEFAULT_CLASSES = [ + ("integrations/eat/*", "eat-stock"), ("examples/kuberay/*", "eat-stock"), + ("examples/agents/litellm-gateway/*", "eat-stock"), ("contrib/eat/*", "eat-stock"), + ("bench/*", "ledger"), ("oracle/seed/*", "historical"), ("console/matrix.v1.json", "historical"), +] +NEVER = ("bench/", "oracle/seed/") +NEVER_FILES = ("EXPERIMENTS.md", "console/matrix.v1.json") + +REQUIRED = {"id": str, "title": str, "kind": str, "upstream": str, "component": str, "scheme": str, + "affected": str, "fixed_in": str, "adopt": str, "sites": list, "workarounds": list, "notes": str} +OPTIONAL = {"fix": dict, "retained": dict, "ledger_refs": list, "advisories": list, "image": str, "test": str} + + +class ApiError(Exception): + def __init__(self, status, detail): + super().__init__("HTTP %s: %s" % (status, detail)) + self.status = status + + +# --- loading ------------------------------------------------------------------- + +def load(root, registry=None): + """The [[issue]] entries of the registry, in file order.""" + path = pathlib.Path(registry) if registry else pathlib.Path(root) / REGISTRY + with open(path, "rb") as f: + data = tomllib.load(f) + extra = sorted(set(data) - {"issue"}) + if extra: + raise ValueError("%s: unknown top-level key(s) %s" % (REGISTRY, ", ".join(extra))) + issues = data.get("issue", []) + if not isinstance(issues, list): + raise ValueError("%s: issue must be an array of tables ([[issue]])" % REGISTRY) + return issues + + +def load_classes(root): + """[(glob, class)]: pins/classes lines first (first match wins), then the defaults.""" + out = [] + p = pathlib.Path(root) / CLASSES + if p.is_file(): + for line in p.read_text().splitlines(): + parts = line.split() + if not parts or parts[0].startswith("#") or parts[0] == "historical-ref" or len(parts) < 2: + continue + out.append((parts[0], parts[1])) + return out + DEFAULT_CLASSES + + +def path_class(classes, rel): + for glob, cls in classes: + if fnmatch.fnmatchcase(rel, glob): + return cls + return "ours" + + +def forbidden(classes, rel): + """Why a site or marker may not live in rel, or None.""" + if rel.startswith(NEVER) or rel in NEVER_FILES: + return "a ledger or historical file" + cls = path_class(classes, rel) + if cls not in ("ours", "eat-stock"): + return "a %s file" % cls + return None + + +def ledger_rows(root): + """{row: verdict} of the target tree's EXPERIMENTS.md via tests/ledger_drift_check.py, or None.""" + if not (pathlib.Path(root) / "EXPERIMENTS.md").is_file(): + return None + src = HERE.parent / "tests" / "ledger_drift_check.py" + loader = importlib.machinery.SourceFileLoader("ledger_drift_check", str(src)) + spec = importlib.util.spec_from_loader("ledger_drift_check", loader) + mod = importlib.util.module_from_spec(spec) + loader.exec_module(mod) + return mod.ledger(pathlib.Path(root)) + + +def list_files(root): + """Repo-relative paths: git's tracked and unignored files when root is a work tree, else a walk.""" + root = pathlib.Path(root) + try: + top = subprocess.run(["git", "-C", str(root), "rev-parse", "--show-toplevel"], + capture_output=True, text=True, check=True).stdout.strip() + if pathlib.Path(top).resolve() == root.resolve(): + out = subprocess.run(["git", "-C", str(root), "ls-files", "-z", "--cached", "--others", + "--exclude-standard"], capture_output=True, check=True).stdout + return sorted({p for p in out.decode().split("\0") if p and (root / p).is_file()}) + except (OSError, subprocess.CalledProcessError): + pass + found = [] + for d, dirs, files in os.walk(root): + dirs[:] = sorted(x for x in dirs if x != ".git") + found += [os.path.relpath(os.path.join(d, f), root) for f in files] + return sorted(found) + + +# --- sites and markers --------------------------------------------------------- + +def matches(root, rel, regex, scheme): + """(versions, error) of a {file, regex} pair.""" + p = pathlib.Path(root) / rel + if not p.is_file(): + return None, "%s: no such file" % rel + try: + rx = re.compile(regex, re.M) + except re.error as e: + return None, "regex %r does not compile (%s)" % (regex, e) + if "v" not in rx.groupindex: + return None, "regex %r has no named group v" % regex + found = [] + for m in rx.finditer(p.read_text(errors="replace")): + if m.group("v") not in found: + found.append(m.group("v")) + if not found: + return None, "regex %r matches nothing in %s" % (regex, rel) + for v in found: + try: + pinver.parse(scheme, v) + except ValueError as e: + return None, "%s: %s" % (rel, e) + return found, None + + +def scan_markers(root, files): + out = [] + for rel in files: + if rel in SELF: + continue + try: + data = (pathlib.Path(root) / rel).read_bytes() + except OSError: + continue + if b"known-issue" not in data: + continue + for n, line in enumerate(data.decode("utf-8", errors="replace").splitlines(), 1): + for m in MARKER_RE.finditer(line): + out.append({"file": rel, "line": n, "kind": (m.group(1) or "-marker")[1:], "id": m.group(2)}) + return out + + +# --- validation ---------------------------------------------------------------- + +def _str_list(v): + return isinstance(v, list) and all(isinstance(x, str) and x for x in v) + + +def validate_fix(fix, kind): + if not fix: + return [] + keys = set(fix) + errs = [] + if "repo" in keys and keys <= {"repo", "sha", "pr"} and keys & {"sha", "pr"}: + if not (isinstance(fix["repo"], str) and REPO_RE.match(fix["repo"])): + errs.append("fix.repo must be owner/name") + if "sha" in fix and not (isinstance(fix["sha"], str) and re.match(r"^[0-9a-f]{40}$", fix["sha"])): + errs.append("fix.sha must be a full 40-hex commit") + if "pr" in fix and not (isinstance(fix["pr"], int) and not isinstance(fix["pr"], bool) and fix["pr"] > 0): + errs.append("fix.pr must be a positive integer") + elif keys == {"repo", "path", "contains"}: + if not (isinstance(fix["repo"], str) and REPO_RE.match(fix["repo"])): + errs.append("fix.repo must be owner/name") + if not all(isinstance(fix[k], str) and fix[k] for k in ("path", "contains")): + errs.append("fix.path and fix.contains must be non-empty strings") + elif keys == {"ecosystem", "package"}: + if kind != "advisory": + errs.append("fix {ecosystem, package} is for kind advisory") + if not all(isinstance(fix[k], str) and fix[k] for k in keys): + errs.append("fix.ecosystem and fix.package must be non-empty strings") + else: + errs.append("fix must be {}, {repo, sha and/or pr}, {repo, path, contains} or {ecosystem, package}") + return errs + + +def validate(root, issues, classes, rows): + """Schema errors, as 'pins/known-issues.toml: : ' lines; and the ids that passed.""" + errors, ok, seen = [], set(), set() + for i, e in enumerate(issues): + ident = e.get("id") if isinstance(e, dict) and isinstance(e.get("id"), str) else "issue %d" % (i + 1) + errs = [] + + def err(msg, errs=errs): + errs.append(msg) + + if not isinstance(e, dict): + errors.append("%s: %s: not a table" % (REGISTRY, ident)) + continue + for k in sorted(set(e) - set(REQUIRED) - set(OPTIONAL)): + err("unknown field %s" % k) + for k, t in REQUIRED.items(): + if k not in e: + err("missing field %s" % k) + elif not isinstance(e[k], t): + err("%s must be a %s" % (k, t.__name__)) + for k, t in OPTIONAL.items(): + if k in e and not isinstance(e[k], t): + err("%s must be a %s" % (k, t.__name__)) + if errs: + errors += ["%s: %s: %s" % (REGISTRY, ident, m) for m in errs] + continue + if not ID_RE.match(e["id"]): + err("id must be lowercase letters, digits, '.' and '-'") + if e["id"] in seen: + err("duplicate id %s" % e["id"]) + seen.add(e["id"]) + if e["kind"] not in KINDS: + err("kind %r is not one of %s" % (e["kind"], ", ".join(KINDS))) + if e["adopt"] not in ADOPT: + err("adopt %r is not one of %s" % (e["adopt"], ", ".join(ADOPT))) + if e["upstream"] and not UPSTREAM_RE.match(e["upstream"]): + err("upstream %r is not owner/repo#n" % e["upstream"]) + if not e["title"].strip(): + err("title is empty") + errs += validate_fix(e.get("fix", {}), e["kind"]) + scheme = e["scheme"] + if scheme not in pinver.SCHEMES: + err("scheme %r is not one of %s" % (scheme, ", ".join(pinver.SCHEMES))) + errors += ["%s: %s: %s" % (REGISTRY, ident, m) for m in errs] + continue + try: + pinver.check_range(scheme, e["affected"]) + except ValueError as x: + err("affected: %s" % x) + if e["fixed_in"]: + try: + if pinver.is_prerelease(scheme, e["fixed_in"]): + err("fixed_in %s is a prerelease" % e["fixed_in"]) + if pinver.in_range(scheme, e["fixed_in"], e["affected"]): + err("fixed_in %s is inside affected %s; narrow affected" % (e["fixed_in"], e["affected"])) + except ValueError as x: + err("fixed_in: %s" % x) + for s in e["sites"]: + if not (isinstance(s, dict) and set(s) == {"file", "regex"} and all(isinstance(s[k], str) for k in s)): + err("each site is {file, regex}") + continue + why = forbidden(classes, s["file"]) + if why: + err("site %s is in %s; sites are ours or eat-stock files" % (s["file"], why)) + _, problem = matches(root, s["file"], s["regex"], scheme) + if problem: + err("site %s" % problem) + if not _str_list(e["workarounds"]) or len(set(e["workarounds"])) != len(e["workarounds"]): + err("workarounds must be a list of distinct file paths") + else: + for w in e["workarounds"]: + why = forbidden(classes, w) + if why: + err("workaround %s is in %s" % (w, why)) + elif not (pathlib.Path(root) / w).is_file(): + err("workaround %s: no such file" % w) + retained = e.get("retained", {}) + if retained and not (set(retained) == {"reason", "pr"} and isinstance(retained["reason"], str) + and retained["reason"].strip() and isinstance(retained["pr"], (str, int))): + err("retained must be {} or {reason, pr} with a reason") + for r in e.get("ledger_refs", []): + if not (isinstance(r, dict) and {"row", "path"} <= set(r) <= {"row", "path", "regex"} + and all(isinstance(r[k], str) for k in r)): + err("each ledger_ref is {row, path, regex?}") + continue + if not ROW_RE.match(r["row"]): + err("ledger_ref row %r is not a ledger id" % r["row"]) + elif rows is not None and r["row"] not in rows: + err("ledger_ref row %s is not in the EXPERIMENTS.md hypothesis table" % r["row"]) + if not (pathlib.Path(root) / r["path"]).is_file(): + err("ledger_ref %s: no such file" % r["path"]) + elif "regex" in r: + _, problem = matches(root, r["path"], r["regex"], scheme) + if problem: + err("ledger_ref %s" % problem) + if "advisories" in e: + if e["kind"] != "advisory": + err("advisories is for kind advisory") + elif not _str_list(e["advisories"]) or not all(GHSA_RE.match(a) for a in e["advisories"]): + err("advisories must be GHSA ids") + if "image" in e and "{tag}" not in e["image"] and "{version}" not in e["image"]: + err("image must hold {tag} or {version}") + test = e.get("test", "") + if test and not test.startswith("canary:") and not (pathlib.Path(root) / test).is_file(): + err("test %s: no such file" % test) + errors += ["%s: %s: %s" % (REGISTRY, ident, m) for m in errs] + if not errs: + ok.add(e["id"]) + return errors, ok + + +def check_markers(issues, markers, classes): + errors = [] + by_id = {e["id"]: e for e in issues if isinstance(e, dict) and isinstance(e.get("id"), str)} + open_blocks = {} + for m in markers: + where = "%s:%d" % (m["file"], m["line"]) + tag = "known-issue%s: %s" % ("" if m["kind"] == "marker" else "-" + m["kind"], m["id"]) + why = forbidden(classes, m["file"]) + if why: + errors.append("%s: %s is in %s; markers go in ours or eat-stock files only" % (where, tag, why)) + e = by_id.get(m["id"]) + if e is None: + errors.append("%s: %s names no entry in %s" % (where, tag, REGISTRY)) + elif m["file"] not in e.get("workarounds", []): + errors.append("%s: %s, but %s's workarounds do not list %s" % (where, tag, m["id"], m["file"])) + key = (m["file"], m["id"]) + if m["kind"] == "begin": + open_blocks.setdefault(key, []).append(m["line"]) + elif m["kind"] == "end": + if not open_blocks.get(key): + errors.append("%s: %s has no known-issue-begin: %s before it" % (where, tag, m["id"])) + else: + open_blocks[key].pop() + for (f, i), lines in sorted(open_blocks.items()): + for n in lines: + errors.append("%s:%d: known-issue-begin: %s has no known-issue-end: %s" % (f, n, i, i)) + have = {(m["file"], m["id"]) for m in markers} + for e in by_id.values(): + for w in e.get("workarounds", []) if isinstance(e.get("workarounds"), list) else []: + if isinstance(w, str) and (w, e["id"]) not in have: + errors.append("%s: %s: %s has no known-issue marker for %s" % (REGISTRY, e["id"], w, e["id"])) + return errors + + +# --- check --------------------------------------------------------------------- + +def at_or_past(scheme, v, fixed): + try: + return pinver.in_range(scheme, v, ">=" + fixed) + except ValueError: + return False + + +def evaluate(root, e, classes, markers): + """(record, failures, reports) for one valid entry.""" + scheme, aff, fixed, ident = e["scheme"], e["affected"], e["fixed_in"], e["id"] + failures, reports, sites = [], [], [] + tail = ", fixed in %s" % fixed if fixed else ", no fixed release known" + for s in e["sites"]: + versions, _ = matches(root, s["file"], s["regex"], scheme) + cls = path_class(classes, s["file"]) + inside = [v for v in versions if pinver.in_range(scheme, v, aff)] + sites.append({"file": s["file"], "class": cls, "versions": versions, "in_affected": bool(inside)}) + for v in inside: + what = "%s %s is inside %s's affected range %s%s" % (e["component"], v, ident, aff, tail) + if cls == "ours" and fixed: + failures.append("%s: %s; bump %s to >= %s" % (s["file"], what, s["file"], fixed)) + else: + reports.append("%s %s: %s" % (cls, s["file"], what)) + refs = [] + for r in e.get("ledger_refs", []): + versions = matches(root, r["path"], r["regex"], scheme)[0] if "regex" in r else [] + inside = [v for v in versions if pinver.in_range(scheme, v, aff)] + refs.append({"row": r["row"], "path": r["path"], "versions": versions, "in_affected": bool(inside)}) + for v in inside: + reports.append("ledger %s %s: %s %s is inside %s's affected range %s%s (informational; the ledger " + "changes only by amendment)" % (r["row"], r["path"], e["component"], v, ident, aff, tail)) + mine = [m for m in markers if m["id"] == ident] + governing = [s for s in sites if s["class"] == "ours"] or sites + past = bool(fixed) and bool(governing) and all( + at_or_past(scheme, v, fixed) for s in governing for v in s["versions"]) + if any(s["class"] == "ours" and s["in_affected"] for s in sites) and fixed: + status = "needs-bump" + elif not fixed: + status = "open" + elif past and mine and not e.get("retained"): + status = "stale-workaround" + files = sorted({m["file"] for m in mine}) + failures.append("%s: workaround markers remain in %s though every ours site is at or past %s; remove " + "them (known-issue-begin/end blocks) or set retained" % (ident, ", ".join(files), fixed)) + elif mine and e.get("retained"): + status = "retained" + else: + status = "fixed" + record = {"id": ident, "kind": e["kind"], "component": e["component"], "scheme": scheme, "affected": aff, + "fixed_in": fixed, "upstream": e["upstream"], "adopt": e["adopt"], "status": status, "sites": sites, + "ledger_refs": refs, "workarounds": e["workarounds"], + "markers": [{k: m[k] for k in ("file", "line", "kind")} for m in mine]} + return record, failures, reports + + +def check(root, registry=None): + root = pathlib.Path(root) + try: + issues = load(root, registry) + except (OSError, ValueError, tomllib.TOMLDecodeError) as x: + return {"ok": False, "errors": ["%s: %s" % (REGISTRY, x)], "reports": [], "issues": []} + classes = load_classes(root) + errors, valid = validate(root, issues, classes, ledger_rows(root)) + markers = scan_markers(root, list_files(root)) + errors += check_markers(issues, markers, classes) + reports, records = [], [] + for e in issues: + if isinstance(e, dict) and e.get("id") in valid: + record, failures, rep = evaluate(root, e, classes, markers) + records.append(record) + errors += failures + reports += rep + return {"ok": not errors, "errors": errors, "reports": reports, "issues": records} + + +def print_check(result): + sites = sum(len(i["sites"]) for i in result["issues"]) + marks = sum(len(i["markers"]) for i in result["issues"]) + print("known-issues: %d issue(s), %d site(s), %d marker(s)" % (len(result["issues"]), sites, marks)) + for r in result["reports"]: + print("report %s" % r) + for x in result["errors"]: + print("FAIL %s" % x) + if result["ok"]: + print("known-issues: ok") + else: + print("known-issues: %d problem(s)" % len(result["errors"])) + + +# --- poll ---------------------------------------------------------------------- + +def gh_api(path): + p = subprocess.run(["gh", "api", "-H", "Accept: application/vnd.github+json", path], + capture_output=True, text=True) + if p.returncode != 0: + m = re.search(r"HTTP (\d{3})", p.stderr) + raise ApiError(int(m.group(1)) if m else 0, "%s: %s" % (path, p.stderr.strip()[:300])) + return json.loads(p.stdout) if p.stdout.strip() else None + + +def images_lock_digest(ref): + p = subprocess.run([sys.executable, str(HERE.parent / "scripts" / "images-lock"), "digest", ref], + capture_output=True, text=True) + if p.returncode != 0 or not p.stdout.strip().startswith("sha256:"): + raise ValueError("%s does not resolve: %s" % (ref, (p.stderr or p.stdout).strip()[:300])) + return p.stdout.strip() + + +def norm(tag): + return tag[1:] if re.match(r"^v\d", tag) else tag + + +def site_floor(root, e): + """The lowest site version (a pinver.Version), or None.""" + found = [] + for s in e["sites"]: + versions, _ = matches(root, s["file"], s["regex"], e["scheme"]) + found += [pinver.parse(e["scheme"], v) for v in versions or []] + found = [v for v in found if pinver.comparable(e["scheme"], v, found[0])] if found else [] + return min(found, key=functools.cmp_to_key(lambda a, b: pinver.compare(e["scheme"], a, b))) if found else None + + +def stable_releases(api, repo, scheme, floor): + """[(Version, tag, release)] ascending: published, not prerelease, parseable, of floor's flavour.""" + out = [] + for page in range(1, 11): + batch = api("repos/%s/releases?per_page=100&page=%d" % (repo, page)) or [] + for r in batch: + if r.get("draft") or r.get("prerelease"): + continue + try: + v = pinver.parse(scheme, r["tag_name"]) + except ValueError: + continue + if v.pre or (floor is not None and not pinver.comparable(scheme, v, floor)): + continue + out.append((v, r["tag_name"], r)) + if len(batch) < 100: + break + if floor is None and out: + out = [t for t in out if pinver.comparable(scheme, t[0], out[0][0])] + out.sort(key=functools.cmp_to_key(lambda a, b: pinver.compare(scheme, a[0], b[0]))) + return out + + +def first_containing(tags, floor, scheme, contains, budget=60): + """Index of the first release that contains the fix, walking from the floor's release, or None.""" + if not tags: + return None + start = 0 + if floor is not None: + start = next((i for i, t in enumerate(tags) if pinver.compare(scheme, t[0], floor) >= 0), len(tags) - 1) + calls = [0] + + def has(i): + calls[0] += 1 + if calls[0] > budget: + raise ValueError("more than %d release checks; narrow the sites or the fix" % budget) + return contains(tags[i][1]) + + if has(start): + i = start + while i > 0 and has(i - 1): + i -= 1 + return i + for i in range(start + 1, len(tags)): + if has(i): + return i + return None + + +def pr_commit(api, repo, n): + pr = api("repos/%s/pulls/%s" % (repo, n)) or {} + if pr.get("merged") or pr.get("merged_at"): + return "merged", repo, pr.get("merge_commit_sha") + return ("open" if pr.get("state") == "open" else "closed-unmerged"), None, None + + +def fix_commit(api, e): + """(state, repo, sha or None) of an entry's fix.""" + fix = e.get("fix") or {} + if fix.get("sha"): + return "merged", fix["repo"], fix["sha"] + if fix.get("pr"): + return pr_commit(api, fix["repo"], fix["pr"]) + if not e["upstream"]: + return "untracked", None, None + repo, n = e["upstream"].split("#") + issue = api("repos/%s/issues/%s" % (repo, n)) or {} + if issue.get("pull_request"): + return pr_commit(api, repo, n) + if issue.get("state") == "open": + return "open", None, None + if issue.get("state_reason") == "not_planned": + return "closed-not-planned", None, None + closing = None + for page in range(1, 11): + events = api("repos/%s/issues/%s/timeline?per_page=100&page=%d" % (repo, n, page)) or [] + for ev in events: + if ev.get("event") == "closed" and ev.get("commit_id"): + m = re.search(r"/repos/([^/]+/[^/]+)/commits/", ev.get("commit_url") or "") + closing = (m.group(1) if m else repo, ev["commit_id"]) + if len(events) < 100: + break + if closing: + return "closed", closing[0], closing[1] + return "closed-no-commit", None, None + + +def gh_range(text): + """A GitHub advisory range ('>= 1.65.5, < 1.85.0') as a pinver spec.""" + parts = [] + for p in text.split(","): + m = re.match(r"^\s*(<=|>=|<|>|=)\s*(\S+)\s*$", p) + if not m: + raise ValueError("advisory range %r" % text) + parts.append(("==" if m.group(1) == "=" else m.group(1)) + m.group(2)) + return ",".join(parts) + + +def poll_advisory(api, root, e, out, rounds=10): + """The first stable patched version above the lowest site version that no advisory affects: + the advisories of the site version, then of each candidate, until a candidate has none.""" + fix, scheme = e["fix"], e["scheme"] + floor = site_floor(root, e) + if floor is None: + raise ValueError("no site version to query the advisory database with") + ranges, patched, affected = {}, set(), set() + + def read(version): + """Add the advisories that affect version; True when there is one.""" + q = urllib.parse.quote("%s@%s" % (fix["package"], version)) + hit = False + for a in api("advisories?ecosystem=%s&affects=%s&per_page=100" % (fix["ecosystem"], q)) or []: + spec = [] + for vul in a.get("vulnerabilities") or []: + pkg = vul.get("package") or {} + if (pkg.get("name") or "").lower() != fix["package"].lower(): + continue + spec.append(gh_range(vul.get("vulnerable_version_range") or "")) + if vul.get("first_patched_version"): + patched.add(vul["first_patched_version"]) + if spec: + ranges.setdefault(a["ghsa_id"], " || ".join(spec)) + hit = True + return hit + + def clear(p): + try: + return (p not in affected and not pinver.is_prerelease(scheme, p) + and pinver.compare(scheme, p, floor) > 0 + and not any(pinver.in_range(scheme, p, r) for r in ranges.values())) + except ValueError: + return False + + read(floor.raw) + for _ in range(rounds): + cands = [p for p in patched if clear(p)] + best = min(cands, key=functools.cmp_to_key(lambda a, b: pinver.compare(scheme, a, b))) if cands else "" + if not best or not read(best): + break + affected.add(best) + else: + raise ValueError("every patched version had advisories of its own after %d reads" % rounds) + out.update(state="advisory", fixed_in=best, advisories=sorted(ranges), + new_advisories=sorted(set(ranges) - set(e.get("advisories", [])))) + if best and fix["ecosystem"] == "pip": + out["release_url"] = "https://pypi.org/project/%s/%s/" % (fix["package"], best) + + +def poll_entry(api, digest, root, e): + out = {"id": e["id"], "state": "", "fixed_in": "", "tag": "", "release_url": "", "published_at": "", + "recorded": e.get("fixed_in", ""), "mismatch": False} + fix = e.get("fix") or {} + try: + if fix.get("ecosystem"): + poll_advisory(api, root, e, out) + else: + if fix.get("path"): + repo = fix["repo"] + + def contains(ref=None): + path = "repos/%s/contents/%s" % (repo, fix["path"]) + ("?ref=%s" % ref if ref else "") + try: + body = api(path) or {} + except ApiError as x: + if x.status == 404: + return False + raise + if body.get("encoding") != "base64": + raise ValueError("%s: no base64 content (file too large for the contents API?)" % path) + return fix["contains"] in base64.b64decode(body["content"]).decode("utf-8", errors="replace") + else: + state, repo, sha = fix_commit(api, e) + if not sha: + out["state"] = state + return finish(out) + + def contains(ref=None): + try: + status = (api("repos/%s/compare/%s...%s" % (repo, ref, sha)) or {}).get("status") + except ApiError as x: + if x.status == 404: + return False + raise + return status in ("behind", "identical") + floor = site_floor(pathlib.Path(root), e) + tags = stable_releases(api, repo, e["scheme"], floor) + i = first_containing(tags, floor, e["scheme"], contains) + if i is None: + out["state"] = "not-shipped" if fix.get("path") and not contains() else "merged-unreleased" + else: + _, tag, rel = tags[i] + out.update(state="released", fixed_in=norm(tag), tag=tag, release_url=rel.get("html_url", ""), + published_at=rel.get("published_at", "")) + if e.get("image"): + out["image"] = e["image"].format(tag=tag, version=norm(tag)) + out["image_digest"] = digest(out["image"]) + except (ApiError, ValueError, KeyError, TypeError) as x: + out["error"] = str(x) + return out + return finish(out, e["scheme"]) + + +def finish(out, scheme=None): + if out["state"] == "untracked" or "error" in out: + return out + a, b = out["fixed_in"], out["recorded"] + same = a == b + if a and b and not same and scheme: + try: + same = pinver.compare(scheme, a, b) == 0 + except ValueError: + pass + out["mismatch"] = not same + return out + + +def poll(issues, root, api=gh_api, digest=images_lock_digest, ids=None): + return [poll_entry(api, digest, root, e) for e in issues + if isinstance(e, dict) and (not ids or e.get("id") in ids)] + + +def print_poll(rows): + for r in rows: + line = "%s: %s" % (r["id"], r.get("error") and "error: " + r["error"] or r["state"]) + if r["fixed_in"] or r["recorded"]: + line += " fixed_in=%s (recorded %s)%s" % (r["fixed_in"] or "-", r["recorded"] or "-", + " MISMATCH" if r["mismatch"] else "") + if r.get("new_advisories"): + line += " new advisories: %s" % ", ".join(r["new_advisories"]) + if r["release_url"]: + line += " %s" % r["release_url"] + print(line) + + +def main(argv): + ap = argparse.ArgumentParser(prog="known-issues", description="Known upstream issues and their fixed-in releases.") + ap.add_argument("--root", default=str(HERE.parent), help="repo root (default: this repo)") + ap.add_argument("--registry", help="registry file (default: ROOT/%s)" % REGISTRY) + sub = ap.add_subparsers(dest="cmd", required=True) + c = sub.add_parser("check", help="offline: schema, markers, sites, fixed-in rules") + c.add_argument("--json", action="store_true") + p = sub.add_parser("poll", help="network, read-only: find each entry's first fixed release") + p.add_argument("--json", action="store_true") + p.add_argument("--id", action="append", default=[], help="poll only this entry (repeatable)") + args = ap.parse_args(argv) + if args.cmd == "check": + result = check(args.root, args.registry) + if args.json: + print(json.dumps(result, indent=1, sort_keys=True)) + else: + print_check(result) + return 0 if result["ok"] else 1 + try: + issues = load(args.root, args.registry) + except (OSError, ValueError, tomllib.TOMLDecodeError) as x: + print("known-issues: %s" % x, file=sys.stderr) + return 2 + rows = poll(issues, pathlib.Path(args.root), ids=set(args.id)) + if args.json: + print(json.dumps(rows, indent=1, sort_keys=True)) + else: + print_poll(rows) + return 1 if any("error" in r for r in rows) else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/hack/ledger-freshness b/hack/ledger-freshness new file mode 100755 index 00000000..11ef66e0 --- /dev/null +++ b/hack/ledger-freshness @@ -0,0 +1,425 @@ +#!/usr/bin/env python3 +"""ledger-freshness: which SUPPORTED ledger rows were measured on component +versions older than today's pins. Read-only: it never writes EXPERIMENTS.md, +console/ledger-rules.json or any other file. + + hack/ledger-freshness [--root DIR] --json | --md + +For each SUPPORTED row of the Hypotheses table (parsed by ledger() from +tests/ledger_drift_check.py), it follows the run anchors of the Probe and +Verdict cells (links to a heading that says run or result; else a heading +" run|result ...") to the run sections, and reads their records: fenced +blocks that open with "# Reproduce" or "# provenance", and "**Reproduce:**" +paragraphs. From those it extracts the measured versions of vllm-openai-cpu/xpu, +openvino/model_server (a version, or a digest that images.lock maps to a tag), +llama.cpp (bNNNNN), rancher/k3s, the cryptoandcoffee/k3d-gpu node image, and +the Hugging Face revisions on the `models:` line. + +Current versions are today's "ours" pins: images.lock entries with an "ours" +source (pins/classes, as hack/known-issues reads it) and scripts/k3d-gpu's +VERSION for the node image. A row is stale when a measured version is a minor +or major behind the current one (hack/pinver.py update types; a llama.cpp build +change counts as a minor); priority-stale when a measured version is inside a +known issue's affected range (pins/known-issues.toml). + +The queue lists rows in console/ledger-rules.json with non-empty classes first +(they drive optimizer defaults), then rows docs claims cite +(tests/claims_check.sh --cited), then the rest; priority-stale rows first within +each tier. A re-measure is a new pre-registered ledger row; the suggested id is +only a free one. Rows whose records the parser cannot read are listed under +"not machine-readable". + +--json: {schema, measured: {row: {component: version}}, current: {component: +version}, queue: [{row, tier, priority, measured, current, reason, known_issues, +suggested_followup_id}], not_machine_readable: [{row, reason}]} +--md: the same as a Markdown section for the freshness dashboard. + +Python standard library only. +""" + +import argparse +import functools +import importlib.machinery +import importlib.util +import json +import pathlib +import re +import subprocess +import sys + +HERE = pathlib.Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import pinver # noqa: E402 + +SCHEMA = "k3d-gpu.ledger-freshness/v1" +VERSION_RE = re.compile(r'^VERSION="(\d+\.\d+\.\d+)"', re.M) +RUN_RE = re.compile(r"\b(runs?|results?)\b", re.I) +FENCE_RE = re.compile(r"^\s*(`{3,}|~{3,})") +HEADING_RE = re.compile(r"^(#{2,6})\s+(.*?)\s*#*\s*$") +ANCHOR_RE = re.compile(r"\]\(#([^)\s]+)\)") +FIELD_RE = re.compile(r"^([a-z][a-z0-9 /()-]*?):\s+(.*)$") +HF_RE = re.compile(r"(?[A-Za-z0-9][\w.-]*/[\w.-]+)\s+@?\s*(?P[0-9a-f]{7,40})\b") +OVMS_DIGEST_RE = re.compile(r"openvino/model_server@sha256:(?P[0-9a-f]{8,64})") +CAL = r"\d{4}\.\d+(?:\.\d+)?" + +# component: (scheme, images.lock names, tag -> version regex, record regexes with group v) +COMPONENTS = { + "vllm-openai-cpu": ("zerover", ("docker.io/vllm/vllm-openai-cpu",), r"^v?(\d+\.\d+\.\d+)$", + (r"vllm/vllm-openai-cpu:v(?P\d+\.\d+\.\d+)",)), + "vllm-openai-xpu": ("zerover", ("docker.io/vllm/vllm-openai-xpu",), r"^v?(\d+\.\d+\.\d+)$", + (r"vllm/vllm-openai-xpu:v(?P\d+\.\d+\.\d+)",)), + "openvino/model_server": ("calver", ("docker.io/openvino/model_server",), r"^(" + CAL + r")", + (r"openvino/model_server:(?P" + CAL + r")", + r"openvino/model_server@sha256:[0-9a-f.…]+`?\s*\((?P" + CAL + r")\)", + r"\bOVMS\s+(?P" + CAL + r")\b")), + "llama.cpp": ("llama", ("ghcr.io/ggml-org/llama.cpp",), r"(?:^|-)(b\d+)$", + (r"llama\.cpp:(?:[a-z]+-)*(?Pb\d{3,})\b", r"\bllama\.cpp\s+(?:server-)?(?Pb\d{3,})\b")), + "rancher/k3s": ("k8s", ("docker.io/rancher/k3s",), r"^(v\d+\.\d+\.\d+[-+]k3s\d+)$", + (r"rancher/k3s:(?Pv\d+\.\d+\.\d+[-+]k3s\d+)", r"\bk3s (?Pv\d+\.\d+\.\d+\+k3s\d+)")), + "cryptoandcoffee/k3d-gpu": ("semver", (), None, (r"cryptoandcoffee/k3d-gpu:(?P\d+\.\d+\.\d+)\b",)), +} + + +def load_path(name, path): + loader = importlib.machinery.SourceFileLoader(name, str(path)) + spec = importlib.util.spec_from_loader(name, loader) + mod = importlib.util.module_from_spec(spec) + loader.exec_module(mod) + return mod + + +drift = load_path("ledger_drift_check", HERE.parent / "tests" / "ledger_drift_check.py") +known = load_path("known_issues", HERE / "known-issues") + + +def row_key(row): + m = re.match(r"^H(\d+)([a-z]*)$", row) + return (int(m.group(1)), m.group(2)) if m else (1 << 30, row) + + +def cmp_key(scheme): + return functools.cmp_to_key(lambda a, b: pinver.compare(scheme, a, b)) + + +# --- the ledger ---------------------------------------------------------------- + +def table_cells(text): + """{row: (probe cell, verdict cell)} of the Hypotheses table.""" + out, inside, header = {}, False, None + for line in text.splitlines(): + if line.startswith("## "): + inside = line.startswith("## Hypotheses") + continue + if not (inside and line.startswith("|")): + continue + cells = [c.strip() for c in re.sub(r"\\\|", "", line).strip().strip("|").split("|")] + if header is None and cells and cells[0] == "#": + header = [c.lower() for c in cells] + continue + m = re.match(r"^(H\d+[a-z]{0,2})$", cells[0]) if cells else None + if m and header: + probe = cells[header.index("probe")] if "probe" in header and header.index("probe") < len(cells) else "" + out[m.group(1)] = (probe, cells[-1]) + return out + + +def slugify(text, seen): + s = re.sub(r"[^\w\- ]", "", text.strip().lower()).replace(" ", "-") + n = seen.get(s, 0) + seen[s] = n + 1 + return s if n == 0 else "%s-%d" % (s, n) + + +def closes(line, fence): + """A closing fence (CommonMark): the opening character, at least as many, nothing after.""" + t = line.strip() + return t.startswith(fence) and not t[len(fence):].strip(fence[0]).strip() + + +def sections(lines): + """[(level, title, slug, first body line, end line)] of the headings outside fences.""" + heads, seen, fence = [], {}, None + for i, line in enumerate(lines): + if fence: + if closes(line, fence): + fence = None + continue + f = FENCE_RE.match(line) + if f: + fence = f.group(1) + continue + h = HEADING_RE.match(line) + if h: + heads.append([len(h.group(1)), h.group(2), slugify(h.group(2), seen), i + 1, len(lines)]) + for j, h in enumerate(heads): + for k in heads[j + 1:]: + if k[0] <= h[0]: + h[4] = k[3] - 1 + break + return heads + + +def records(body): + """(text of the Reproduce/provenance records, models field text, problem) of a section body. + problem: a record block that holds a Markdown heading, the sign of a fence left open above it.""" + texts, models, problem, i = [], [], None, 0 + while i < len(body): + f = FENCE_RE.match(body[i]) + if f: + j = i + 1 + while j < len(body) and not closes(body[j], f.group(1)): + j += 1 + block = body[i + 1:j] + first = next((b.strip() for b in block if b.strip()), "") + if first.startswith(("# Reproduce", "# provenance")): + swallowed = next((b for b in block if HEADING_RE.match(b)), None) + if swallowed: + problem = "a record block runs into the heading %r (a fence left open?)" % swallowed.strip()[:80] + texts.append("\n".join(block)) + field = None + for b in block: + m = FIELD_RE.match(b) + if m: + field = m.group(1) + if field == "models": + models.append(m.group(2)) + elif field == "models" and b.startswith((" ", "\t")): + models.append(b.strip()) + i = j + 1 + continue + if "**Reproduce:**" in body[i]: + j = i + while j < len(body) and body[j].strip() and not FENCE_RE.match(body[j]): + j += 1 + texts.append("\n".join(body[i:j])) + i = j + continue + i += 1 + return "\n".join(texts), "\n".join(models), problem + + +def lock_tags(lock): + """{(name, digest): tag} of the tagged images.lock entries.""" + return {(e["name"], e["digest"]): e["tag"] for e in lock.get("images", []) if e.get("tag")} + + +def extract(text, models, lock): + found = {} + for comp, (scheme, names, _, regexes) in COMPONENTS.items(): + vals = [m.group("v") for rx in regexes for m in re.finditer(rx, text)] + if comp == "openvino/model_server" and not vals: + tags = lock_tags(lock) + for m in OVMS_DIGEST_RE.finditer(text): + for (name, digest), tag in tags.items(): + if name in names and digest.startswith("sha256:" + m.group("d")): + t = re.match(COMPONENTS[comp][2], tag) + if t: + vals.append(t.group(1)) + good = [] + for v in vals: + try: + pinver.parse(scheme, v) + good.append(v) + except ValueError: + pass + if good: + group = [v for v in good if pinver.comparable(scheme, v, good[0])] + found[comp] = min(group, key=cmp_key(scheme)) + for m in HF_RE.finditer(models): + found.setdefault("hf:" + m.group("repo"), m.group("v")) + return found + + +def measured_rows(root, lock): + text = (root / "EXPERIMENTS.md").read_text() + lines = text.splitlines() + verdicts = drift.ledger(root) + cells = table_cells(text) + heads = sections(lines) + by_slug = {h[2]: h for h in heads} + measured, unreadable = {}, [] + for row in sorted((r for r, v in verdicts.items() if v == "SUPPORTED"), key=row_key): + probe, verdict = cells.get(row, ("", "")) + runs = [by_slug[a] for a in ANCHOR_RE.findall(probe + " " + verdict) + if a in by_slug and RUN_RE.search(by_slug[a][1])] + if not runs: + pat = re.compile(r"^%s (run|result)s?\b" % re.escape(row)) + runs = [h for h in heads if pat.match(h[1])] + if not runs: + unreadable.append({"row": row, "reason": "no run section linked from the Probe column"}) + continue + got, saw_record, problem = {}, False, None + for h in runs: + rec, models, bad = records(lines[h[3]:h[4]]) + saw_record, problem = saw_record or bool(rec), problem or bad + for comp, v in extract(rec, models, lock).items(): + got.setdefault(comp, v) + if problem: + unreadable.append({"row": row, "reason": problem}) + elif not saw_record: + unreadable.append({"row": row, "reason": "run section has no '# Reproduce' or '# provenance' block " + "and no **Reproduce:** paragraph"}) + elif not got: + unreadable.append({"row": row, "reason": "records name no component version this parser reads"}) + else: + measured[row] = got + return measured, unreadable + + +# --- current pins -------------------------------------------------------------- + +def current_versions(root, lock): + classes = known.load_classes(root) + tags = lock_tags(lock) + found = {} + for e in lock.get("images", []): + srcs = [s.split()[0] for s in e.get("sources", []) if s.split()] + if not any(known.forbidden(classes, s) is None and known.path_class(classes, s) == "ours" for s in srcs): + continue + tag = e.get("tag") or tags.get((e["name"], e["digest"])) + for comp, (scheme, names, tag_rx, _) in COMPONENTS.items(): + if e["name"] in names and tag and tag_rx: + m = re.search(tag_rx, tag) + if m: + found.setdefault(comp, []).append(m.group(1)) + script = root / "scripts" / "k3d-gpu" + if script.is_file(): + m = VERSION_RE.search(script.read_text(errors="replace")) + if m: + found["cryptoandcoffee/k3d-gpu"] = [m.group(1)] + out = {} + for comp, vals in found.items(): + scheme = COMPONENTS[comp][0] + group = [v for v in vals if pinver.comparable(scheme, v, vals[0])] + out[comp] = max(group, key=cmp_key(scheme)) + return out + + +# --- the queue ----------------------------------------------------------------- + +def cited_rows(root): + p = subprocess.run(["bash", str(HERE.parent / "tests" / "claims_check.sh"), "--cited", str(root)], + capture_output=True, text=True) + if p.returncode != 0: + print("ledger-freshness: claims_check.sh --cited failed; tier 2 is empty: %s" % p.stdout.strip()[-300:], + file=sys.stderr) + return set() + return set(p.stdout.split()) + + +def rule_rows(root): + p = root / "console" / "ledger-rules.json" + if not p.is_file(): + return set() + return {r["id"] for r in json.loads(p.read_text()).get("rows", []) if r.get("classes")} + + +def followup_id(row, taken): + m = re.match(r"^H(\d+)([a-z]*)$", row) + cands = [row + "r"] if len(m.group(2)) < 2 else [] + cands += ["H%sr" % m.group(1)] + ["H%sr%s" % (m.group(1), c) for c in "abcdefghijklmnopqstuvwxyz"] + return next(c for c in cands if c not in taken) + + +def queue(root, measured, current, issues): + rules, cited, taken = rule_rows(root), cited_rows(root), set(drift.ledger(root)) + items = [] + for row, comps in measured.items(): + reasons, prio, ids, m_out, c_out = [], [], [], {}, {} + for comp, mv in comps.items(): + if comp.startswith("hf:"): + continue + scheme, cur = COMPONENTS[comp][0], current.get(comp) + if cur and pinver.comparable(scheme, mv, cur) and pinver.compare(scheme, mv, cur) < 0: + t = pinver.update_type(scheme, mv, cur) + if t in ("minor", "major"): + reasons.append("%s %s -> %s (%s)" % (comp, mv, cur, t)) + m_out[comp], c_out[comp] = mv, cur + for e in issues: + if not isinstance(e, dict) or e.get("component") != comp: + continue + try: + inside = pinver.in_range(e["scheme"], mv, e["affected"]) + except (KeyError, ValueError): + continue + if inside: + since = "" + if e.get("fixed_in"): + since = ", %s in %s" % ("changed" if e.get("kind") == "change" else "fixed", e["fixed_in"]) + prio.append("priority-stale: %s %s is inside %s (%s%s)" % (comp, mv, e["id"], e["affected"], since)) + ids.append(e["id"]) + m_out[comp] = mv + if cur: + c_out[comp] = cur + if not (reasons or prio): + continue + tier = 1 if row in rules else 2 if row in cited else 3 + follow = followup_id(row, taken) + taken.add(follow) + items.append({"row": row, "tier": tier, "priority": bool(prio), "measured": m_out, "current": c_out, + "reason": "; ".join(prio + reasons), "known_issues": sorted(set(ids)), + "suggested_followup_id": follow}) + items.sort(key=lambda q: (q["tier"], not q["priority"], row_key(q["row"]))) + return items + + +def build(root): + root = pathlib.Path(root) + lock_path = root / "images.lock" + lock = json.loads(lock_path.read_text()) if lock_path.is_file() else {"images": []} + reg = root / known.REGISTRY + issues = known.load(root) if reg.is_file() else [] + measured, unreadable = measured_rows(root, lock) + current = current_versions(root, lock) + return {"schema": SCHEMA, "measured": measured, "current": current, + "queue": queue(root, measured, current, issues), "not_machine_readable": unreadable} + + +TIERS = {1: "ledger-rules (drives optimizer defaults)", 2: "cited by a docs claim", 3: "other"} + + +def cell(text): + return str(text).replace("|", "\\|") + + +def markdown(data): + out = ["## Ledger re-measure queue", "", + "Read-only. SUPPORTED rows of EXPERIMENTS.md whose recorded versions are a minor or major behind " + "today's pins, or inside a known issue's affected range. A re-measure is a new pre-registered row; " + "nothing here edits the ledger.", ""] + if data["queue"]: + out += ["| # | Row | Why first | Reason | Suggested row |", "|---|-----|-----------|--------|---------------|"] + for n, q in enumerate(data["queue"], 1): + out.append("| %d | %s | %s | %s | %s |" % (n, q["row"], TIERS[q["tier"]], cell(q["reason"]), + q["suggested_followup_id"])) + else: + out.append("No SUPPORTED row is behind today's pins.") + out += ["", "### Measured versions (SUPPORTED rows)", "", "| Row | Measured | In the queue |", "|---|---|---|"] + queued = {q["row"] for q in data["queue"]} + for row in sorted(data["measured"], key=row_key): + comps = "; ".join("%s %s" % kv for kv in sorted(data["measured"][row].items())) + out.append("| %s | %s | %s |" % (row, cell(comps), "yes" if row in queued else "no")) + out += ["", "Current pins: " + "; ".join("%s %s" % kv for kv in sorted(data["current"].items())), ""] + if data["not_machine_readable"]: + out += ["### Not machine-readable", ""] + out += ["- %s: %s" % (r["row"], r["reason"]) for r in data["not_machine_readable"]] + return "\n".join(out).rstrip() + "\n" + + +def main(argv): + ap = argparse.ArgumentParser(prog="ledger-freshness", description="Read-only ledger re-measure queue.") + ap.add_argument("--root", default=str(HERE.parent), help="repo root (default: this repo)") + mode = ap.add_mutually_exclusive_group(required=True) + mode.add_argument("--json", action="store_true") + mode.add_argument("--md", action="store_true") + args = ap.parse_args(argv) + data = build(args.root) + if args.json: + print(json.dumps(data, indent=1, sort_keys=True)) + else: + sys.stdout.write(markdown(data)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/hack/pinver.py b/hack/pinver.py new file mode 100755 index 00000000..b89506dc --- /dev/null +++ b/hack/pinver.py @@ -0,0 +1,477 @@ +#!/usr/bin/env python3 +"""pinver: version schemes, ranges and update types for the pins k3d-gpu tracks. + +A library (hack/known-issues, hack/ledger-freshness and the freshness tooling +import it) and a small CLI. Python standard library only. + +Schemes: + semver 1.2.3, v1.2.3, 1.2 (padded); a prerelease (-rc.1, rc1) sorts + before its release; build metadata (+x) is ignored. Breaking: major. + zerover semver, but while the major is 0 a minor is breaking (vLLM 0.x). + k8s v1.37.1+k3s1 or v1.37.1-k3s1, -rcN prereleases; the distro build + (k3s1, k3s2) orders after the patch. Breaking: minor and major. + llama llama.cpp: build tags bNNNNN (server-b11960, full-b11960 read as + b11960), compared numerically, any change a non-breaking minor; + and vX.Y.Z releases under zerover rules. A build and a release + are incomparable. + calver 2026.4.1 (2026.4 = 2026.4.0); year = major. Breaking: major. + pep440 Python versions: epoch, release, aN/bN/rcN, .postN, .devN, +local. + A range without a local label ignores the candidate's. Breaking: major. + docker-suffix numeric core with a prefix and suffix that must match: + 3.14.8-slim, contrib-v1.40.0. Tags of another flavour are + incomparable (latest() skips them); a range reads the core + only. Breaking: major. + eat Intel EAT release tags: release-1.1.0, and release-1.0.3-hotfix + (after release-1.0.3), one stream. Breaking: major. + apt Debian versions (dpkg order: epoch, upstream, revision, ~ before + the end). Breaking: major of the upstream's numeric prefix. + hf-sha a Hugging Face revision: no ordering; ranges are * and ==/!= only. + +API (stable; later tooling imports it): + parse(scheme, v) -> Version compare(scheme, a, b) -> -1 | 0 | 1 + comparable(scheme, a, b) -> bool is_prerelease(scheme, v) -> bool + in_range(scheme, v, spec) -> bool check_range(scheme, spec) (raises ValueError) + update_type(scheme, old, new) -> "digest" | "patch" | "minor" | "major" + breaking(scheme, old, new) -> bool classify(scheme, old, new) -> {type, breaking} + latest(scheme, current, candidates) -> newest stable comparable candidate + above current, or None + +update_type reports where two versions differ ("digest" when they are the same +version); breaking applies the scheme's rule, so a zerover 0.24 -> 0.25 is a +"minor" that is breaking. Range spec: "*", or clauses joined by "||", each a +comma-separated list of <, <=, >, >=, ==, != comparators ("<0.25.0", +">=0.24.0,<0.25.0", "<1.95.1 || >=1.96.0,<1.96.2"). Bad input raises ValueError. + +CLI: + pinver.py parse SCHEME V JSON of the parsed version + pinver.py compare SCHEME A B -1, 0 or 1 + pinver.py in-range SCHEME V SPEC exit 0 inside, 1 outside + pinver.py update-type SCHEME OLD NEW "minor breaking", "patch", ... + pinver.py latest SCHEME CURRENT CAND... the newest stable candidate, or nothing +Exit 2 on bad input. +""" + +import functools +import json +import re +import sys + +SCHEMES = ("semver", "zerover", "k8s", "llama", "calver", "pep440", "docker-suffix", "eat", "apt", "hf-sha") +TYPES = ("digest", "patch", "minor", "major") + + +class Version: + """A parsed version. key orders versions of one flavour; release is the + numeric (major, minor, patch, ...) tuple update_type reads.""" + + __slots__ = ("scheme", "raw", "flavour", "key", "release", "pre", "local") + + def __init__(self, scheme, raw, flavour, key, release, pre, local=None): + self.scheme, self.raw, self.flavour, self.key = scheme, raw, flavour, key + self.release, self.pre, self.local = release, pre, local + + def as_dict(self): + return {"scheme": self.scheme, "raw": self.raw, "flavour": list(self.flavour) if self.flavour else None, + "release": list(self.release), "prerelease": self.pre} + + def __repr__(self): + return "Version(%s %r)" % (self.scheme, self.raw) + + +def _bad(scheme, v, why="not a version"): + return ValueError("%s: %r is %s" % (scheme, v, why)) + + +def _pad(nums, n=3): + nums = tuple(nums) + return nums + (0,) * (n - len(nums)) if len(nums) < n else nums + + +# --- semver / zerover ----------------------------------------------------------- + +SEMVER_RE = re.compile( + r"^v?(\d+)(?:\.(\d+))?(?:\.(\d+))?" + r"(?:-([0-9A-Za-z][0-9A-Za-z.-]*)|((?:rc|alpha|beta|a|b|dev|pre)\.?\d+[0-9A-Za-z.]*))?" + r"(?:\+[0-9A-Za-z][0-9A-Za-z.-]*)?$") + + +def _pre_ids(text): + out = [] + for part in re.split(r"[.-]", text): + if not part: + raise ValueError("empty prerelease identifier") + out.append((0, int(part), "") if part.isdigit() else (1, 0, part)) + return tuple(out) + + +def _semver(scheme, v): + m = SEMVER_RE.match(v) + if not m: + raise _bad(scheme, v) + rel = _pad(int(x) for x in m.group(1, 2, 3) if x is not None) + pre_text = m.group(4) or m.group(5) + try: + pre_key = (0,) + _pre_ids(pre_text) if pre_text else (1,) + except ValueError: + raise _bad(scheme, v) from None + return Version(scheme, v, ("",), (rel, pre_key), rel, bool(pre_text)) + + +# --- k8s ----------------------------------------------------------------------- + +K8S_RE = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)(?:-(alpha|beta|rc)\.?(\d+))?(?:[+-]([a-z][a-z0-9]*?)(\d+))?$") +K8S_PRE = {"alpha": 0, "beta": 1, "rc": 2} + + +def _k8s(scheme, v): + m = K8S_RE.match(v) + if not m: + raise _bad(scheme, v) + rel = tuple(int(x) for x in m.group(1, 2, 3)) + pre = (0, K8S_PRE[m.group(4)], int(m.group(5))) if m.group(4) else (1, 0, 0) + distro = m.group(6) or "" + build = int(m.group(7)) if m.group(7) else 0 + return Version(scheme, v, (distro,), (rel, pre, build), rel, bool(m.group(4))) + + +# --- llama.cpp ----------------------------------------------------------------- + +LLAMA_BUILD_RE = re.compile(r"^(?:[a-z][a-z0-9]*(?:-[a-z0-9]+)*-)?b(\d+)$") + + +def _llama(scheme, v): + m = LLAMA_BUILD_RE.match(v) + if m: + n = int(m.group(1)) + return Version(scheme, v, ("build",), (n,), (n,), False) + if not re.match(r"^v\d+\.\d+\.\d+$", v): + raise _bad(scheme, v) + s = _semver(scheme, v) + return Version(scheme, v, ("release",), s.key, s.release, s.pre) + + +# --- calver -------------------------------------------------------------------- + +CALVER_RE = re.compile(r"^(\d{4})\.(\d+)(?:\.(\d+))?$") + + +def _calver(scheme, v): + m = CALVER_RE.match(v) + if not m: + raise _bad(scheme, v) + rel = _pad(int(x) for x in m.group(1, 2, 3) if x is not None) + return Version(scheme, v, ("",), (rel,), rel, False) + + +# --- pep440 -------------------------------------------------------------------- + +PEP440_RE = re.compile( + r"^v?(?:(\d+)!)?(\d+(?:\.\d+)*)" + r"(?:[-_.]?(a|b|rc|c|alpha|beta|pre|preview)[-_.]?(\d+)?)?" + r"(?:-(\d+)|[-_.]?(post|rev|r)[-_.]?(\d+)?)?" + r"(?:[-_.]?(dev)[-_.]?(\d+)?)?" + r"(?:\+([a-z0-9]+(?:[-_.][a-z0-9]+)*))?$", re.I) +PEP440_PRE = {"a": 0, "alpha": 0, "b": 1, "beta": 1, "c": 2, "rc": 2, "pre": 2, "preview": 2} + + +def _pep440(scheme, v): + m = PEP440_RE.match(v.strip()) + if not m: + raise _bad(scheme, v) + epoch = int(m.group(1) or 0) + release = tuple(int(x) for x in m.group(2).split(".")) + trimmed = release + while len(trimmed) > 1 and trimmed[-1] == 0: + trimmed = trimmed[:-1] + pre_l, pre_n = m.group(3), m.group(4) + post_n = m.group(5) or (m.group(7) or "0" if m.group(6) else None) + dev = m.group(8) is not None + dev_n = int(m.group(9) or 0) + if pre_l: + pre = (PEP440_PRE[pre_l.lower()], int(pre_n or 0)) + elif post_n is None and dev: + pre = (-1, 0) # 1.0.dev1 sorts before 1.0a1 + else: + pre = (9, 0) + post = (int(post_n),) if post_n is not None else (-1,) + devk = (0, dev_n) if dev else (1, 0) + local = None + lockey = (0,) + if m.group(10): + local = m.group(10).lower() + segs = tuple((1, int(s), "") if s.isdigit() else (0, 0, s) for s in re.split(r"[-_.]", local)) + lockey = (1, segs) + key = (epoch, trimmed, pre, post, devk, lockey) + return Version(scheme, v, ("",), key, _pad(release), bool(pre_l) or dev, local) + + +# --- docker-suffix ------------------------------------------------------------- + +DOCKER_RE = re.compile(r"^(\D*)(\d+(?:\.\d+)*)(.*)$") +DOCKER_PRE_RE = re.compile(r"^[-.]?(?:rc|alpha|beta|a|b|dev)\d", re.I) + + +def _docker(scheme, v): + m = DOCKER_RE.match(v) + if not m or not v: + raise _bad(scheme, v) + prefix, core, suffix = m.groups() + nums = tuple(int(x) for x in core.split(".")) + rel = _pad(nums) + return Version(scheme, v, (prefix, suffix), (_pad(nums, 8),), rel, bool(DOCKER_PRE_RE.match(suffix))) + + +# --- eat ----------------------------------------------------------------------- + +EAT_RE = re.compile(r"^release-(\d+)\.(\d+)\.(\d+)(-hotfix)?$") + + +def _eat(scheme, v): + m = EAT_RE.match(v) + if not m: + raise _bad(scheme, v) + rel = tuple(int(x) for x in m.group(1, 2, 3)) + return Version(scheme, v, ("",), (rel, bool(m.group(4))), rel, False) + + +# --- apt (dpkg) ---------------------------------------------------------------- + +APT_RE = re.compile(r"^(?:(\d+):)?([0-9][A-Za-z0-9.+~:-]*)$") + + +def _dpkg_order(c): + if c == "~": + return -1 + if c.isalpha(): + return ord(c) + return ord(c) + 256 + + +def _verrevcmp(a, b): + """dpkg's verrevcmp: alternating non-digit and digit runs.""" + i = j = 0 + while i < len(a) or j < len(b): + first_diff = 0 + while (i < len(a) and not a[i].isdigit()) or (j < len(b) and not b[j].isdigit()): + ac = _dpkg_order(a[i]) if i < len(a) and not a[i].isdigit() else 0 + bc = _dpkg_order(b[j]) if j < len(b) and not b[j].isdigit() else 0 + if ac != bc: + return -1 if ac < bc else 1 + i += 1 + j += 1 + while i < len(a) and a[i] == "0": + i += 1 + while j < len(b) and b[j] == "0": + j += 1 + while i < len(a) and a[i].isdigit() and j < len(b) and b[j].isdigit(): + if not first_diff: + first_diff = (a[i] > b[j]) - (a[i] < b[j]) + i += 1 + j += 1 + if i < len(a) and a[i].isdigit(): + return 1 + if j < len(b) and b[j].isdigit(): + return -1 + if first_diff: + return first_diff + return 0 + + +_DPKG_KEY = functools.cmp_to_key(_verrevcmp) + + +def _apt(scheme, v): + m = APT_RE.match(v) + if not m: + raise _bad(scheme, v) + epoch = int(m.group(1) or 0) + rest = m.group(2) + upstream, revision = (rest.rsplit("-", 1) if "-" in rest else (rest, "0")) + if not upstream or not upstream[0].isdigit() or not revision: + raise _bad(scheme, v) + nums = re.match(r"\d+(?:\.\d+)*", upstream).group(0) + rel = _pad(int(x) for x in nums.split(".")) + key = (epoch, _DPKG_KEY(upstream), _DPKG_KEY(revision)) + return Version(scheme, v, ("",), key, rel, "~" in upstream) + + +# --- hf-sha -------------------------------------------------------------------- + +def _hf(scheme, v): + if not re.match(r"^[0-9a-f]{7,40}$", v): + raise _bad(scheme, v, "not a hex revision") + return Version(scheme, v, ("",), None, (), False) + + +PARSERS = {"semver": _semver, "zerover": _semver, "k8s": _k8s, "llama": _llama, "calver": _calver, + "pep440": _pep440, "docker-suffix": _docker, "eat": _eat, "apt": _apt, "hf-sha": _hf} + + +def parse(scheme, v): + if scheme not in PARSERS: + raise ValueError("unknown scheme %r (want one of %s)" % (scheme, ", ".join(SCHEMES))) + if not isinstance(v, str): + raise _bad(scheme, v, "not a string") + return PARSERS[scheme](scheme, v.strip()) + + +def _parsed(scheme, v): + return v if isinstance(v, Version) else parse(scheme, v) + + +def comparable(scheme, a, b): + try: + a, b = _parsed(scheme, a), _parsed(scheme, b) + except ValueError: + return False + return scheme != "hf-sha" and a.flavour == b.flavour + + +def compare(scheme, a, b): + pa, pb = _parsed(scheme, a), _parsed(scheme, b) + if scheme == "hf-sha": + raise ValueError("hf-sha: revisions have no order (%r, %r)" % (pa.raw, pb.raw)) + if pa.flavour != pb.flavour: + raise ValueError("%s: %r and %r are different flavours and do not compare" % (scheme, pa.raw, pb.raw)) + return (pa.key > pb.key) - (pa.key < pb.key) + + +def is_prerelease(scheme, v): + return _parsed(scheme, v).pre + + +# --- ranges -------------------------------------------------------------------- + +CMP_RE = re.compile(r"^(<=|>=|==|!=|<|>)\s*(\S+)$") +OPS = {"<": lambda c: c < 0, "<=": lambda c: c <= 0, ">": lambda c: c > 0, + ">=": lambda c: c >= 0, "==": lambda c: c == 0, "!=": lambda c: c != 0} + + +def _clauses(scheme, spec): + """[[(op, Version), ...], ...] of a spec; [] for "*".""" + if not isinstance(spec, str) or not spec.strip(): + raise ValueError("%s: empty range" % scheme) + if spec.strip() == "*": + return [] + out = [] + for clause in spec.split("||"): + comps = [] + for part in clause.split(","): + m = CMP_RE.match(part.strip()) + if not m: + raise ValueError("%s: bad comparator %r in range %r" % (scheme, part.strip(), spec)) + op = m.group(1) + if scheme == "hf-sha" and op not in ("==", "!="): + raise ValueError("hf-sha: only ==, != and * ranges (got %r)" % spec) + comps.append((op, parse(scheme, m.group(2)))) + out.append(comps) + return out + + +def check_range(scheme, spec): + _clauses(scheme, spec) + + +def _range_cmp(scheme, v, bound): + if scheme == "hf-sha": + same = v.raw.startswith(bound.raw) or bound.raw.startswith(v.raw) + return 0 if same else 1 + if scheme == "docker-suffix": + return (v.key > bound.key) - (v.key < bound.key) + if scheme == "pep440" and v.local is not None and bound.local is None: + k = v.key[:-1] + ((0,),) + return (k > bound.key) - (k < bound.key) + return compare(scheme, v, bound) + + +def in_range(scheme, v, spec): + pv = _parsed(scheme, v) + clauses = _clauses(scheme, spec) + if not clauses: + return True + return any(all(OPS[op](_range_cmp(scheme, pv, bound)) for op, bound in clause) for clause in clauses) + + +# --- update types -------------------------------------------------------------- + +def update_type(scheme, old, new): + po, pn = _parsed(scheme, old), _parsed(scheme, new) + if scheme == "hf-sha": + return "digest" + if compare(scheme, po, pn) == 0: + return "digest" + if scheme == "llama" and po.flavour == ("build",): + return "minor" + ro, rn = _pad(po.release), _pad(pn.release) + if ro[0] != rn[0]: + return "major" + if ro[1] != rn[1]: + return "minor" + return "patch" + + +def breaking(scheme, old, new): + po, pn = _parsed(scheme, old), _parsed(scheme, new) + t = update_type(scheme, po, pn) + if t == "major": + return True + if t != "minor": + return False + if scheme == "k8s": + return True + zero = scheme == "zerover" or (scheme == "llama" and po.flavour == ("release",)) + return zero and po.release[0] == 0 + + +def classify(scheme, old, new): + return {"type": update_type(scheme, old, new), "breaking": breaking(scheme, old, new)} + + +def latest(scheme, current, candidates): + cur = _parsed(scheme, current) + best = None + for c in candidates: + try: + pc = parse(scheme, c) + except ValueError: + continue + if pc.pre or not comparable(scheme, cur, pc) or compare(scheme, pc, cur) <= 0: + continue + if best is None or compare(scheme, pc, best) > 0: + best = pc + return best.raw if best else None + + +# --- CLI ----------------------------------------------------------------------- + +def main(argv): + usage = __doc__.split("CLI:", 1)[1].split("Exit 2", 1)[0] + if len(argv) < 2 or argv[0] in ("-h", "--help"): + print("usage:" + usage, file=sys.stderr) + return 2 + cmd, scheme, args = argv[0], argv[1], argv[2:] + try: + if cmd == "parse" and len(args) == 1: + print(json.dumps(parse(scheme, args[0]).as_dict(), sort_keys=True)) + elif cmd == "compare" and len(args) == 2: + print(compare(scheme, *args)) + elif cmd == "in-range" and len(args) == 2: + return 0 if in_range(scheme, *args) else 1 + elif cmd == "update-type" and len(args) == 2: + c = classify(scheme, *args) + print(c["type"] + (" breaking" if c["breaking"] else "")) + elif cmd == "latest" and args: + got = latest(scheme, args[0], args[1:]) + if got: + print(got) + else: + print("usage:" + usage, file=sys.stderr) + return 2 + except ValueError as e: + print("pinver: %s" % e, file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/images.unpinned b/images.unpinned index a6785cf0..98180692 100644 --- a/images.unpinned +++ b/images.unpinned @@ -29,3 +29,4 @@ tests/image_test.sh * node tests/model_share_test.sh localhost:5000/agent checks a registry port is not read as a tag tests/image_pins_test.sh * the pin gate's own unpinned inputs tests/eat_test.sh * stubs of EAT's gateway chart files, the input gateway-charts.py pins +tests/ledger_freshness_test.py cryptoandcoffee/k3d-gpu:0.1.70 a fixture ledger row's node image the re-measure queue parses, never pulled diff --git a/pins/known-issues.toml b/pins/known-issues.toml new file mode 100644 index 00000000..c0c587a4 --- /dev/null +++ b/pins/known-issues.toml @@ -0,0 +1,219 @@ +# Known upstream issues k3d-gpu depends on: bugs, behaviour changes and security +# advisories, each with the release that fixes it. hack/known-issues reads this +# file (Python tomllib); `hack/known-issues check` runs in CI, offline. +# +# Fields of each [[issue]]: +# id lowercase id; markers name it +# title one line +# kind bug | change | advisory +# upstream "owner/repo#n" (an issue or a pull request), or "" +# fix {} when unknown, or one of: +# {repo, sha and/or pr} the commit or pull request that fixes it +# {repo, path, contains} fixed in the first release whose file +# `path` contains `contains` +# {ecosystem, package} GitHub advisory database (kind advisory) +# component what the sites pin (vllm-openai-cpu, kserve, litellm, ...) +# scheme a hack/pinver.py scheme for the site versions and the fields below +# affected a pinver range of affected versions ("*" until the fix is known) +# fixed_in the first release with the fix; "" until known. Must lie outside +# `affected` (narrow `affected` when setting it). +# adopt review | auto. auto is owner consent to take a patch or minor fix +# without review; a major always needs review. +# sites [{file, regex}]: where k3d-gpu pins the component. Each regex +# (Python re, MULTILINE) has a named group `v` and must match at +# least once. Files are "ours" or "eat-stock" only (pins/classes). +# workarounds files carrying `known-issue: ` markers (or a +# `known-issue-begin: ` .. `known-issue-end: ` block) for +# code or docs that exist because of the issue. Never in bench/, +# oracle/seed/, console/matrix.v1.json or EXPERIMENTS.md. +# retained {} or {reason, pr}: the workarounds stay after the fix, and why +# ledger_refs [{row, path, regex?}]: ledger rows whose recorded versions the +# issue touches. Informational: reported, never failed, never edited. +# advisories kind advisory: the GHSA ids this entry covers +# image optional image reference template ({tag} = release tag, +# {version} = tag without a leading v); poll checks it resolves +# test optional: a test path or canary: +# notes free text +# +# A check fails when an "ours" site's version is inside `affected` while `fixed_in` +# is set (bump it), or when workarounds remain after every "ours" site reached +# `fixed_in` and `retained` is empty (remove them, or say why they stay). +# `hack/known-issues poll` (network, read-only) finds fixed_in for open entries. + +[[issue]] +id = "vllm-45844" +title = "vLLM CPU attention sizes its split-KV scratchpad too small (Gemma 4 global layers stall, EngineDeadError)" +kind = "bug" +upstream = "vllm-project/vllm#45844" +fix = {repo = "vllm-project/vllm", sha = "26eb87204d58d9d8a25c378fbaec9da23db3fb85", pr = 45844} +component = "vllm-openai-cpu" +scheme = "zerover" +affected = "<0.25.0" +fixed_in = "0.25.0" +adopt = "review" +image = "vllm/vllm-openai-cpu:{tag}" +sites = [ + {file = "scripts/k3d-gpu", regex = '^AMX_TEST_IMAGE="\$\{K3D_GPU_AMX_TEST_IMAGE:-vllm/vllm-openai-cpu:v(?P\d+\.\d+\.\d+)@sha256:'}, + {file = "console/moe_predict.py", regex = '"vllm/vllm-openai-cpu:v(?P\d+\.\d+\.\d+)@sha256:'}, + {file = "integrations/eat/vllm-values.yaml", regex = 'repository: vllm/vllm-openai-cpu\n\s+tag: "v(?P\d+\.\d+\.\d+)@sha256:'}, +] +workarounds = [] +ledger_refs = [ + {row = "H54a", path = "bench/h54/h54.py", regex = '"vllm024": "vllm/vllm-openai-cpu:v(?P\d+\.\d+\.\d+)@'}, + {row = "H53", path = "bench/eatflow/live.sh", regex = 'B1_TAG=\$\{LIVE_EAT_B1_TAG:-v(?P\d+\.\d+\.\d+)@'}, +] +notes = """The B1g stall (EXPERIMENTS.md, H54 note N2). Merged 2026-07-04 as 26eb8720; GitHub's \ +compare puts it in v0.25.0 (behind 106, ahead 0) and not in v0.24.0 (diverged). EAT \ +release-1.0.3-hotfix pins v0.24.0; EAT release-1.1.0 moves to v0.28.0. The ledger arms stay \ +on v0.24.0: changing them takes a dated amendment.""" + +[[issue]] +id = "vllm-50801" +title = "vLLM no longer reads VLLM_CPU_SGL_KERNEL (CPU kernel dispatch refined)" +kind = "change" +upstream = "vllm-project/vllm#50801" +fix = {repo = "vllm-project/vllm", sha = "c8602c79062440074a018c1d5f875a5571eb6881", pr = 50801} +component = "vllm-openai-cpu" +scheme = "zerover" +affected = "<0.28.0" +fixed_in = "0.28.0" +adopt = "review" +sites = [ + {file = "integrations/eat/vllm-values.yaml", regex = 'repository: vllm/vllm-openai-cpu\n\s+tag: "v(?P\d+\.\d+\.\d+)@sha256:'}, +] +workarounds = [] +notes = """EAT's xeon-values.yaml sets VLLM_CPU_SGL_KERNEL=1 (integrations/eat/README.md); from \ +v0.28.0 on it is a no-op, so an EAT image at or past v0.28.0 runs without the SGL kernels it \ +asks for. k3d-gpu's own vLLM CPU path does not set it.""" + +[[issue]] +id = "vllm-pr-51493" +title = "vLLM's V1 model runner fails Gemma 4 MoE MTP drafting (pre_projection shape mismatch); VLLM_USE_V2_MODEL_RUNNER=1 works around it" +kind = "bug" +upstream = "vllm-project/vllm#51493" +fix = {repo = "vllm-project/vllm", pr = 51493} +component = "vllm" +scheme = "zerover" +affected = "*" +fixed_in = "" +adopt = "review" +sites = [ + {file = "console/model_opt.py", regex = 'o\.get\("vllm_version"\) or "(?P\d+\.\d+\.\d+)"'}, +] +workarounds = ["console/model_opt.py"] +notes = """The optimizer's MoE drafter note points at VLLM_USE_V2_MODEL_RUNNER=1 until the pull \ +request is merged and released.""" + +[[issue]] +id = "linux-npu-driver-56" +title = "Intel's Linux NPU driver has no WSL2 support" +kind = "bug" +upstream = "intel/linux-npu-driver#56" +fix = {} +component = "linux-npu-driver" +scheme = "semver" +affected = "*" +fixed_in = "" +adopt = "review" +sites = [ + {file = "scripts/k3d-gpu", regex = '^HS_NPU_REL=v(?P\d+\.\d+\.\d+)$'}, +] +workarounds = ["scripts/k3d-gpu", "docs/first-run.md", "docs/quickstart.md"] +notes = """quickstart's WSL2 guidance and the WSL2 docs say the NPU is unavailable there.""" + +[[issue]] +id = "kserve-4592" +title = "KServe's kserve-openvino ClusterServingRuntime is in the source tree but in no release's runtimes chart" +kind = "bug" +upstream = "kserve/kserve#4592" +fix = {repo = "kserve/kserve", path = "charts/kserve-runtime-configs/files/runtimes/resources.yaml", contains = "name: kserve-openvino"} +component = "kserve" +scheme = "semver" +affected = "*" +fixed_in = "" +adopt = "review" +image = "ghcr.io/kserve/charts/kserve-resources:{tag}" +sites = [ + {file = "examples/kserve/install.sh", regex = '^KSERVE_VERSION=v(?P\d+\.\d+\.\d+)$'}, +] +workarounds = ["examples/kserve/openvino-runtime.yaml", "examples/kserve/install.sh"] +notes = """kserve#4592 (merged 2026-01-16) added config/runtimes/kserve-openvino.yaml, and every \ +release since contains that commit, but neither config/runtimes/kustomization.yaml nor the \ +kserve-runtime-configs chart lists the runtime (v0.21.0 and master, read 2026-10-29). Fixed in \ +the first release whose runtimes chart names kserve-openvino; until then the example vendors \ +the runtime.""" + +[[issue]] +id = "eat-9" +title = "EAT's vLLM chart default (apisix.enabled: false) serves with no authentication" +kind = "bug" +upstream = "intel/enterprise-agent-toolkit#9" +fix = {} +component = "enterprise-agent-toolkit" +scheme = "eat" +affected = "*" +fixed_in = "" +adopt = "review" +sites = [ + {file = "integrations/eat/eat-up.sh", regex = '^EAT_TAG="(?Prelease-[^"]+)"$'}, +] +workarounds = ["console/audit.py"] +notes = """`k3d-gpu audit` fails any unauthenticated listener off loopback, the EAT#9 class. The \ +rule is general; when EAT ships the fix, keep it with `retained`.""" + +[[issue]] +id = "litellm-1.83.14-advisories" +title = "LiteLLM 1.83.14 is affected by published security advisories" +kind = "advisory" +upstream = "" +fix = {ecosystem = "pip", package = "litellm"} +component = "litellm" +scheme = "pep440" +affected = "<1.100.4 || >=1.101.0,<1.101.3 || >=1.102.0,<1.102.2 || >=1.103.0,<1.103.1 || >=1.104.0rc1,<1.104.0rc2" +fixed_in = "1.100.4" +adopt = "review" +advisories = [ + "GHSA-4xpc-pv4p-pm3w", + "GHSA-4jcj-7x88-m979", + "GHSA-7488-6r32-c95q", + "GHSA-3cv6-jpf6-8222", + "GHSA-hhww-mrg2-969h", + "GHSA-g5ff-637f-6q2m", + "GHSA-7hp6-4w63-5g45", +] +sites = [ + {file = "integrations/eat/gateway-values.yaml", regex = 'repository: docker\.litellm\.ai/berriai/litellm\n\s+tag: "v(?P\d+\.\d+\.\d+)-'}, + {file = "examples/agents/litellm-gateway/requirements.txt", regex = '^litellm==(?P\S+)$'}, + {file = "examples/agents/litellm-gateway/constraints.txt", regex = '^litellm==(?P\S+)$'}, +] +workarounds = [] +notes = """Read from the GitHub advisory database 2026-10-29: fixed in 1.84.0 (GHSA-4xpc, -4jcj, \ +-7488), 1.85.0 (GHSA-hhww), 1.88.6 with backports up to 1.96.2 (GHSA-3cv6), 1.95.0 (GHSA-g5ff) \ +and 1.100.4 with backports 1.101.3, 1.102.2 and 1.103.1 (GHSA-7hp6, high, published 2026-10-27; \ +it affects 1.91.0 through 1.100.3, so not 1.83.14 itself, but 1.95.1 and 1.96.2). 1.100.4 is the \ +first release outside every range, and the advisory database lists none for it. Both sites are \ +EAT's stock gateway, so the check reports and does not fail.""" + +[[issue]] +id = "vllm-0.32-num-spec-tokens" +title = "vLLM v0.32.0 compares num_speculative_tokens before defaulting it (gemma4_mtp drafter with no length exits with a TypeError)" +kind = "bug" +upstream = "" +fix = {} +component = "vllm" +scheme = "zerover" +affected = ">=0.32.0" +fixed_in = "" +adopt = "review" +sites = [ + {file = "console/model_opt.py", regex = 'o\.get\("vllm_version"\) or "(?P\d+\.\d+\.\d+)"'}, +] +workarounds = [] +ledger_refs = [ + {row = "H54q", path = "bench/h54/h54.py"}, +] +notes = """File upstream: config/speculative.py rewrites a gemma4_assistant drafter to gemma4_mtp \ +with n_predict 1, then reaches the MTP branch's num_speculative_tokens > 1 check before the line \ +that defaults num_speculative_tokens to n_predict ("TypeError: '>' not supported between \ +instances of 'NoneType' and 'int'"). H54 amendment R2 names the length (1). The optimizer always \ +names num_speculative_tokens, so its plans do not hit it.""" diff --git a/scripts/k3d-gpu b/scripts/k3d-gpu index efd27ea6..d439a0a5 100755 --- a/scripts/k3d-gpu +++ b/scripts/k3d-gpu @@ -10206,7 +10206,8 @@ qs_step() { echo "${GRN}==>${RST} [$1/${QS_STEPS}] $(progress_bar "$(($1 - 1))" # paravirtualization for NVIDIA only (https://docs.docker.com/desktop/features/gpu/); # the Intel path is /dev/dxg plus /usr/lib/wsl, as in OpenVINO's container recipe # (https://github.com/openvinotoolkit/docker_ci/blob/master/docs/accelerators.md). -# The Intel NPU has no WSL2 driver (https://github.com/intel/linux-npu-driver/issues/56). +# The Intel NPU has no WSL2 driver (https://github.com/intel/linux-npu-driver/issues/56; +# known-issue: linux-npu-driver-56). # agent_device requests gpu.intel.com/xe, i915, a Gaudi or the CPU, never # gpu.intel.com/dxg, so under WSL2 quickstart's LLM runs on the CPU. # Host changes are printed as suggestions, never run. @@ -10227,7 +10228,9 @@ qs_wsl_guidance() { fi echo " Docker: Docker Desktop with WSL integration enabled for this distro, or Docker Engine inside it." echo " Docker Desktop's --gpus flag is NVIDIA-only; the Intel GPU goes through /dev/dxg instead." + # known-issue-begin: linux-npu-driver-56 echo " The Intel NPU is not reachable from WSL2 (intel/linux-npu-driver#56)." + # known-issue-end: linux-npu-driver-56 } # Pid of the port-forward quickstart started, while it runs. A pid whose diff --git a/tests/claims_check.sh b/tests/claims_check.sh index 67ea642e..0db9f460 100755 --- a/tests/claims_check.sh +++ b/tests/claims_check.sh @@ -21,10 +21,16 @@ # docs/provenance.md is the source of truth for cited numbers and is not scanned # either. A file the guard cannot read is an error. # -# Usage: tests/claims_check.sh [repo-root] +# --cited prints, instead of checking, the ids of the ledger rows that claims cite +# (a [ledger:Hn] reference on a claim line, or on either line of a claim split over +# two lines), sorted, one per line, and exits 0. hack/ledger-freshness reads it. +# +# Usage: tests/claims_check.sh [--cited] [repo-root] set -uo pipefail +CITED=0 +if [ "${1:-}" = "--cited" ]; then CITED=1; shift; fi ROOT=${1:-"$(cd "$(dirname "$0")/.." && pwd)"} LEDGER="${ROOT}/EXPERIMENTS.md" PROVENANCE="${ROOT}/docs/provenance.md" @@ -87,7 +93,7 @@ report="" for f in "${files[@]}"; do rel=${f#"${ROOT}/"} [ -r "${f}" ] || read_error "${f}" - out=$(awk -v rel="${rel}" -v ledger="${ledger}" -v prov="${provenance}" ' + out=$(awk -v rel="${rel}" -v ledger="${ledger}" -v prov="${provenance}" -v cited="${CITED}" ' function load(s, m, n, i, kv, p) { n = split(s, kv, ";") for (i = 1; i <= n; i++) if ((p = index(kv[i], "=")) > 0) m[substr(kv[i], 1, p - 1)] = substr(kv[i], p + 1) @@ -154,13 +160,14 @@ for f in "${files[@]}"; do claim = is_claim(low) - backed = linked; notes = ""; rest = s + backed = linked; notes = ""; rest = s; rows = "" while (match(rest, /\[(ledger|provenance):[A-Za-z0-9]+\]/)) { ref = substr(rest, RSTART, RLENGTH) rest = substr(rest, RSTART + RLENGTH) kind = substr(ref, 2, index(ref, ":") - 2) id = substr(ref, index(ref, ":") + 1); sub(/\]$/, "", id) if (kind == "ledger") { + rows = rows " " id if (!(id in L)) printf "%s:%d: %s not in EXPERIMENTS.md hypothesis table\n", rel, NR, ref else if (L[id] == "SUPPORTED") backed = 1 else notes = notes sprintf("; %s is %s", ref, L[id]) @@ -179,14 +186,22 @@ for f in "${files[@]}"; do } if ((claim || split_claim) && !backed) printf "%s:%d: unbacked claim (no ledger or provenance reference that backs it%s): %s\n", rel, NR, notes, $0 + if (cited && (claim || split_claim)) { + n = split(rows (split_claim ? prevrows : ""), ids, " ") + for (i = 1; i <= n; i++) if (ids[i] != "") printf "cite %s\n", ids[i] + } if (low ~ /^[ \t]*$/) prev = "" - else { prev = low; prevclaim = claim || split_claim; prevbacked = backed } + else { prev = low; prevclaim = claim || split_claim; prevbacked = backed; prevrows = rows } }' "${f}") || read_error "${f}" report+=${out}$'\n' done report=$(sed '/^$/d' <<< "${report}") +if [ "${CITED}" = 1 ]; then + sed -n 's/^cite //p' <<< "${report}" | sort -u + exit 0 +fi if [ -z "${report}" ]; then echo "claims_check: ${#files[@]} file(s) clean" exit 0 diff --git a/tests/claims_check_test.sh b/tests/claims_check_test.sh index 7290299d..2c2df5dd 100755 --- a/tests/claims_check_test.sh +++ b/tests/claims_check_test.sh @@ -228,6 +228,32 @@ mkdir -p "${intel}/docs/intel" printf '%s\n' 'The iGPU serves 42 tok/s.' > "${intel}/docs/intel/report.md" check "an unbacked claim in docs/intel/ fails" "${intel}" fail 'docs/intel/report.md:1' +# --- --cited: the ledger rows docs claims cite (hack/ledger-freshness reads it) ---- +# A row counts when its [ledger:Hn] reference sits on a claim line, or on the +# previous line of a claim split over two lines; a reference without a claim, one +# in a fenced block or one in an HTML comment does not. +cited=$(repo 'The CPU serves 42 tok/s [ledger:H2]. +See [ledger:H3] for context. +' 'Measured in [ledger:H1]: the iGPU decodes 42 +tok/s on this host. + +```text +50 tok/s [ledger:H3] +```') +out=$("${GUARD}" --cited "${cited}" 2>&1); rc=$? +if [ "${rc}" -eq 0 ] && [ "${out}" = $'H1\nH2' ]; then + PASS=$((PASS + 1)); echo "ok --cited lists the rows claim lines cite, sorted, once each" +else + FAILS=$((FAILS + 1)); echo "FAIL --cited lists the rows claim lines cite, sorted, once each" + echo " | rc=${rc} output: ${out//$'\n'/ | }" +fi +out=$("${GUARD}" --cited "$(repo 'An unbacked 42 tok/s claim.')" 2>&1); rc=$? +if [ "${rc}" -eq 0 ] && [ -z "${out}" ]; then + PASS=$((PASS + 1)); echo "ok --cited prints nothing and exits 0 when no claim cites a row" +else + FAILS=$((FAILS + 1)); echo "FAIL --cited prints nothing and exits 0 when no claim cites a row (rc=${rc}: ${out})" +fi + # --- the real repo ----------------------------------------------------------- check "this repo's own docs pass" "$(cd "$(dirname "$0")/.." && pwd)" 0 diff --git a/tests/known_issues_test.py b/tests/known_issues_test.py new file mode 100755 index 00000000..3ed6deb7 --- /dev/null +++ b/tests/known_issues_test.py @@ -0,0 +1,581 @@ +#!/usr/bin/env python3 +"""Tests for hack/known-issues. Each case writes a throwaway repo tree (registry, +site files, markers, a small ledger) and runs `check` or `poll` on it; poll runs +against a stubbed GitHub API. The last cases run check on this repo. + +Usage: python3 tests/known_issues_test.py +""" + +import base64 +import hashlib +import importlib.machinery +import importlib.util +import io +import json +import contextlib +import pathlib +import re +import shutil +import subprocess +import sys +import tempfile +import textwrap +import tomllib +import unittest + +ROOT = pathlib.Path(__file__).resolve().parent.parent +TOOL = ROOT / "hack" / "known-issues" + + +def load_tool(): + loader = importlib.machinery.SourceFileLoader("known_issues", str(TOOL)) + spec = importlib.util.spec_from_loader("known_issues", loader) + mod = importlib.util.module_from_spec(spec) + loader.exec_module(mod) + return mod + + +ki = load_tool() + +LEDGER = """# Experiments + +## Hypotheses + +| # | Hypothesis | Prediction | Probe | Verdict | +|---|------------|------------|-------|---------| +| H53 | stock arm | p | probe | PENDING | +| H54a | engine | p | probe | INCONCLUSIVE | + +## Falsification log +""" + +VLLM = """ +[[issue]] +id = "vllm-fix" +title = "a CPU attention bug" +kind = "bug" +upstream = "vllm-project/vllm#45844" +fix = {repo = "vllm-project/vllm", sha = "26eb87204d58d9d8a25c378fbaec9da23db3fb85"} +component = "vllm-openai-cpu" +scheme = "zerover" +affected = "<0.25.0" +fixed_in = "0.25.0" +adopt = "review" +image = "vllm/vllm-openai-cpu:{tag}" +sites = [ + {file = "scripts/tool.sh", regex = '^CPU_IMAGE="vllm/vllm-openai-cpu:v(?P[0-9.]+)@'}, + {file = "integrations/eat/values.yaml", regex = 'tag: "v(?P[0-9.]+)@'}, +] +workarounds = [] +ledger_refs = [{row = "H53", path = "bench/live.sh", regex = 'B1_TAG=v(?P[0-9.]+)'}] +notes = "" +""" + +OPEN = """ +[[issue]] +id = "open-bug" +title = "an open upstream bug" +kind = "bug" +upstream = "acme/engine#7" +fix = {repo = "acme/engine", pr = 7} +component = "engine" +scheme = "semver" +affected = "*" +fixed_in = "" +adopt = "review" +sites = [{file = "console/opt.py", regex = 'ENGINE = "(?P[0-9.]+)"'}] +workarounds = ["console/opt.py"] +notes = "" +""" + +FILES = { + "EXPERIMENTS.md": LEDGER, + "scripts/tool.sh": 'CPU_IMAGE="vllm/vllm-openai-cpu:v0.32.0@sha256:' + "a" * 64 + '"\n', + "integrations/eat/values.yaml": 'image:\n tag: "v0.24.0@sha256:' + "b" * 64 + '"\n', + "bench/live.sh": "B1_TAG=v0.24.0\n", + "console/opt.py": textwrap.dedent("""\ + ENGINE = "1.4.0" + # known-issue-begin: open-bug + NOTE = "set ENGINE_FALLBACK=1 until the fix ships" + # known-issue-end: open-bug + """), +} + + +def make_tree(registry, files=None, extra=None): + d = pathlib.Path(tempfile.mkdtemp(prefix="known_issues.")) + tree = dict(FILES if files is None else files) + if 'id = "open-bug"' not in registry: + tree.pop("console/opt.py", None) # its markers name open-bug + tree.update(extra or {}) + tree["pins/known-issues.toml"] = registry + for rel, text in tree.items(): + p = d / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(text) + return d + + +def run_check(root): + out = io.StringIO() + with contextlib.redirect_stdout(out): + rc = ki.main(["--root", str(root), "check"]) + return rc, out.getvalue() + + +def run_check_json(root): + out = io.StringIO() + with contextlib.redirect_stdout(out): + rc = ki.main(["--root", str(root), "check", "--json"]) + return rc, json.loads(out.getvalue()) + + +class Check(unittest.TestCase): + def setUp(self): + self.dirs = [] + + def tearDown(self): + for d in self.dirs: + shutil.rmtree(d, ignore_errors=True) + + def tree(self, registry, files=None, extra=None): + d = make_tree(registry, files, extra) + self.dirs.append(d) + return d + + def assertFails(self, root, needle): + rc, out = run_check(root) + self.assertEqual(rc, 1, out) + self.assertIn(needle, out) + return out + + def test_fixture_passes(self): + rc, out = run_check(self.tree(VLLM + OPEN)) + self.assertEqual(rc, 0, out) + + def test_ours_site_in_range_with_fixed_in_fails_with_bump_message(self): + files = dict(FILES) + files["scripts/tool.sh"] = files["scripts/tool.sh"].replace("v0.32.0", "v0.24.0") + self.assertFails(self.tree(VLLM + OPEN, files), "bump scripts/tool.sh to >= 0.25.0") + + def test_in_range_site_with_fixed_in_empty_passes(self): + # open-bug: affected "*" covers console/opt.py's 1.4.0, fixed_in is empty. + rc, data = run_check_json(self.tree(OPEN)) + self.assertEqual(rc, 0) + site = data["issues"][0]["sites"][0] + self.assertEqual((site["versions"], site["in_affected"]), (["1.4.0"], True)) + + def test_eat_stock_in_range_only_reports(self): + rc, data = run_check_json(self.tree(VLLM)) + self.assertEqual(rc, 0) + self.assertTrue(any("integrations/eat/values.yaml" in r and "eat-stock" in r for r in data["reports"]), + data["reports"]) + + def test_ledger_ref_in_range_only_reports(self): + rc, data = run_check_json(self.tree(VLLM)) + self.assertEqual(rc, 0) + self.assertTrue(any("H53" in r and "bench/live.sh" in r for r in data["reports"]), data["reports"]) + ref = data["issues"][0]["ledger_refs"][0] + self.assertEqual((ref["row"], ref["versions"], ref["in_affected"]), ("H53", ["0.24.0"], True)) + + def stale(self, retained=""): + reg = OPEN.replace('fixed_in = ""', 'fixed_in = "1.5.0"').replace('affected = "*"', 'affected = "<1.5.0"') + reg += retained + files = dict(FILES) + files["console/opt.py"] = files["console/opt.py"].replace('"1.4.0"', '"1.5.0"') + return self.tree(reg, files) + + def test_stale_workaround_without_retained_fails(self): + self.assertFails(self.stale(), "open-bug: workaround markers remain") + + def test_stale_workaround_with_retained_passes(self): + rc, out = run_check(self.stale('retained = {reason = "the note helps older engines", pr = "#12"}\n')) + self.assertEqual(rc, 0, out) + + def test_workaround_not_stale_while_an_ours_site_is_behind(self): + reg = OPEN.replace('fixed_in = ""', 'fixed_in = "1.5.0"').replace('affected = "*"', 'affected = "<1.5.0"') + # console/opt.py stays at 1.4.0: the bump message fires, the stale rule does not. + out = self.assertFails(self.tree(reg), "bump console/opt.py to >= 1.5.0") + self.assertNotIn("workaround markers remain", out) + + def test_site_regex_without_match_fails(self): + reg = VLLM.replace("CPU_IMAGE=", "GPU_IMAGE=") + self.assertFails(self.tree(reg), "matches nothing in scripts/tool.sh") + + def test_missing_site_file_fails(self): + reg = VLLM.replace('file = "scripts/tool.sh"', 'file = "scripts/gone.sh"') + self.assertFails(self.tree(reg), "scripts/gone.sh") + + def test_unparseable_site_version_fails(self): + files = dict(FILES) + files["scripts/tool.sh"] = files["scripts/tool.sh"].replace("v0.32.0", "v0.32.x") + reg = VLLM.replace("[0-9.]+)@'}", "[0-9.x]+)@'}", 1) + self.assertFails(self.tree(reg, files), "0.32.x") + + def test_marker_in_unlisted_file_fails(self): + self.assertFails(self.tree(OPEN, extra={"docs/guide.md": "Use the fallback. \n"}), + "docs/guide.md") + + def test_listed_workaround_without_marker_fails(self): + files = dict(FILES) + files["console/opt.py"] = 'ENGINE = "1.4.0"\n' + self.assertFails(self.tree(OPEN, files), "console/opt.py has no known-issue marker for open-bug") + + def test_marker_for_unknown_issue_fails(self): + self.assertFails(self.tree(OPEN, extra={"scripts/x.sh": "# known-issue: no-such-issue\n"}), + "no-such-issue") + + def test_marker_in_ledger_file_fails(self): + reg = OPEN.replace('workarounds = ["console/opt.py"]', 'workarounds = ["console/opt.py", "bench/live.sh"]') + files = dict(FILES) + files["bench/live.sh"] += "# known-issue: open-bug\n" + self.assertFails(self.tree(reg, files), "bench/live.sh") + + def test_unpaired_block_fails(self): + files = dict(FILES) + files["console/opt.py"] = 'ENGINE = "1.4.0"\n# known-issue-begin: open-bug\nNOTE = 1\n' + self.assertFails(self.tree(OPEN, files), "known-issue-begin: open-bug") + + def test_fixed_in_inside_affected_fails(self): + self.assertFails(self.tree(VLLM.replace('affected = "<0.25.0"', 'affected = "<=0.25.0"')), + "fixed_in 0.25.0 is inside affected") + + def test_unknown_key_fails(self): + self.assertFails(self.tree(OPEN.replace('notes = ""', 'notes = ""\nfixed_by = "x"')), "fixed_by") + + def test_missing_field_fails(self): + self.assertFails(self.tree(OPEN.replace('adopt = "review"\n', "")), "adopt") + + def test_bad_enum_and_scheme_fail(self): + self.assertFails(self.tree(OPEN.replace('kind = "bug"', 'kind = "wish"')), "kind") + self.assertFails(self.tree(OPEN.replace('scheme = "semver"', 'scheme = "romver"')), "romver") + + def test_bad_upstream_fails(self): + self.assertFails(self.tree(OPEN.replace('"acme/engine#7"', '"acme/engine/7"')), "upstream") + + def test_duplicate_id_fails(self): + self.assertFails(self.tree(OPEN + OPEN), "duplicate id open-bug") + + def test_ledger_ref_row_must_exist(self): + self.assertFails(self.tree(VLLM.replace('row = "H53"', 'row = "H99"')), "H99") + + def test_site_in_ledger_file_fails(self): + reg = VLLM.replace('file = "scripts/tool.sh", regex = \'^CPU_IMAGE="vllm/vllm-openai-cpu:v(?P[0-9.]+)@\'', + 'file = "bench/live.sh", regex = \'B1_TAG=v(?P[0-9.]+)\'') + self.assertFails(self.tree(reg), "bench/live.sh") + + def test_pins_classes_file_decides_the_class(self): + # pins/classes (first match wins) makes scripts/ eat-stock: the in-range site only reports. + files = dict(FILES) + files["scripts/tool.sh"] = files["scripts/tool.sh"].replace("v0.32.0", "v0.24.0") + rc, out = run_check(self.tree(VLLM, files, {"pins/classes": "scripts/** eat-stock test\n"})) + self.assertEqual(rc, 0, out) + + def test_json_shape(self): + rc, data = run_check_json(self.tree(VLLM + OPEN)) + self.assertEqual(rc, 0) + self.assertEqual(set(data), {"ok", "errors", "reports", "issues"}) + self.assertEqual([i["id"] for i in data["issues"]], ["vllm-fix", "open-bug"]) + self.assertEqual(data["issues"][0]["status"], "fixed") + self.assertEqual(data["issues"][1]["status"], "open") + self.assertEqual(data["issues"][1]["markers"][0]["file"], "console/opt.py") + + +# --- poll ---------------------------------------------------------------------- + +SHA = "26eb87204d58d9d8a25c378fbaec9da23db3fb85" + + +def releases(*tags, pre=()): + return [{"tag_name": t, "prerelease": t in pre, "draft": False, + "html_url": "https://github.com/x/releases/tag/" + t, "published_at": "2026-07-0%dT00:00:00Z" % (i + 1)} + for i, t in enumerate(tags)] + + +class Api: + """GitHub API stub: routes path -> JSON; 404 for anything else. Records calls.""" + + def __init__(self, routes): + self.routes, self.calls = routes, [] + + def __call__(self, path): + self.calls.append(path) + if path in self.routes: + return self.routes[path] + raise ki.ApiError(404, path) + + +def vllm_routes(compare): + r = {"repos/vllm-project/vllm/releases?per_page=100&page=1": + releases("v0.26.0", "v0.25.0", "v0.25.0rc1", "v0.24.0", "v0.23.0", pre=("v0.25.0rc1",)), + "repos/vllm-project/vllm/releases?per_page=100&page=2": []} + for tag, status in compare.items(): + r["repos/vllm-project/vllm/compare/%s...%s" % (tag, SHA)] = {"status": status} + return r + + +class Poll(unittest.TestCase): + def setUp(self): + self.dirs = [] + + def tearDown(self): + for d in self.dirs: + shutil.rmtree(d, ignore_errors=True) + + def poll(self, registry, api, files=None, digests=None): + d = make_tree(registry, files) + self.dirs.append(d) + before = {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in d.rglob("*") if p.is_file()} + resolved = [] + + def digest(ref): + resolved.append(ref) + return (digests or {}).get(ref, "sha256:" + "c" * 64) + + out = ki.poll(ki.load(d), d, api, digest) + after = {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in d.rglob("*") if p.is_file()} + self.assertEqual(before, after, "poll wrote to the tree") + return {o["id"]: o for o in out}, resolved + + SEQ = {"v0.23.0": "ahead", "v0.24.0": "diverged", "v0.25.0": "behind", "v0.26.0": "behind"} + + def test_first_containing_release_walking_up(self): + # Lowest site is 0.24.0 (diverged): walk up, v0.25.0 is behind -> contains the fix. + api = Api(vllm_routes(self.SEQ)) + got, resolved = self.poll(VLLM.replace('fixed_in = "0.25.0"', 'fixed_in = ""').replace('"<0.25.0"', '"*"'), api) + o = got["vllm-fix"] + self.assertEqual((o["fixed_in"], o["tag"], o["state"]), ("0.25.0", "v0.25.0", "released")) + self.assertEqual(o["release_url"], "https://github.com/x/releases/tag/v0.25.0") + self.assertEqual(resolved, ["vllm/vllm-openai-cpu:v0.25.0"]) + self.assertFalse(any("rc1" in c for c in api.calls)) + + def test_first_containing_release_walking_down(self): + # Every site at 0.26.0 (behind): walk down past v0.25.0 (behind) to v0.24.0 (diverged). + files = dict(FILES) + files["integrations/eat/values.yaml"] = files["integrations/eat/values.yaml"].replace("v0.24.0", "v0.26.0") + files["scripts/tool.sh"] = files["scripts/tool.sh"].replace("v0.32.0", "v0.26.0") + got, _ = self.poll(VLLM, Api(vllm_routes(self.SEQ)), files) + o = got["vllm-fix"] + self.assertEqual((o["fixed_in"], o["recorded"], o["mismatch"]), ("0.25.0", "0.25.0", False)) + + def test_recorded_mismatch_is_flagged(self): + seq = dict(self.SEQ, **{"v0.25.0": "diverged"}) + got, _ = self.poll(VLLM, Api(vllm_routes(seq))) + o = got["vllm-fix"] + self.assertEqual((o["fixed_in"], o["mismatch"]), ("0.26.0", True)) + + def test_merged_but_unreleased(self): + seq = {t: "diverged" for t in self.SEQ} + got, resolved = self.poll(VLLM, Api(vllm_routes(seq))) + self.assertEqual((got["vllm-fix"]["state"], got["vllm-fix"]["fixed_in"]), ("merged-unreleased", "")) + self.assertEqual(resolved, []) + + def test_open_pr_gives_empty(self): + api = Api({"repos/acme/engine/pulls/7": {"state": "open", "merged": False, "merge_commit_sha": None}}) + got, _ = self.poll(OPEN, api) + self.assertEqual((got["open-bug"]["state"], got["open-bug"]["fixed_in"]), ("open", "")) + + def test_merged_pr_uses_its_merge_commit(self): + routes = {"repos/acme/engine/pulls/7": {"state": "closed", "merged": True, "merge_commit_sha": "f" * 40}, + "repos/acme/engine/releases?per_page=100&page=1": releases("v1.6.0", "v1.5.0", "v1.4.0"), + "repos/acme/engine/releases?per_page=100&page=2": [], + "repos/acme/engine/compare/v1.4.0..." + "f" * 40: {"status": "ahead"}, + "repos/acme/engine/compare/v1.5.0..." + "f" * 40: {"status": "identical"}} + got, _ = self.poll(OPEN, Api(routes)) + self.assertEqual((got["open-bug"]["fixed_in"], got["open-bug"]["state"]), ("1.5.0", "released")) + + def test_closed_issue_reads_the_closing_commit_from_the_timeline(self): + reg = OPEN.replace('fix = {repo = "acme/engine", pr = 7}', "fix = {}") + routes = {"repos/acme/engine/issues/7": {"state": "closed", "state_reason": "completed"}, + "repos/acme/engine/issues/7/timeline?per_page=100&page=1": [ + {"event": "labeled"}, + {"event": "closed", "commit_id": "e" * 40, + "commit_url": "https://api.github.com/repos/acme/engine/commits/" + "e" * 40}], + "repos/acme/engine/issues/7/timeline?per_page=100&page=2": [], + "repos/acme/engine/releases?per_page=100&page=1": releases("v1.5.0", "v1.4.0"), + "repos/acme/engine/releases?per_page=100&page=2": [], + "repos/acme/engine/compare/v1.4.0..." + "e" * 40: {"status": "behind"}, + "repos/acme/engine/compare/v1.3.0..." + "e" * 40: {"status": "ahead"}} + got, _ = self.poll(reg, Api(routes)) + self.assertEqual(got["open-bug"]["fixed_in"], "1.4.0") + + def test_open_issue_gives_empty(self): + reg = OPEN.replace('fix = {repo = "acme/engine", pr = 7}', "fix = {}") + got, _ = self.poll(reg, Api({"repos/acme/engine/issues/7": {"state": "open"}})) + self.assertEqual((got["open-bug"]["state"], got["open-bug"]["fixed_in"]), ("open", "")) + + def test_content_probe(self): + reg = OPEN.replace('fix = {repo = "acme/engine", pr = 7}', + 'fix = {repo = "acme/engine", path = "charts/runtimes.yaml", contains = "name: ovms"}') + + def content(text): + return {"content": base64.b64encode(text.encode()).decode(), "encoding": "base64"} + + routes = {"repos/acme/engine/releases?per_page=100&page=1": releases("v1.4.0", "v1.3.0"), + "repos/acme/engine/releases?per_page=100&page=2": [], + "repos/acme/engine/contents/charts/runtimes.yaml?ref=v1.4.0": content("name: sklearn\n"), + "repos/acme/engine/contents/charts/runtimes.yaml": content("name: sklearn\n")} + got, _ = self.poll(reg, Api(routes)) + self.assertEqual((got["open-bug"]["state"], got["open-bug"]["fixed_in"]), ("not-shipped", "")) + routes["repos/acme/engine/contents/charts/runtimes.yaml"] = content("name: ovms\n") + got, _ = self.poll(reg, Api(routes)) + self.assertEqual((got["open-bug"]["state"], got["open-bug"]["fixed_in"]), ("merged-unreleased", "")) + routes["repos/acme/engine/releases?per_page=100&page=1"] = releases("v1.5.0", "v1.4.0", "v1.3.0") + routes["repos/acme/engine/contents/charts/runtimes.yaml?ref=v1.5.0"] = content("name: ovms\n") + got, _ = self.poll(reg, Api(routes)) + self.assertEqual((got["open-bug"]["state"], got["open-bug"]["fixed_in"]), ("released", "1.5.0")) + + ADVISORY_ENTRY = textwrap.dedent(""" + [[issue]] + id = "lib-advisories" + title = "advisories" + kind = "advisory" + upstream = "" + fix = {ecosystem = "pip", package = "lib"} + component = "lib" + scheme = "pep440" + affected = "<1.5.0" + fixed_in = "1.5.0" + adopt = "review" + advisories = ["GHSA-aaaa-aaaa-aaaa"] + sites = [{file = "req.txt", regex = '^lib==(?P\\S+)$'}] + workarounds = [] + notes = "" + """) + + @staticmethod + def adv(ghsa, *vulns): + return {"ghsa_id": ghsa, "html_url": "https://github.com/advisories/" + ghsa, + "vulnerabilities": [{"package": {"ecosystem": "pip", "name": "lib"}, + "vulnerable_version_range": r, "first_patched_version": p} for r, p in vulns]} + + def test_advisories(self): + reg, adv = self.ADVISORY_ENTRY, self.adv + routes = {"advisories?ecosystem=pip&affects=lib%401.2.0&per_page=100": [ + adv("GHSA-aaaa-aaaa-aaaa", ("< 1.5.0", "1.5.0")), + adv("GHSA-bbbb-bbbb-bbbb", ("< 1.6.2", "1.6.2"), (">= 1.7.0, < 1.7.3", "1.7.3")), + ], "advisories?ecosystem=pip&affects=lib%401.6.2&per_page=100": []} + got, _ = self.poll(reg, Api(routes), files={"req.txt": "lib==1.2.0\n"}) + o = got["lib-advisories"] + self.assertEqual((o["state"], o["fixed_in"], o["mismatch"]), ("advisory", "1.6.2", True)) + self.assertEqual(o["new_advisories"], ["GHSA-bbbb-bbbb-bbbb"]) + + def test_advisories_of_the_candidate_count_and_versions_below_the_floor_do_not(self): + # The floor's advisories put the fix at 1.5.0 (1.1.5 is a backport below the floor, not a + # fix for it). 1.5.0 has an advisory of its own that 1.2.0 does not, so the fix is 1.6.0. + reg = self.ADVISORY_ENTRY + routes = {"advisories?ecosystem=pip&affects=lib%401.2.0&per_page=100": [ + self.adv("GHSA-aaaa-aaaa-aaaa", ("< 1.1.5", "1.1.5"), (">= 1.2.0, < 1.5.0", "1.5.0")), + ], "advisories?ecosystem=pip&affects=lib%401.5.0&per_page=100": [ + self.adv("GHSA-cccc-cccc-cccc", (">= 1.4.0, < 1.6.0", "1.6.0")), + ], "advisories?ecosystem=pip&affects=lib%401.6.0&per_page=100": []} + got, _ = self.poll(reg, Api(routes), files={"req.txt": "lib==1.2.0\n"}) + o = got["lib-advisories"] + self.assertEqual((o["state"], o["fixed_in"], o["mismatch"]), ("advisory", "1.6.0", True)) + self.assertEqual(o["advisories"], ["GHSA-aaaa-aaaa-aaaa", "GHSA-cccc-cccc-cccc"]) + self.assertEqual(o["new_advisories"], ["GHSA-cccc-cccc-cccc"]) + + def test_a_prerelease_patch_is_not_a_fix(self): + routes = {"advisories?ecosystem=pip&affects=lib%401.2.0&per_page=100": [ + self.adv("GHSA-aaaa-aaaa-aaaa", ("< 1.5.0rc2", "1.5.0rc2")), + ]} + got, _ = self.poll(self.ADVISORY_ENTRY, Api(routes), files={"req.txt": "lib==1.2.0\n"}) + o = got["lib-advisories"] + self.assertNotIn("error", o) + self.assertEqual((o["state"], o["fixed_in"]), ("advisory", "")) + + def test_eat_entry_sees_a_plain_release_after_a_hotfix(self): + # The registry's eat-9 entry, on EAT's real tag stream: release-1.0.3-hotfix is followed by + # release-1.1.0 and release-1.2.0, and the fix lands in release-1.2.0. + with open(ROOT / "pins" / "known-issues.toml", "rb") as f: + entry = next(e for e in tomllib.load(f)["issue"] if e["id"] == "eat-9") + reg = textwrap.dedent(""" + [[issue]] + id = "eat-9" + title = "t" + kind = "bug" + upstream = "acme/toolkit#9" + fix = {} + component = "enterprise-agent-toolkit" + scheme = "%s" + affected = "*" + fixed_in = "" + adopt = "review" + sites = [{file = "eat-up.sh", regex = '%s'}] + workarounds = [] + notes = "" + """) % (entry["scheme"], entry["sites"][0]["regex"]) + sha = "d" * 40 + routes = {"repos/acme/toolkit/issues/9": {"state": "closed", "state_reason": "completed"}, + "repos/acme/toolkit/issues/9/timeline?per_page=100&page=1": [ + {"event": "closed", "commit_id": sha, + "commit_url": "https://api.github.com/repos/acme/toolkit/commits/" + sha}], + "repos/acme/toolkit/releases?per_page=100&page=1": releases( + "release-1.2.0", "release-1.1.0", "release-1.0.3-hotfix", "release-1.0.2-hotfix", + "release-1.0.0", pre=("release-1.0.3-hotfix",)), + "repos/acme/toolkit/compare/release-1.1.0..." + sha: {"status": "diverged"}, + "repos/acme/toolkit/compare/release-1.2.0..." + sha: {"status": "behind"}} + got, _ = self.poll(reg, Api(routes), files={"eat-up.sh": 'EAT_TAG="release-1.0.3-hotfix"\n'}) + o = got["eat-9"] + self.assertEqual((o["state"], o["fixed_in"]), ("released", "release-1.2.0"), o) + + def test_untracked_entry(self): + reg = OPEN.replace('upstream = "acme/engine#7"', 'upstream = ""').replace( + 'fix = {repo = "acme/engine", pr = 7}', "fix = {}") + got, _ = self.poll(reg, Api({})) + self.assertEqual((got["open-bug"]["state"], got["open-bug"]["fixed_in"]), ("untracked", "")) + + def test_api_error_is_reported_per_entry(self): + got, _ = self.poll(VLLM + OPEN, Api({"repos/acme/engine/pulls/7": {"state": "open", "merged": False}})) + self.assertIn("error", got["vllm-fix"]) + self.assertEqual(got["open-bug"]["state"], "open") + + +# --- this repo ----------------------------------------------------------------- + +class RealRepo(unittest.TestCase): + def test_check_passes_on_this_repo(self): + p = subprocess.run([sys.executable, str(TOOL), "--root", str(ROOT), "check"], capture_output=True, text=True) + self.assertEqual(p.returncode, 0, p.stdout + p.stderr) + + def test_registry_seeds_the_known_items(self): + with open(ROOT / "pins" / "known-issues.toml", "rb") as f: + ids = {e["id"]: e for e in tomllib.load(f)["issue"]} + for want in ("vllm-45844", "vllm-50801", "vllm-pr-51493", "linux-npu-driver-56", "kserve-4592", + "eat-9", "litellm-1.83.14-advisories", "vllm-0.32-num-spec-tokens"): + self.assertIn(want, ids) + self.assertEqual((ids["vllm-45844"]["affected"], ids["vllm-45844"]["fixed_in"]), ("<0.25.0", "0.25.0")) + self.assertEqual(ids["kserve-4592"]["fixed_in"], "") + self.assertEqual(ids["vllm-pr-51493"]["fixed_in"], "") + + def test_moving_the_ours_vllm_cpu_pin_to_0_24_fails_with_the_bump_message(self): + """Copy the registry and every file it names into a fixture; move AMX_TEST_IMAGE to v0.24.0.""" + d = pathlib.Path(tempfile.mkdtemp(prefix="known_issues_real.")) + try: + with open(ROOT / "pins" / "known-issues.toml", "rb") as f: + reg = tomllib.load(f) + paths = {"pins/known-issues.toml", "EXPERIMENTS.md"} + for e in reg["issue"]: + paths |= {s["file"] for s in e["sites"]} | set(e["workarounds"]) + paths |= {r["path"] for r in e.get("ledger_refs", [])} + if (ROOT / "pins" / "classes").is_file(): + paths.add("pins/classes") + for rel in paths: + (d / rel).parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(ROOT / rel, d / rel) + rc, out = run_check(d) + self.assertEqual(rc, 0, out) + k = d / "scripts" / "k3d-gpu" + text = k.read_text() + moved = re.sub(r"(K3D_GPU_AMX_TEST_IMAGE:-vllm/vllm-openai-cpu:)v\d+\.\d+\.\d+@", r"\1v0.24.0@", text) + self.assertNotEqual(text, moved) + k.write_text(moved) + rc, out = run_check(d) + self.assertEqual(rc, 1, out) + self.assertIn("bump scripts/k3d-gpu to >= 0.25.0", out) + finally: + shutil.rmtree(d, ignore_errors=True) + + +if __name__ == "__main__": + unittest.main(verbosity=1) diff --git a/tests/ledger_freshness_test.py b/tests/ledger_freshness_test.py new file mode 100755 index 00000000..2e8a560a --- /dev/null +++ b/tests/ledger_freshness_test.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python3 +"""Tests for hack/ledger-freshness. Each case writes a throwaway repo (ledger, run +sections, images.lock, ledger-rules, a cited doc, a known-issues registry) and +reads the re-measure queue; the last cases run it on this repo and check that +EXPERIMENTS.md and console/ledger-rules.json are byte-identical afterwards. + +Usage: python3 tests/ledger_freshness_test.py +""" + +import hashlib +import importlib.machinery +import importlib.util +import json +import pathlib +import shutil +import subprocess +import sys +import tempfile +import unittest + +ROOT = pathlib.Path(__file__).resolve().parent.parent +TOOL = ROOT / "hack" / "ledger-freshness" + +LEDGER = """# Experiments + +## Hypotheses + +| # | Hypothesis | Prediction | Probe | Verdict | +|---|------------|------------|-------|---------| +| H34 | ovms draft | p | Pre-registered. Result ([H34 result](#h34-result-host-2026-10-17)). | SUPPORTED | +| H57dm | llama mtp | p | Run 2026-10-21 ([H57d run](#h57d-run-host-2026-10-21)). | SUPPORTED | +| H60 | ovms older | p | Run ([H60 run](#h60-run-host-2026-10-22)) \\| piped cell. | SUPPORTED | +| H61 | vllm cpu | p | Pre-registration ([H61 pre-registration](#h61-pre-registration)); run ([H61 run](#h61-run-host-2026-10-23)). | SUPPORTED | +| H62 | no record | p | probe only | SUPPORTED | +| H63 | pending | p | ([H63 run](#h63-run-host)) | PENDING | +| H64 | refuted | p | ([H64 run](#h64-run-host)) | REFUTED | + +## Reproduce + +### H34 result (host, 2026-10-17) + +**Reproduce:** commit `bb382c2`, node image `cryptoandcoffee/k3d-gpu:0.1.55`, OVMS +2026.4.1 (`openvino/model_server@sha256:a928903f...76fc58`), models from `~/m`. + +### H57d run (host, 2026-10-21) + +**Reproduce:** + +```text +# provenance +k3d-gpu commit: 458bca0e, node image + cryptoandcoffee/k3d-gpu:0.1.63@sha256:b6fc +runtimes: ghcr.io/ggml-org/llama.cpp:server-b11749@sha256:9fba; + openvino/model_server@sha256:a928903fe429f1e3 (2026.4.1) +models: google/gemma-4-26B-A4B-it-qat-q4_0-gguf @ d1c082be (gemma.gguf sha256 3eca3b8f...); + OpenVINO/gemma-4-26b-a4b-it-int4-ov @ 4e5885c3 +records: bench/h57d/results/ +``` + +### H60 run (host, 2026-10-22) + +```text +# Reproduce +runtime: openvino/model_server:2026.3-gpu@sha256:aaaa +``` + +### H61 pre-registration + +Arms on vllm/vllm-openai-cpu:v0.20.0 (not a run; ignored). + +### H61 run (host, 2026-10-23) + +```text +# Reproduce +runtime: vllm/vllm-openai-cpu:v0.24.0@sha256:b0e6 +node image: cryptoandcoffee/k3d-gpu:0.1.70 +``` + +### H64 run (host) + +```text +# Reproduce +runtime: openvino/model_server:2025.1 +``` +""" + +LOCK = {"images": [ + {"name": "docker.io/openvino/model_server", "tag": "2026.4.1-gpu", "digest": "sha256:a928903fe429f1e3" + "0" * 48, + "sources": ["scripts/k3d-gpu NPU_UMD_IMAGE"]}, + {"name": "docker.io/openvino/model_server", "digest": "sha256:a928903fe429f1e3" + "0" * 48, + "sources": ["scripts/k3d-gpu AGENT_OVMS_IMAGE"]}, + {"name": "ghcr.io/ggml-org/llama.cpp", "tag": "server-b11960", "digest": "sha256:" + "1" * 64, + "sources": ["scripts/k3d-gpu K3D_GPU_LLAMA_CPU_IMAGE"]}, + {"name": "docker.io/vllm/vllm-openai-cpu", "tag": "v0.32.0", "digest": "sha256:" + "2" * 64, + "sources": ["scripts/k3d-gpu K3D_GPU_AMX_TEST_IMAGE"]}, + {"name": "docker.io/vllm/vllm-openai-cpu", "tag": "v0.24.0", "digest": "sha256:" + "3" * 64, + "sources": ["integrations/eat/vllm-values.yaml"]}, +]} + +RULES = {"rows": [{"id": "H57dm", "verdict": "SUPPORTED", "classes": ["i9-13900t"]}, + {"id": "H34", "verdict": "SUPPORTED", "classes": []}]} + +KNOWN = """ +[[issue]] +id = "vllm-fix" +title = "a CPU attention bug" +kind = "bug" +upstream = "" +component = "vllm-openai-cpu" +scheme = "zerover" +affected = "<0.25.0 || >=0.30.0,<0.30.1" +fixed_in = "0.25.0" +adopt = "review" +sites = [{file = "scripts/k3d-gpu", regex = 'vllm-openai-cpu:v(?P[0-9.]+)@'}] +workarounds = [] +notes = "" +""" + +FILES = { + "EXPERIMENTS.md": LEDGER, + "images.lock": json.dumps(LOCK), + "console/ledger-rules.json": json.dumps(RULES), + "docs/perf.md": "OVMS decodes 12 tok/s here [ledger:H60].\n", + "README.md": "k3d-gpu\n", + "scripts/k3d-gpu": 'VERSION="0.1.79"\nAMX="vllm/vllm-openai-cpu:v0.32.0@sha256:x"\n', + "pins/known-issues.toml": KNOWN, +} + + +def run(root, *args): + p = subprocess.run([sys.executable, str(TOOL), "--root", str(root), *args], capture_output=True, text=True) + return p.returncode, p.stdout, p.stderr + + +class Fixture(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.root = pathlib.Path(tempfile.mkdtemp(prefix="ledger_freshness.")) + for rel, text in FILES.items(): + p = cls.root / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(text) + rc, out, err = run(cls.root, "--json") + assert rc == 0, out + err + cls.data = json.loads(out) + cls.queue = {q["row"]: q for q in cls.data["queue"]} + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.root, ignore_errors=True) + + def test_measured_versions(self): + m = self.data["measured"] + self.assertEqual(m["H34"]["openvino/model_server"], "2026.4.1") + self.assertEqual(m["H34"]["cryptoandcoffee/k3d-gpu"], "0.1.55") + self.assertEqual(m["H57dm"]["llama.cpp"], "b11749") + self.assertEqual(m["H57dm"]["openvino/model_server"], "2026.4.1") + self.assertEqual(m["H57dm"]["hf:google/gemma-4-26B-A4B-it-qat-q4_0-gguf"], "d1c082be") + self.assertEqual(m["H57dm"]["hf:OpenVINO/gemma-4-26b-a4b-it-int4-ov"], "4e5885c3") + self.assertEqual(m["H60"]["openvino/model_server"], "2026.3") + self.assertEqual(m["H61"]["vllm-openai-cpu"], "0.24.0") + + def test_only_supported_rows(self): + self.assertNotIn("H63", self.data["measured"]) + self.assertNotIn("H64", self.data["measured"]) + + def test_pre_registration_anchor_is_not_a_run(self): + self.assertNotIn("0.20.0", json.dumps(self.data["measured"]["H61"])) + + def test_current_versions_from_ours_pins(self): + c = self.data["current"] + self.assertEqual(c["openvino/model_server"], "2026.4.1") + self.assertEqual(c["llama.cpp"], "b11960") + self.assertEqual(c["vllm-openai-cpu"], "0.32.0") # the eat-stock v0.24.0 is not ours + self.assertEqual(c["cryptoandcoffee/k3d-gpu"], "0.1.79") + + def test_ovms_2026_3_against_2026_4_1_is_stale(self): + q = self.queue["H60"] + self.assertEqual((q["measured"]["openvino/model_server"], q["current"]["openvino/model_server"]), + ("2026.3", "2026.4.1")) + self.assertIn("minor", q["reason"]) + self.assertFalse(q["priority"]) + + def test_same_minor_is_not_stale(self): + # H34: OVMS 2026.4.1 = current; node image 0.1.55 -> 0.1.79 is a patch. + self.assertNotIn("H34", self.queue) + + def test_llama_build_behind_is_stale(self): + self.assertIn("llama.cpp b11749 -> b11960", self.queue["H57dm"]["reason"]) + + def test_measured_inside_a_known_issue_is_priority_stale(self): + q = self.queue["H61"] + self.assertTrue(q["priority"]) + self.assertEqual(q["known_issues"], ["vllm-fix"]) + self.assertIn("priority-stale", q["reason"]) + + def test_unparseable_row_is_listed(self): + rows = {r["row"]: r["reason"] for r in self.data["not_machine_readable"]} + self.assertIn("H62", rows) + self.assertNotIn("H34", rows) + + def test_queue_order(self): + # ledger-rules rows with classes, then rows docs claims cite, then the rest. + self.assertEqual([q["row"] for q in self.data["queue"]], ["H57dm", "H60", "H61"]) + self.assertEqual([q["tier"] for q in self.data["queue"]], [1, 2, 3]) + + def test_suggested_followup_ids_are_free(self): + rows = set(self.data["measured"]) | {"H63", "H64", "H62"} + for q in self.data["queue"]: + self.assertRegex(q["suggested_followup_id"], r"^H\d+[a-z]{1,2}$") + self.assertNotIn(q["suggested_followup_id"], rows) + self.assertEqual(self.queue["H60"]["suggested_followup_id"], "H60r") + + def test_two_queued_rows_of_one_family_get_different_followup_ids(self): + # H57dm and H57de both suggest H57r first; the second must take the next free id. + loader = importlib.machinery.SourceFileLoader("ledger_freshness", str(TOOL)) + spec = importlib.util.spec_from_loader("ledger_freshness", loader) + mod = importlib.util.module_from_spec(spec) + loader.exec_module(mod) + measured = {"H57dm": {"llama.cpp": "b11749"}, "H57de": {"llama.cpp": "b11749"}} + got = [q["suggested_followup_id"] for q in mod.queue(self.root, measured, {"llama.cpp": "b11960"}, [])] + self.assertEqual(len(got), 2) + self.assertEqual(len(set(got)), 2, got) + + def test_markdown(self): + rc, out, err = run(self.root, "--md") + self.assertEqual(rc, 0, err) + for needle in ("| 1 | H57dm |", "b11749", "| H34 |", "openvino/model_server 2026.4.1", "H62", "priority-stale"): + self.assertIn(needle, out) + # A pipe inside a cell (a range union) is escaped, so every queue row keeps 5 cells. + self.assertIn("<0.25.0 \\|\\| >=0.30.0", out) + for line in out.splitlines(): + if line.startswith("| 3 |"): + self.assertEqual(len(line.replace("\\|", "").split("|")), 7, line) + + def test_needs_an_output_mode(self): + rc, _, _ = run(self.root) + self.assertEqual(rc, 2) + + +def digest(paths): + return {p: hashlib.sha256((ROOT / p).read_bytes()).hexdigest() for p in paths} + + +class RealRepo(unittest.TestCase): + PROTECTED = ("EXPERIMENTS.md", "console/ledger-rules.json") + + def test_runs_read_only_and_lists_the_supported_engine_rows(self): + before = digest(self.PROTECTED) + rc, out, err = run(ROOT, "--json") + self.assertEqual(rc, 0, err) + rc2, md, err2 = run(ROOT, "--md") + self.assertEqual(rc2, 0, err2) + self.assertEqual(before, digest(self.PROTECTED)) + data = json.loads(out) + # The ledger's recorded versions, read by shape: a pin bump substitutes version strings in tests/. + self.assertRegex(data["measured"]["H34"]["openvino/model_server"], r"^\d{4}\.\d+(\.\d+)?$") + self.assertRegex(data["measured"]["H57dm"]["llama.cpp"], r"^b\d+$") + self.assertIn("| H34 |", md) + self.assertIn("| H57dm |", md) + + +if __name__ == "__main__": + unittest.main(verbosity=1) diff --git a/tests/pinver_test.py b/tests/pinver_test.py new file mode 100755 index 00000000..32d2b16d --- /dev/null +++ b/tests/pinver_test.py @@ -0,0 +1,324 @@ +#!/usr/bin/env python3 +"""Tests for hack/pinver.py: every versioning scheme, ranges, update types, +prerelease exclusion and the CLI. + +Usage: python3 tests/pinver_test.py +""" + +import pathlib +import subprocess +import sys +import unittest + +ROOT = pathlib.Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "hack")) +import pinver # noqa: E402 + + +class Semver(unittest.TestCase): + def test_v_prefix_is_optional(self): + self.assertEqual(pinver.compare("semver", "v1.2.3", "1.2.3"), 0) + + def test_numeric_order(self): + self.assertEqual(pinver.compare("semver", "1.9.0", "1.10.0"), -1) + self.assertEqual(pinver.compare("semver", "2.0.0", "1.99.99"), 1) + + def test_prerelease_sorts_before_release(self): + self.assertEqual(pinver.compare("semver", "1.2.3-rc.1", "1.2.3"), -1) + self.assertEqual(pinver.compare("semver", "1.2.3-rc.2", "1.2.3-rc.10"), -1) + self.assertEqual(pinver.compare("semver", "1.2.3-alpha", "1.2.3-alpha.1"), -1) + + def test_prerelease_flag(self): + self.assertTrue(pinver.is_prerelease("semver", "v1.2.3-rc1")) + self.assertFalse(pinver.is_prerelease("semver", "v1.2.3")) + + def test_build_metadata_ignored_in_order(self): + self.assertEqual(pinver.compare("semver", "1.2.3+a", "1.2.3+b"), 0) + + def test_update_types(self): + self.assertEqual(pinver.update_type("semver", "1.2.3", "1.2.4"), "patch") + self.assertEqual(pinver.update_type("semver", "1.2.3", "1.3.0"), "minor") + self.assertEqual(pinver.update_type("semver", "1.2.3", "2.0.0"), "major") + self.assertEqual(pinver.update_type("semver", "v1.2.3", "1.2.3"), "digest") + + def test_breaking_is_major_only(self): + self.assertFalse(pinver.breaking("semver", "1.2.3", "1.3.0")) + self.assertTrue(pinver.breaking("semver", "1.2.3", "2.0.0")) + # Under plain semver a 0.x minor is not breaking; zerover says it is. + self.assertFalse(pinver.breaking("semver", "0.24.0", "0.25.0")) + + def test_garbage_is_rejected(self): + for bad in ("", "latest", "1.2.x", "v"): + with self.assertRaises(ValueError, msg=bad): + pinver.parse("semver", bad) + + +class Zerover(unittest.TestCase): + def test_zerover_minor_is_minor_but_breaking(self): + self.assertEqual(pinver.update_type("zerover", "v0.24.0", "v0.25.0"), "minor") + self.assertTrue(pinver.breaking("zerover", "v0.24.0", "v0.25.0")) + self.assertEqual(pinver.classify("zerover", "v0.24.0", "v0.25.0"), + {"type": "minor", "breaking": True}) + + def test_zerover_patch_is_not_breaking(self): + self.assertEqual(pinver.update_type("zerover", "v0.25.0", "v0.25.1"), "patch") + self.assertFalse(pinver.breaking("zerover", "v0.25.0", "v0.25.1")) + + def test_zerover_past_one_is_semver(self): + self.assertFalse(pinver.breaking("zerover", "1.2.0", "1.3.0")) + self.assertTrue(pinver.breaking("zerover", "0.32.0", "1.0.0")) + + def test_vllm_fixed_in(self): + self.assertTrue(pinver.in_range("zerover", "v0.24.0", "<0.25.0")) + self.assertFalse(pinver.in_range("zerover", "v0.25.0", "<0.25.0")) + self.assertFalse(pinver.in_range("zerover", "v0.32.0", "<0.25.0")) + + +class K8s(unittest.TestCase): + def test_plus_and_dash_k3s_forms(self): + self.assertEqual(pinver.compare("k8s", "v1.37.1+k3s1", "v1.37.1-k3s1"), 0) + self.assertEqual(pinver.compare("k8s", "v1.37.1+k3s1", "v1.37.1+k3s2"), -1) + self.assertEqual(pinver.compare("k8s", "v1.37.1+k3s9", "v1.37.2+k3s1"), -1) + + def test_k8s_minor_is_breaking(self): + self.assertEqual(pinver.update_type("k8s", "v1.37.1+k3s1", "v1.38.0+k3s1"), "minor") + self.assertTrue(pinver.breaking("k8s", "v1.37.1+k3s1", "v1.38.0+k3s1")) + + def test_k8s_patch_and_distro_build(self): + self.assertEqual(pinver.update_type("k8s", "v1.37.1+k3s1", "v1.37.2+k3s1"), "patch") + self.assertEqual(pinver.update_type("k8s", "v1.37.1+k3s1", "v1.37.1+k3s2"), "patch") + self.assertFalse(pinver.breaking("k8s", "v1.37.1+k3s1", "v1.37.2+k3s1")) + + def test_k8s_rc_is_prerelease(self): + self.assertTrue(pinver.is_prerelease("k8s", "v1.38.0-rc1+k3s1")) + self.assertEqual(pinver.compare("k8s", "v1.38.0-rc1+k3s1", "v1.38.0+k3s1"), -1) + + +class Llama(unittest.TestCase): + def test_build_numbers_compare_numerically(self): + self.assertEqual(pinver.compare("llama", "b9999", "b11749"), -1) + self.assertEqual(pinver.compare("llama", "server-b11960", "b11960"), 0) + self.assertEqual(pinver.compare("llama", "b11749", "b11960"), -1) + + def test_build_bump_is_minor_and_not_breaking(self): + self.assertEqual(pinver.update_type("llama", "b11749", "b11960"), "minor") + self.assertFalse(pinver.breaking("llama", "b11749", "b11960")) + + def test_releases_are_zerover(self): + self.assertEqual(pinver.compare("llama", "v0.7.0", "v0.8.0"), -1) + self.assertTrue(pinver.breaking("llama", "v0.7.0", "v0.8.0")) + + def test_build_and_release_are_incomparable(self): + self.assertFalse(pinver.comparable("llama", "b11960", "v0.8.0")) + with self.assertRaises(ValueError): + pinver.compare("llama", "b11960", "v0.8.0") + + def test_build_range(self): + self.assertTrue(pinver.in_range("llama", "b11749", "=2.9.0,<2.10")) + + def test_prerelease_flag(self): + for v in ("1.0a1", "1.0rc1", "1.0.dev3"): + self.assertTrue(pinver.is_prerelease("pep440", v), v) + for v in ("1.0", "1.0.post1", "1.0+cpu"): + self.assertFalse(pinver.is_prerelease("pep440", v), v) + + def test_update_types(self): + self.assertEqual(pinver.update_type("pep440", "1.83.14", "1.84.0"), "minor") + self.assertEqual(pinver.update_type("pep440", "1.83.14", "1.83.14.post1"), "patch") + self.assertEqual(pinver.update_type("pep440", "1.83.14", "2.0"), "major") + self.assertEqual(pinver.update_type("pep440", "1.83.14", "1.83.14"), "digest") + + +class DockerSuffix(unittest.TestCase): + def test_core_order_within_one_flavour(self): + self.assertEqual(pinver.compare("docker-suffix", "3.14.8-slim", "3.14.10-slim"), -1) + self.assertEqual(pinver.compare("docker-suffix", "contrib-v1.40.0", "contrib-v1.41.0"), -1) + + def test_different_suffix_or_prefix_is_incomparable(self): + self.assertFalse(pinver.comparable("docker-suffix", "3.14.8-slim", "3.14.9-alpine")) + self.assertFalse(pinver.comparable("docker-suffix", "contrib-v1.40.0", "v1.41.0")) + with self.assertRaises(ValueError): + pinver.update_type("docker-suffix", "3.14.8-slim", "3.14.9-alpine") + + def test_latest_ignores_a_different_suffix(self): + got = pinver.latest("docker-suffix", "3.14.8-slim", + ["3.14.9-slim", "3.15.0-alpine", "3.15.0-slim-bookworm", "3.14.7-slim"]) + self.assertEqual(got, "3.14.9-slim") + + def test_update_types_follow_the_core(self): + self.assertEqual(pinver.update_type("docker-suffix", "3.14.8-slim", "3.15.0-slim"), "minor") + self.assertEqual(pinver.update_type("docker-suffix", "release-1.0.3-hotfix", "release-1.0.4-hotfix"), "patch") + + def test_range_reads_the_core(self): + self.assertTrue(pinver.in_range("docker-suffix", "v1.83.14-stable.patch.2", "<1.84.0")) + self.assertFalse(pinver.in_range("docker-suffix", "v1.88.6-stable", "<1.84.0")) + + +class Eat(unittest.TestCase): + """Intel EAT's release tags: one stream of release-X.Y.Z and release-X.Y.Z-hotfix.""" + + def test_hotfix_and_plain_releases_are_one_stream(self): + self.assertTrue(pinver.comparable("eat", "release-1.0.3-hotfix", "release-1.1.0")) + self.assertEqual(pinver.compare("eat", "release-1.0.3-hotfix", "release-1.1.0"), -1) + self.assertEqual(pinver.compare("eat", "release-1.0.2-hotfix", "release-1.0.3-hotfix"), -1) + self.assertEqual(pinver.compare("eat", "release-1.0.3", "release-1.0.3-hotfix"), -1) + + def test_latest_moves_from_a_hotfix_to_the_next_release(self): + got = pinver.latest("eat", "release-1.0.3-hotfix", + ["release-1.0.2-hotfix", "release-1.1.0", "release-1.0.0", "1.2.0", "latest"]) + self.assertEqual(got, "release-1.1.0") + + def test_update_types_and_ranges(self): + self.assertEqual(pinver.update_type("eat", "release-1.0.3-hotfix", "release-1.1.0"), "minor") + self.assertFalse(pinver.breaking("eat", "release-1.0.3-hotfix", "release-1.1.0")) + self.assertTrue(pinver.breaking("eat", "release-1.1.0", "release-2.0.0")) + self.assertTrue(pinver.in_range("eat", "release-1.0.3-hotfix", "1.2.3")) + + def test_bad_specs(self): + for bad in ("", "~1.2", "<", "<1.2.3,,>1", "1.2.3"): + with self.assertRaises(ValueError, msg=bad): + pinver.in_range("semver", "1.2.3", bad) + + def test_check_range_reports_syntax_only(self): + pinver.check_range("zerover", "<0.25.0 || >=0.26.0,<0.26.1") + with self.assertRaises(ValueError): + pinver.check_range("zerover", "<0.25.x") + + +class Latest(unittest.TestCase): + def test_prerelease_excluded(self): + got = pinver.latest("zerover", "v0.24.0", ["v0.25.0", "v0.26.0rc1", "v0.26.0-rc.1", "v0.24.1"]) + self.assertEqual(got, "v0.25.0") + + def test_never_backwards(self): + self.assertIsNone(pinver.latest("zerover", "v0.32.0", ["v0.25.0", "v0.31.0"])) + + def test_unparseable_candidates_skipped(self): + self.assertEqual(pinver.latest("calver", "2026.4.1", ["latest", "2026.5.0", "nightly"]), "2026.5.0") + + +class UnknownScheme(unittest.TestCase): + def test_unknown_scheme(self): + with self.assertRaises(ValueError): + pinver.parse("romver", "1.0") + + +class Cli(unittest.TestCase): + def run_cli(self, *args): + return subprocess.run([sys.executable, str(ROOT / "hack" / "pinver.py"), *args], + capture_output=True, text=True) + + def test_compare(self): + p = self.run_cli("compare", "zerover", "v0.24.0", "0.25.0") + self.assertEqual((p.returncode, p.stdout.strip()), (0, "-1")) + + def test_in_range_exit_code(self): + self.assertEqual(self.run_cli("in-range", "zerover", "v0.24.0", "<0.25.0").returncode, 0) + self.assertEqual(self.run_cli("in-range", "zerover", "v0.32.0", "<0.25.0").returncode, 1) + + def test_update_type(self): + p = self.run_cli("update-type", "zerover", "v0.24.0", "v0.25.0") + self.assertEqual(p.stdout.strip(), "minor breaking") + + def test_bad_input_exit_2(self): + p = self.run_cli("compare", "semver", "x", "1.0.0") + self.assertEqual(p.returncode, 2) + self.assertIn("semver", p.stderr) + + +if __name__ == "__main__": + unittest.main(verbosity=1)