diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml new file mode 100644 index 00000000..8be93728 --- /dev/null +++ b/.github/workflows/canary.yml @@ -0,0 +1,237 @@ +name: Canary + +# Every engine or node-image bump, from a bot or a human, checked at runtime before it +# merges (canary/README.md): +# runtime vllm-cpu, ovms-cpu, llama-cpu (and eat-vllm-cpu, informational): pull the +# pinned image, serve a pinned 0.6B model the way the launcher would, one real +# and one streamed completion, the version the server reports; +# surface every engine image: the flags and env names the launcher emits exist in it, +# and the constants k3d-gpu derives from it still match (no device needed); +# node build the node image, start a k3d cluster on it, everything baked comes up. +# A pull request or `engines=auto` runs the engines whose pin, flags, settings or golden +# differ from merge-base(origin/main, HEAD), or from the pull request's base branch +# (canary/plan.py matrix); the schedule runs all. +# The report job writes canary-result.json (artifact canary-result) and is green only when +# every selected engine's verdict is pass with hard true. +# +# Read-only: no secrets, no registry login (public images), GitHub-hosted runners only. +# The re-dispatch job alone may write: it starts this workflow again (attempt + 1, at most +# 3) when an engine was isa-skip, the runner's CPU lacking an instruction set it needs. + +on: + schedule: + - cron: '29 3 * * *' + workflow_dispatch: + inputs: + engines: + description: 'engine ids from canary/engines.toml, comma separated, or auto (the engines that differ from main)' + type: string + default: auto + attempt: + description: 'attempt number: an isa-skip re-dispatches with attempt + 1, at most 3' + type: number + default: 1 + pull_request: + paths: + - Dockerfile + - scripts/k3d-gpu + - 'share/**' + - console/moe_predict.py + - console/plan.py + - console/model_opt.py + - console/fit.py + - bench/h54/h54.py # sglang_args, the sglang flags the surface check reads + - 'canary/**' + - packaging/install.sh + - integrations/eat/vllm-values.yaml + - .github/workflows/canary.yml + +permissions: + contents: read + +concurrency: + group: canary-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + plan: + name: Plan + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + outputs: + runtime: ${{ steps.plan.outputs.runtime }} + surface: ${{ steps.plan.outputs.surface }} + node: ${{ steps.plan.outputs.node }} + selected: ${{ steps.plan.outputs.selected }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 # merge-base(origin/main, HEAD) + persist-credentials: false + # auto: against merge-base(origin/, HEAD) + - id: plan + env: + EVENT: ${{ github.event_name }} + ENGINES: ${{ inputs.engines }} + BASE: ${{ github.base_ref || 'main' }} + run: | + mode=auto + [ "$EVENT" != schedule ] || mode=all + python3 canary/plan.py matrix --mode "$mode" --engines "${ENGINES:-auto}" --base "origin/$BASE" \ + --github-output "$GITHUB_OUTPUT" + + runtime: + name: Runtime (${{ matrix.id }}) + needs: plan + if: needs.plan.outputs.runtime != '[]' + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.runtime) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # The model at its pinned revision; canary/run.sh downloads it on a miss. + - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ matrix.cache_path }} + key: ${{ matrix.cache_key }} + - name: canary/run.sh + env: + ID: ${{ matrix.id }} + CANARY_FREE_DISK: '1' + CANARY_OUT_DIR: ${{ runner.temp }}/canary-out + run: canary/run.sh "$ID" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: always() + with: + name: canary-part-${{ matrix.id }} + path: ${{ runner.temp }}/canary-out/ + if-no-files-found: ignore + retention-days: 14 + + surface: + name: Surface (${{ matrix.id }}) + needs: plan + if: needs.plan.outputs.surface != '[]' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + strategy: + fail-fast: false + max-parallel: 4 + matrix: + include: ${{ fromJSON(needs.plan.outputs.surface) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: canary/run.sh --surface + env: + ID: ${{ matrix.id }} + CANARY_FREE_DISK: '1' + CANARY_OUT_DIR: ${{ runner.temp }}/canary-out + run: canary/run.sh --surface "$ID" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: always() + with: + name: canary-part-${{ matrix.id }} + path: ${{ runner.temp }}/canary-out/ + if-no-files-found: ignore + retention-days: 14 + + node: + name: Node image + needs: plan + if: needs.plan.outputs.node == 'true' + runs-on: ubuntu-latest + timeout-minutes: 40 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: canary/node.sh + env: + CANARY_OUT_DIR: ${{ runner.temp }}/canary-out + run: canary/node.sh + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: always() + with: + name: canary-part-node + path: ${{ runner.temp }}/canary-out/ + if-no-files-found: ignore + retention-days: 14 + + report: + name: Report + needs: [plan, runtime, surface, node] + if: always() && needs.plan.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + outputs: + isa_skip: ${{ steps.report.outputs.isa_skip }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: canary-part-* + merge-multiple: true + path: ${{ runner.temp }}/parts + - id: report + env: + SELECTED: ${{ needs.plan.outputs.selected }} + SHA: ${{ github.event.pull_request.head.sha || github.sha }} + REF: ${{ github.head_ref || github.ref_name }} + run: | + mkdir -p "$RUNNER_TEMP/parts" + rc=0 + python3 canary/plan.py report --selected "$SELECTED" --parts "$RUNNER_TEMP/parts" --sha "$SHA" --ref "$REF" \ + --out canary-result.json --summary "$GITHUB_STEP_SUMMARY" --github-output "$GITHUB_OUTPUT" || rc=$? + echo "rc=$rc" >> "$GITHUB_OUTPUT" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: canary-result + path: canary-result.json + retention-days: 30 + - name: Green only when every selected engine passed + env: + RC: ${{ steps.report.outputs.rc }} + run: exit "${RC:-1}" + + redispatch: + name: Re-dispatch isa-skip engines + needs: report + if: >- + always() && needs.report.outputs.isa_skip != '' && + (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + actions: write + steps: + - env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + REF: ${{ github.head_ref || github.ref_name }} + ENGINES: ${{ needs.report.outputs.isa_skip }} + ATTEMPT: ${{ inputs.attempt || 1 }} + run: | + if [ "$ATTEMPT" -ge 3 ]; then + echo "::warning::$ENGINES isa-skip on attempt $ATTEMPT of 3: no runner with the instruction set it needs; the canary stays not green" + exit 0 + fi + gh workflow run canary.yml --repo "$REPO" --ref "$REF" -f engines="$ENGINES" -f attempt="$((ATTEMPT + 1))" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ebc42427..ba2ec8a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -483,3 +483,22 @@ jobs: - run: python3 tests/known_issues_test.py - run: python3 tests/ledger_freshness_test.py - run: hack/known-issues check + + # Part canary (.github/workflows/canary.yml runs the engines themselves): canary/run.sh + # against stub engines and a docker shim, canary/node.sh against scripted pod states, the + # surface parser on vLLM v0.32.0's real sources and the pinned images' help output, + # flags.sh, the plan and the report. No image is pulled. + canary-tests: + name: Canary harness (stub engines, surface parser, plan, report) + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - run: tests/canary_test.sh + - run: tests/canary_node_test.sh + - run: tests/canary_spec_test.sh + - run: python3 tests/canary_surface_test.py diff --git a/canary/README.md b/canary/README.md new file mode 100644 index 00000000..bf00314f --- /dev/null +++ b/canary/README.md @@ -0,0 +1,98 @@ +# Canary + +Every engine or node-image bump, whether a bot or a person opened it, is checked at +runtime before it merges. Engines move fast: a vLLM bug today is fixed tomorrow, and a +flag the launcher passes today is renamed tomorrow. Before this canary, no CI job built the +node image, started a cluster, or pulled and served an engine. Every engine path in the +tests was a stub. + +`.github/workflows/canary.yml` (name `Canary`) runs at 03:29 UTC every night, on pull +requests that touch a pin or the launcher, and on dispatch (`engines`: ids or `auto`; +`attempt`). It runs only on GitHub-hosted runners. It has read-only permissions and no +secrets. + +| Kind | Engines | What it proves | +|---|---|---| +| runtime | `vllm-cpu`, `ovms-cpu`, `llama-cpu`; `eat-vllm-cpu` (informational) | The pinned image serves a pinned Qwen3-0.6B the way the launcher runs it. It returns one completion (HTTP 200, `finish_reason` stop or length, tokens > 0, the text matches `/Paris/`) and one stream (at least 2 chunks, then `[DONE]`), and the server reports the version its tag names. | +| surface | `vllm-xpu`, `llm-scaler`, `gaudi-vllm`, `llama-sycl`, `llama-vulkan`, `llama-full`, `dlstreamer`, `sglang-xeon`, `npu-umd`, `ovms` (and the runtime engines first) | Every flag and env name the launcher and `console/model_opt.py` emit (`canary/flags.sh`) exists in the image, and the constants derived from it still match: `console/plan.py`'s vLLM versions, and the NPU driver and firmware date. No device is needed. | +| node | `node` | The root Dockerfile builds. `k3d cluster create --image` it with the k3d `packaging/install.sh` pins and the arguments the launcher passes on a host without NVIDIA (no load balancer, no traefik, the node labelled `nvidia.com/gpu.present=false`). Within 5 minutes the node is Ready, every baked manifest is in place, every DaemonSet they define exists, and every pod in every namespace (kube-system, NFD, the device plugins) is Running and ready or Succeeded; then the pods hold for 20 s with no container restarting. A plugin that needs hardware gets no pod on a hosted runner (its node selector), so a pod still Pending at the timeout, such as one whose image does not pull, fails. | + +## Files + +| File | Role | +|---|---| +| `engines.toml` | One table per engine. Its `locator` points at the pin in the tree (never a copy of it), plus the model and revision, the launcher-equivalent arguments, the golden, the version probe, the flag sources and the derived constants. | +| `plan.py` | Resolves the pins, plans the job matrix and merges the report. | +| `run.sh` | Runtime and surface: disk, CPU flags, model, size, pull, surface, serve, probe, ISA. | +| `probe.py` | The HTTP checks against a serving engine. | +| `surface.py` | The flag, env and derived-constant check. It reads the engine's own `--help`, or for vLLM statically parses `arg_utils.py`, `entrypoints/**/cli_args.py` and `envs.py` inside the image, without importing them. | +| `flags.sh` | Lists the names k3d-gpu emits, per engine family. | +| `node.sh` | The node canary. | +| `goldens/.json` | `{prompt, regex, exact, recorded_with}` for each runtime engine. | + +## How names and versions compare + +- A flag matches exactly; for vLLM, `--a_b` and `--a-b` are one flag. +- sglang's parser is argparse with abbreviations allowed, and `bench/h54` passes `--tp`. + So for sglang, a long flag also matches when it begins exactly one option (`--tp` is + `--tp-size`). Once a second option begins with it, argparse calls it ambiguous, and so + does the canary. vLLM flags never match by prefix, so a renamed flag cannot hide behind + a longer one. +- A derived constant matches the image's version by release (`0.32.0` = `0.32.0+cpu`). + A setuptools-scm build of a tag with patches applied, `X.Y.(Z+1).dev0+g.d`, + is release `X.Y.Z`: intel/llm-scaler-vllm 1.5 reports `0.26.1.dev0+g568afb3a1.d20260907.xpu`, + which is vLLM v0.26.0's commit with Intel's patches. A build commits after a tag + (`.dev1` and up) matches nothing. + +## Which engines run + +`schedule` runs every engine. + +A pull request or `engines=auto` runs the engines that differ from +`merge-base(origin/main, HEAD)` in any of these: + +- the resolved image reference; +- their `engines.toml` table; +- their golden; +- a derived constant; +- a flag `flags.sh` names for them. + +The node engine also runs when the Dockerfile, `share/` or `packaging/install.sh` change. A +change to the harness itself runs every engine. + +`engines=a,b` runs exactly those engines. + +## Verdicts (`canary-result` artifact, `canary-result.json`) + +```json +{"sha": "...", "ref": "...", "engines": {"": {"kind": "runtime|surface|node", + "pinned_ref": "...", "verdict": "pass|fail|isa-skip|too-big|unparseable", + "hard": true, "exact": true, "cpu_flags": ["avx2"], "detail": "..."}}} +``` + +Only `verdict` pass with `hard` true is green. The report job fails unless every selected +non-informational engine is green. + +| Value | Meaning | +|---|---| +| `exact` false | The text differs from the golden, but the hard checks passed. It is still green, but it is not evidence for an automatic merge. | +| `isa-skip` | The runner's CPU lacks an instruction set the engine needs: no `avx512f`, and the engine's `isa_signal` in its log, or SIGILL. The workflow dispatches itself again, up to `attempt` 3. Hosted runners vary in CPU, so a later attempt may land on one with AVX-512. | +| `too-big` | The image needs more disk than the runner has, even after the toolchains are deleted. | +| `unparseable` | A help output or a source the surface check could not read. It is red, never skipped. | + +## Recording a golden + +`exact` is the completion text the hosted runner produced with temperature 0, top_k 1 and +seed 0. Take it from the `detail` (`text '...'`) of a green run, and write it with the +image, model and CPU it was recorded on (the job log's `cpu:` line names the CPU). A `null` +`exact` means nothing is recorded yet. The vLLM goldens come from an AVX2-only runner; a +runner with AVX-512 runs other kernels and may produce other text. That clears `exact`, +which sends the bump to review, but it is never red. + +## Adding an engine + +1. Add an `[[engine]]` with a `locator` that matches exactly one reference. + `tests/canary_spec_test.sh` checks it against the tree. +2. If the launcher emits flags for a new family, teach `flags.sh` its source. A flag + literal in `console/model_opt.py` or `console/fit.py` that no rule attributes to an + engine fails `flags.sh`. diff --git a/canary/engines.toml b/canary/engines.toml new file mode 100644 index 00000000..741cb95d --- /dev/null +++ b/canary/engines.toml @@ -0,0 +1,257 @@ +# The engines the canary checks (.github/workflows/canary.yml, canary/README.md). +# +# Each [[engine]] names where its image is pinned in the tree, never the pin itself, +# so a bump (bot or human) moves the canary with it. Fields: +# id the engine id canary-result.json and `engines=` use +# kind runtime: serve one real and one streamed completion (canary/run.sh), after +# the surface check when it has a `surface` +# surface: check the flags, env names and derived constants (canary/surface.py) +# node: build the node image and start a cluster on it (canary/node.sh) +# family the canary/flags.sh engine family whose names apply to it +# informational a runtime engine whose result never fails the report (an EAT pin) +# locator {file, regex}: the regex matches exactly one image reference in file, +# as a named group `ref` (name@sha256 or name:tag@sha256), or as groups +# `tag` and `digest` with a fixed `name` +# model, revision, files +# the Hugging Face repo, its 40-hex commit and the files a runtime engine +# serves (fetched at that commit, cached by actions/cache) +# args, env the engine's arguments and environment, as the launcher (or the fit, +# for vLLM on the CPU) sets them for this model; {name} is the served +# model name, {mount} the model's path in the container, {model} the +# repo id, {threads} the runner's CPU count +# mount where the model directory is mounted (read-only); with `writable`, a copy +# of it is, inside a writable directory mounted there +# prepare, prepare_env +# a run before the server's, on the same mounts (OVMS: --pull writes graph.pbtxt) +# cap_add, shm_size +# what the launcher's pod gets (IPC_LOCK for mlock, the shm emptyDir) +# api, port the OpenAI API root and the container port +# golden canary/goldens/.json: prompt, the hard regex and the exact text +# version_probe how the engine reports its version: {path, key, regex} (a server +# endpoint), {args[, entrypoint], regex} (a command in the image) or +# {package} (the installed package); `tag` reads the expected version +# from the pinned tag (a pin without a tag is not compared) +# surface {driver, ...}: how canary/surface.py reads the image's flags (see there) +# flags_source the canary/flags.sh sources (of `family`) whose names this image must accept +# derived constants elsewhere in the tree that must equal a fact of the image +# ({file, regex with group v, of}; `of` names the image fact: package, +# deb, firmware) +# isa_signal runtime: a log regex that, on a runner without avx512f, makes the +# verdict isa-skip instead of fail (the image cannot run on this ISA) +# paths node: files and directories whose change selects it + +[defaults] +ready_timeout = 600 # seconds the server gets to list its model at {api}/models +size_factor = 3.5 # pulled and unpacked bytes per compressed byte (too-big check) +name = "canary" # served model name + +# ---- runtime --------------------------------------------------------------------- + +[[engine]] +id = "vllm-cpu" +kind = "runtime" +family = "vllm" +locator = { file = "scripts/k3d-gpu", regex = 'AMX_TEST_IMAGE="\$\{K3D_GPU_AMX_TEST_IMAGE:-(?Pvllm/vllm-openai-cpu(?::[^@"}]+)?@sha256:[0-9a-f]{64})\}"' } +model = "Qwen/Qwen3-0.6B" +revision = "c1899de289a04d12100db370d81485cdf75e47ca" +files = ["config.json", "generation_config.json", "merges.txt", "model.safetensors", "tokenizer.json", "tokenizer_config.json", "vocab.json"] +mount = "/models/model" +# vLLM on the CPU as the fit sizes it (console/fit.py _flags): --max-model-len, +# --max-num-seqs and VLLM_CPU_KVCACHE_SPACE (GiB). +args = ["{mount}", "--served-model-name={name}", "--port=8000", "--dtype=bfloat16", "--max-model-len=2048", "--max-num-seqs=1"] +env = ["VLLM_CPU_KVCACHE_SPACE=1"] +shm_size = "2g" # the pod's shm emptyDir (vllm_manifest): worker IPC +api = "/v1" +port = 8000 +golden = "canary/goldens/vllm-cpu.json" +version_probe = { path = "/version", key = "version", regex = '^(?P\d+\.\d+\.\d+)', tag = '^v(?P\d+\.\d+\.\d+)$' } +surface = { driver = "vllm" } +flags_source = ["scripts/k3d-gpu:vllm_manifest", "console/fit.py:_flags", "console/model_opt.py:optimize", "console/model_opt.py:spec_decide", "console/model_opt.py:eat_patch"] +isa_signal = 'Illegal instruction|Failed to import from vllm\._C_AVX2' + +[[engine]] +id = "ovms-cpu" +kind = "runtime" +family = "ovms" +locator = { file = "scripts/k3d-gpu", regex = 'AGENT_OVMS_IMAGE="(?Popenvino/model_server(?::[^@"]+)?@sha256:[0-9a-f]{64})"' } +model = "OpenVINO/Qwen3-0.6B-int8-ov" +revision = "ac8e917fb8350f528ada41b02c60ea3a2853955f" +files = ["added_tokens.json", "config.json", "generation_config.json", "merges.txt", "openvino_detokenizer.bin", "openvino_detokenizer.xml", "openvino_model.bin", "openvino_model.xml", "openvino_tokenizer.bin", "openvino_tokenizer.xml", "special_tokens_map.json", "tokenizer.json", "tokenizer_config.json", "vocab.json"] +# agent_manifest's pull init container and server, on the CPU plugin. The model is +# already at {mount} (pinned revision), so --pull only writes graph.pbtxt; HF_ENDPOINT +# points nowhere so it can never fetch another revision. +mount = "/models/{model}" +writable = "/models" +prepare = ["--pull", "--source_model={model}", "--model_repository_path=/models", "--model_name={name}", "--task=text_generation", "--target_device=CPU"] +prepare_env = ["HF_ENDPOINT=http://127.0.0.1:9"] +args = ["--rest_port=8000", "--model_path={mount}", "--model_name={name}", "--cache_dir=/models/.ov_cache", "--log_level=INFO"] +api = "/v3" +port = 8000 +golden = "canary/goldens/ovms-cpu.json" +version_probe = { path = "/v2", key = "version", regex = '^(?P\d+\.\d+\.\d+)', tag = '^(?P\d+\.\d+\.\d+)' } +flags_source = ["scripts/k3d-gpu:agent_manifest", "scripts/k3d-gpu:agent_ovms_addon_manifest", "console/fit.py:_flags", "console/model_opt.py:spec_decide"] + +[[engine]] +id = "llama-cpu" +kind = "runtime" +family = "llama.cpp" +locator = { file = "scripts/k3d-gpu", regex = 'LLAMA_CPU_IMAGE="\$\{K3D_GPU_LLAMA_CPU_IMAGE:-(?Pghcr\.io/ggml-org/llama\.cpp(?::[^@"}]+)?@sha256:[0-9a-f]{64})\}"' } +model = "ggml-org/Qwen3-0.6B-GGUF" +revision = "13ac06624948e4d8589cb4b6f90100837342fcb8" +files = ["Qwen3-0.6B-Q8_0.gguf"] +mount = "/models" +# llamacpp_manifest on a CPU placement: the fit's -c/-np/-ngl, threads, mmap+mlock +# (with IPC_LOCK, as the pod gets), thinking off. +args = ["--model", "{mount}/Qwen3-0.6B-Q8_0.gguf", "--alias", "{name}", "--host", "0.0.0.0", "--port", "8000", "--jinja", "--reasoning", "off", "-c", "2048", "-np", "1", "-ngl", "0", "--threads", "{threads}", "--threads-batch", "{threads}", "--load-mode", "mmap+mlock"] +cap_add = ["IPC_LOCK"] +api = "/v1" +port = 8000 +golden = "canary/goldens/llama-cpu.json" +version_probe = { path = "/props", key = "build_info", regex = '^b(?P\d+)', tag = 'b(?P\d+)$' } +surface = { driver = "llama-server" } +flags_source = ["scripts/k3d-gpu:llamacpp_manifest", "console/fit.py:_flags", "console/fit.py:fit_groups", "console/model_opt.py:spec_decide"] + +[[engine]] +id = "eat-vllm-cpu" +kind = "runtime" +family = "vllm" +informational = true +locator = { file = "integrations/eat/vllm-values.yaml", name = "vllm/vllm-openai-cpu", regex = 'repository: vllm/vllm-openai-cpu\n tag: "(?P[^"@]+)@(?Psha256:[0-9a-f]{64})"' } +model = "Qwen/Qwen3-0.6B" +revision = "c1899de289a04d12100db370d81485cdf75e47ca" +files = ["config.json", "generation_config.json", "merges.txt", "model.safetensors", "tokenizer.json", "tokenizer_config.json", "vocab.json"] +mount = "/models/model" +args = ["{mount}", "--served-model-name={name}", "--port=8000", "--dtype=bfloat16", "--max-model-len=2048", "--max-num-seqs=1"] +env = ["VLLM_CPU_KVCACHE_SPACE=1"] +shm_size = "2g" # the pod's shm emptyDir (vllm_manifest): worker IPC +api = "/v1" +port = 8000 +golden = "canary/goldens/eat-vllm-cpu.json" +version_probe = { path = "/version", key = "version", regex = '^(?P\d+\.\d+\.\d+)', tag = '^v(?P\d+\.\d+\.\d+)$' } +surface = { driver = "vllm" } +# model_opt.py's eat_patch adds these to EAT's vLLM modelConfigs entry. +flags_source = ["console/model_opt.py:eat_patch", "console/model_opt.py:optimize", "console/model_opt.py:spec_decide"] +isa_signal = 'Illegal instruction|Failed to import from vllm\._C_AVX2' + +# ---- node ------------------------------------------------------------------------ + +[[engine]] +id = "node" +kind = "node" +locator = { file = "Dockerfile", name = "rancher/k3s", regex = 'ARG K3S_TAG="(?P[^"]+)"\nARG K3S_DIGEST="(?Psha256:[0-9a-f]{64})"' } +# Besides its k3s pin, the node is selected when these change. +paths = ["Dockerfile", "share/", "packaging/install.sh"] + +# ---- surface --------------------------------------------------------------------- + +[[engine]] +id = "vllm-xpu" +kind = "surface" +family = "vllm" +locator = { file = "scripts/k3d-gpu", regex = 'AGENT_DEFAULT_IMAGE="(?Pvllm/vllm-openai-xpu(?::[^@"]+)?@sha256:[0-9a-f]{64})"' } +version_probe = { package = "vllm", tag = '^v(?P\d+\.\d+\.\d+)$' } +surface = { driver = "vllm" } +flags_source = ["scripts/k3d-gpu:vllm_manifest", "scripts/k3d-gpu:VLLM_XPU_ENV", "console/fit.py:_flags", "console/model_opt.py:optimize", "console/model_opt.py:spec_decide"] +derived = [ + { file = "console/plan.py", regex = 'AGENT_VLLM_VERSION = \{[^}]*"GPU": "(?P[0-9.]+)"', of = "package" }, + { file = "console/plan.py", regex = 'AGENT_VLLM_VERSION\.get\(device, "(?P[0-9.]+)"\)', of = "package" }, + { file = "console/model_opt.py", regex = '\(o\.get\("vllm_version"\) or "(?P[0-9.]+)"\)', of = "package" }, + { file = "scripts/k3d-gpu", regex = 'device=\$\{AGENT_SEL%% \*\} vver=(?P[0-9.]+) ', of = "package" }, +] + +[[engine]] +id = "llm-scaler" +kind = "surface" +family = "vllm" +locator = { file = "scripts/k3d-gpu", regex = 'AGENT_DEFAULT_IMAGE="(?Pintel/llm-scaler-vllm(?::[^@"]+)?@sha256:[0-9a-f]{64})"' } +version_probe = { package = "vllm" } +surface = { driver = "vllm" } +flags_source = ["scripts/k3d-gpu:vllm_manifest", "scripts/k3d-gpu:VLLM_XPU_ENV", "console/fit.py:_flags", "console/model_opt.py:optimize", "console/model_opt.py:spec_decide"] +derived = [ + { file = "console/plan.py", regex = '"(?P[0-9.]+)" if backend == "llm-scaler"', of = "package" }, + { file = "scripts/k3d-gpu", regex = 'llm-scaler\) vver=(?P[0-9.]+) ;;', of = "package" }, +] + +[[engine]] +id = "gaudi-vllm" +kind = "surface" +family = "vllm" +locator = { file = "scripts/k3d-gpu", regex = 'AGENT_DEFAULT_IMAGE="(?Pvault\.habana\.ai/gaudi-docker/[^@":]+(?::[^@"]+)?@sha256:[0-9a-f]{64})"' } +version_probe = { package = "vllm" } +surface = { driver = "vllm" } +flags_source = ["scripts/k3d-gpu:vllm_manifest", "scripts/k3d-gpu:VLLM_HPU_ENV", "console/model_opt.py:optimize", "console/model_opt.py:spec_decide"] +derived = [ + { file = "console/plan.py", regex = 'AGENT_VLLM_VERSION = \{[^}]*"HPU": "(?P[0-9.]+)"', of = "package" }, + { file = "scripts/k3d-gpu", regex = '\[ -z "\$\{AGENT_HPU:-\}" \] \|\| vver=(?P[0-9.]+)', of = "package" }, +] + +[[engine]] +id = "llama-sycl" +kind = "surface" +family = "llama.cpp" +locator = { file = "scripts/k3d-gpu", regex = 'LLAMA_SYCL_IMAGE="\$\{K3D_GPU_LLAMA_SYCL_IMAGE:-(?Pghcr\.io/ggml-org/llama\.cpp(?::[^@"}]+)?@sha256:[0-9a-f]{64})\}"' } +version_probe = { args = ["--version"], regex = 'build (?P\d+)', tag = 'b(?P\d+)$' } +surface = { driver = "llama-server" } +flags_source = ["scripts/k3d-gpu:llamacpp_manifest", "console/fit.py:_flags", "console/fit.py:fit_groups", "console/model_opt.py:spec_decide"] + +[[engine]] +id = "llama-vulkan" +kind = "surface" +family = "llama.cpp" +locator = { file = "scripts/k3d-gpu", regex = 'LLAMA_VULKAN_IMAGE="\$\{K3D_GPU_LLAMA_VULKAN_IMAGE:-(?Pghcr\.io/ggml-org/llama\.cpp(?::[^@"}]+)?@sha256:[0-9a-f]{64})\}"' } +version_probe = { args = ["--version"], regex = 'build (?P\d+)', tag = 'b(?P\d+)$' } +surface = { driver = "llama-server" } +flags_source = ["scripts/k3d-gpu:llamacpp_manifest", "console/fit.py:_flags", "console/fit.py:fit_groups", "console/model_opt.py:spec_decide"] + +[[engine]] +id = "llama-full" +kind = "surface" +family = "llama-convert" +locator = { file = "scripts/k3d-gpu", regex = 'LLAMA_FULL_IMAGE="\$\{K3D_GPU_LLAMA_FULL_IMAGE:-(?Pghcr\.io/ggml-org/llama\.cpp(?::[^@"}]+)?@sha256:[0-9a-f]{64})\}"' } +version_probe = { entrypoint = "/app/llama-cli", args = ["--version"], regex = 'build (?P\d+)', tag = 'b(?P\d+)$' } +surface = { driver = "llama-convert" } +flags_source = ["scripts/k3d-gpu:gguf_convert", "scripts/k3d-gpu:llamacpp_manifest", "console/fit.py:convert_plan"] + +[[engine]] +id = "dlstreamer" +kind = "surface" +family = "dlstreamer" +locator = { file = "scripts/k3d-gpu", regex = 'AGENT_VIDEO_IMAGE="\$\{K3D_GPU_AGENT_VIDEO_IMAGE:-(?Pintel/dlstreamer(?::[^@"}]+)?@sha256:[0-9a-f]{64})\}"' } +version_probe = { entrypoint = "gst-inspect-1.0", args = ["--version"], regex = 'GStreamer (?P[0-9.]+)' } +# va* elements register only on a VA device (GStreamer's va plugin enumerates +# /dev/dri), which a hosted runner has none of: they are listed, not checked. +surface = { driver = "gst", device_only = ["va"] } +flags_source = ["scripts/k3d-gpu:agent_video_script"] + +[[engine]] +id = "sglang-xeon" +kind = "surface" +family = "sglang" +locator = { file = "console/moe_predict.py", regex = '"images": \("(?Plmsysorg/sglang(?::[^@"]+)?@sha256:[0-9a-f]{64})",\)' } +version_probe = { package = "sglang", tag = '^v(?P\d+\.\d+\.\d+)' } +surface = { driver = "sglang" } +flags_source = ["bench/h54/h54.py:sglang_args"] + +[[engine]] +id = "npu-umd" +kind = "surface" +family = "ovms" +locator = { file = "scripts/k3d-gpu", regex = 'NPU_UMD_IMAGE="(?Popenvino/model_server(?::[^@"]+)?@sha256:[0-9a-f]{64})"' } +version_probe = { args = ["--version"], regex = 'OpenVINO Model Server (?P\d+\.\d+\.\d+)', tag = '^(?P\d+\.\d+\.\d+)' } +# The NPU user space k3d-gpu checks host firmware against (npu_doctor): the driver +# package's version and the build date inside its firmware. +surface = { driver = "ovms", deb = "intel-level-zero-npu", firmware = "/lib/firmware/updates/intel/vpu/vpu_37xx_v1.bin" } +flags_source = ["scripts/k3d-gpu:agent_manifest", "scripts/k3d-gpu:agent_ovms_addon_manifest", "console/fit.py:_flags", "console/model_opt.py:spec_decide"] +derived = [ + { file = "scripts/k3d-gpu", regex = 'NPU_UMD_DRIVER="v(?P[0-9.]+)"', of = "deb" }, + { file = "scripts/k3d-gpu", regex = 'NPU_UMD_FW_DATE="(?P[0-9]{8})"', of = "firmware" }, +] + +[[engine]] +id = "ovms" +kind = "surface" +family = "ovms" +locator = { file = "scripts/k3d-gpu", regex = 'AGENT_OVMS_IMAGE="(?Popenvino/model_server(?::[^@"]+)?@sha256:[0-9a-f]{64})"' } +version_probe = { args = ["--version"], regex = 'OpenVINO Model Server (?P\d+\.\d+\.\d+)', tag = '^(?P\d+\.\d+\.\d+)' } +surface = { driver = "ovms" } +flags_source = ["scripts/k3d-gpu:agent_manifest", "scripts/k3d-gpu:agent_ovms_addon_manifest", "console/fit.py:_flags", "console/model_opt.py:spec_decide"] diff --git a/canary/flags.sh b/canary/flags.sh new file mode 100755 index 00000000..01dfdae8 --- /dev/null +++ b/canary/flags.sh @@ -0,0 +1,244 @@ +#!/usr/bin/env bash +# The engine flags and environment names k3d-gpu emits, per engine family, for the +# canary's surface check (canary/surface.py asserts each one exists in the image). +# +# canary/flags.sh [--root DIR] [FAMILY...] +# +# Prints one tab-separated line per name, sorted and unique: +# +# family vllm | ovms | llama.cpp | llama-convert | dlstreamer | sglang +# source where the name is emitted, : (console/*.py: :) +# kind flag | env | element | property +# name --flag or -f (no value), VLLM_* env name, GStreamer element, element.property +# +# Sources: +# scripts/k3d-gpu vllm_manifest (and VLLM_XPU_ENV, VLLM_HPU_ENV), agent_manifest (OVMS), +# llamacpp_manifest, gguf_convert, llamacpp_convert_job callers, +# agent_video_script (DL Streamer pipeline) +# console/model_opt.py, console/fit.py +# the optimizer's and the fit's engine arguments (agent_opt_items and +# AGENT_LLAMA_FLAGS feed them to the manifests): each string literal +# that starts with - in a list or an .append() argument, attributed to +# the engine of the innermost `engine == "X"` / `eng == "X"` test, or of +# the one engine left after earlier `if engine == "X": ... return` +# branches; FUNCTION_FAMILY names the functions with no engine test +# bench/h54/h54.py sglang_args, the only sglang command line in the repo (read only) +# A flag literal no rule attributes is an error, so a new emitter cannot go unchecked. + +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +if [ "${1:-}" = --root ]; then + [ $# -ge 2 ] || { echo "flags.sh: --root needs a directory" >&2; exit 2; } + ROOT=$2 + shift 2 +fi +LAUNCHER=${ROOT}/scripts/k3d-gpu +[ -f "${LAUNCHER}" ] || { echo "flags.sh: no ${LAUNCHER}" >&2; exit 1; } + +# fn_body : the launcher function's lines, comments dropped. +fn_body() { + awk -v f="$1" '$0 ~ "^"f"\\(\\) \\{" {p = 1} p {print} p && /^}/ {exit}' "${LAUNCHER}" \ + | grep -vE '^[[:space:]]*#' || true +} + +# array_body : the items of a launcher array NAME=( ... ). +array_body() { + awk -v a="$1" '$0 ~ "^"a"=\\($" {p = 1; next} p && /^\)/ {exit} p {print}' "${LAUNCHER}" \ + | grep -vE '^[[:space:]]*#' || true +} + +FLAG_RE='--?[A-Za-z][A-Za-z0-9_-]*' + +# shell_flags: flag words a manifest function puts in the server's argument list: +# "${item}--flag", a YAML list line "- --flag", llama_items arguments and the fit's +# default flags="..." string. +shell_flags() { + local body + body=$(cat) + { + grep -oE "\\\$\\{item\\}'?${FLAG_RE}" <<< "${body}" | sed -E "s/^\\\$\\{item\\}'?//" + grep -E '^[[:space:]]+- -' <<< "${body}" | sed -E "s/^[[:space:]]+- (${FLAG_RE}).*/\\1/" + grep -oE "\\\\n[[:space:]]+- ${FLAG_RE}" <<< "${body}" | sed -E 's/^\\n[[:space:]]+- //' + grep -oE 'llama_items [^)]*' <<< "${body}" | grep -oE "(^|[[:space:]\"])${FLAG_RE}" + grep -oE 'flags="-[^"]*"' <<< "${body}" | grep -oE "(^|[[:space:]\"])${FLAG_RE}" + } | tr -d ' "' | grep -xE -- "${FLAG_RE}" || true +} + +emit() { # emit : one line per name on stdin + local n + while IFS= read -r n; do + [ -z "${n}" ] || printf '%s\t%s\t%s\t%s\n' "$1" "$2" "$3" "${n}" + done +} + +{ + fn_body vllm_manifest | shell_flags | emit vllm scripts/k3d-gpu:vllm_manifest flag + for a in VLLM_XPU_ENV VLLM_HPU_ENV; do + array_body "${a}" | sed -nE 's/^[[:space:]]*"?(VLLM_[A-Z0-9_]+)=.*/\1/p' | emit vllm "scripts/k3d-gpu:${a}" env + done + fn_body agent_manifest | shell_flags | emit ovms scripts/k3d-gpu:agent_manifest flag + fn_body agent_ovms_addon_manifest | shell_flags | emit ovms scripts/k3d-gpu:agent_ovms_addon_manifest flag + fn_body llamacpp_manifest | shell_flags | emit llama.cpp scripts/k3d-gpu:llamacpp_manifest flag + # The conversion's own flags: gguf_convert's convert_hf_to_gguf.py call, and the + # arguments llamacpp_manifest passes to llamacpp_convert_job. + fn_body gguf_convert | sed -nE '/convert_hf_to_gguf\.py/{s/(^|[[:space:]"])(--[A-Za-z][A-Za-z0-9_-]*)/\n\2\n/g; p}' \ + | grep -xE -- "${FLAG_RE}" | emit llama-convert scripts/k3d-gpu:gguf_convert flag + fn_body llamacpp_manifest | sed -nE '/llamacpp_convert_job /{s/.*"(--[^"]*)"\).*/\1/p}' \ + | tr ' ' '\n' | grep -xE -- "${FLAG_RE}" | emit llama-convert scripts/k3d-gpu:llamacpp_manifest flag + python3 - "${ROOT}" <<'PY' +import ast +import re +import sys + +root = sys.argv[1] +FLAG = re.compile(r"^(--?[A-Za-z][A-Za-z0-9_-]*)") +ENVV = re.compile(r"^(VLLM_[A-Z0-9_]+)=") +ENGINES = {"vllm", "ovms", "llama.cpp"} +# Functions that emit engine arguments with no engine test of their own. +FUNCTION_FAMILY = { + ("console/model_opt.py", "eat_patch"): "vllm", # EAT's vLLM modelConfigs entry + ("console/fit.py", "convert_plan"): "llama-convert", # convert_hf_to_gguf.py arguments + ("console/fit.py", "fit_groups"): "llama.cpp", # -hf for a GGUF repo with several models +} +bad = [] + + +def eng_test(t): + if isinstance(t, ast.BoolOp) and isinstance(t.op, ast.And): + for v in t.values: + e = eng_test(v) + if e: + return e + return None + if (isinstance(t, ast.Compare) and len(t.ops) == 1 and isinstance(t.ops[0], ast.Eq) + and isinstance(t.left, ast.Name) and t.left.id in ("engine", "eng") + and isinstance(t.comparators[0], ast.Constant) and t.comparators[0].value in ENGINES): + return t.comparators[0].value + return None + + +def lead(e): + """The literal text an argument starts with: "x", "x" % y, f"x{y}".""" + if isinstance(e, ast.Constant) and isinstance(e.value, str): + return e.value + if isinstance(e, ast.BinOp) and isinstance(e.op, ast.Mod): + return lead(e.left) + if isinstance(e, ast.JoinedStr) and e.values: + return lead(e.values[0]) + return None + + +def literals(node): + out = [] + for n in ast.walk(node): + if isinstance(n, ast.List): + out += [(lead(e), e.lineno) for e in n.elts] + elif (isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute) and n.func.attr == "append" + and len(n.args) == 1): + out.append((lead(n.args[0]), n.args[0].lineno)) + return [(v, ln) for v, ln in out if v is not None] + + +def scan(rel): + found = [] + + def expr(node, fn, eng, excluded): + for v, ln in literals(node): + m, env = FLAG.match(v), ENVV.match(v) + if not m and not env: + continue + who = eng or FUNCTION_FAMILY.get((rel, fn)) + if who is None and len(ENGINES - excluded) == 1: + who = next(iter(ENGINES - excluded)) + if who is None: + bad.append("%s:%d (%s) %s" % (rel, ln, fn, v)) + continue + found.append((who, "%s:%s" % (rel, fn), "env" if env else "flag", (env or m).group(1))) + + def block(stmts, fn, eng, excluded): + excluded = set(excluded) + for s in stmts: + if isinstance(s, (ast.FunctionDef, ast.AsyncFunctionDef)): + block(s.body, s.name, None, set()) + elif isinstance(s, ast.ClassDef): + block(s.body, fn, eng, excluded) + elif isinstance(s, ast.If): + e = eng_test(s.test) + pure = e is not None and isinstance(s.test, ast.Compare) + block(s.body, fn, e or eng, excluded) + block(s.orelse, fn, eng, excluded | ({e} if pure else set())) + if pure and s.body and isinstance(s.body[-1], ast.Return): + excluded.add(e) + elif isinstance(s, (ast.For, ast.AsyncFor, ast.While, ast.With, ast.AsyncWith, ast.Try)): + for f in ("body", "orelse", "finalbody"): + block(getattr(s, f, None) or [], fn, eng, excluded) + for h in getattr(s, "handlers", None) or []: + block(h.body, fn, eng, excluded) + else: + expr(s, fn, eng, excluded) + + with open("%s/%s" % (root, rel), encoding="utf-8") as f: + block(ast.parse(f.read()).body, "", None, set()) + return found + + +rows = scan("console/model_opt.py") + scan("console/fit.py") + +# sglang: bench/h54/h54.py sglang_args() is the repo's sglang command line; its flags +# are the words after the module name (python3 -m sglang.launch_server ...). +try: + tree = ast.parse(open("%s/bench/h54/h54.py" % root, encoding="utf-8").read()) + for fn in ast.walk(tree): + if isinstance(fn, ast.FunctionDef) and fn.name == "sglang_args": + words = [v for v, _ in literals(fn)] + mods = [i for i, v in enumerate(words) if v.endswith("launch_server")] + for v in words[mods[0] + 1:] if mods else []: + m = FLAG.match(v) + if m: + rows.append(("sglang", "bench/h54/h54.py:sglang_args", "flag", m.group(1))) +except FileNotFoundError: + pass + +# DL Streamer: the elements and properties of agent_video_script's gst-launch pipeline, +# with ${src}, ${decode} and ${pre} expanded to every value the script assigns them. +text = open("%s/scripts/k3d-gpu" % root, encoding="utf-8").read() +m = re.search(r"^agent_video_script\(\) \{\n(.*?)^\}", text, re.M | re.S) +if m: + body = m.group(1) + assigned = {} + for k, v in re.findall(r"\b(\w+)=\"([^\"]*)\"", body): + assigned.setdefault(k, []).append(v) + pipe = re.search(r"gst-launch-1\.0 -e (.*?[^\\])\n", re.sub(r"\\\n\s*", " ", body)) + if pipe: + segments, props = [], [] + for seg in pipe.group(1).split("!"): + words = seg.split() + if not words: + continue + ref = re.fullmatch(r"\$\{(\w+)\}", words[0]) + if ref: + for alt in assigned.get(ref.group(1), []): + segments += [s.split() for s in alt.split("!") if s.split()] + continue + segments.append(words) + for words in segments: + el = words[0] + if "/" in el or not re.fullmatch(r"[a-z][a-z0-9_-]*", el): + continue # caps, not an element + rows.append(("dlstreamer", "scripts/k3d-gpu:agent_video_script", "element", el)) + for w in words[1:]: + ref = re.fullmatch(r"\$\{(\w+)\}", w) + for kv in (assigned.get(ref.group(1), []) if ref else [w]): + if "=" in kv: + rows.append(("dlstreamer", "scripts/k3d-gpu:agent_video_script", "property", + "%s.%s" % (el, kv.split("=", 1)[0]))) + +for r in rows: + print("\t".join(r)) +if bad: + sys.stderr.write("flags.sh: engine flag literals no rule attributes to an engine (name the function in FUNCTION_FAMILY):\n " + + "\n ".join(bad) + "\n") + sys.exit(1) +PY +} | if [ $# -gt 0 ]; then awk -F'\t' -v fams=" $* " 'index(fams, " " $1 " ")'; else cat; fi | LC_ALL=C sort -u diff --git a/canary/goldens/eat-vllm-cpu.json b/canary/goldens/eat-vllm-cpu.json new file mode 100644 index 00000000..08b88fbd --- /dev/null +++ b/canary/goldens/eat-vllm-cpu.json @@ -0,0 +1,12 @@ +{ + "prompt": "The capital of France is", + "regex": "Paris", + "exact": " Paris. The capital of France is also", + "recorded_with": { + "engine": "vllm/vllm-openai-cpu:v0.24.0 (0.24.0), --dtype=bfloat16, VLLM_CPU_KVCACHE_SPACE=1", + "model": "Qwen/Qwen3-0.6B@c1899de289a04d12100db370d81485cdf75e47ca", + "request": "POST /v1/completions temperature 0, top_k 1, seed 0, max_tokens 8", + "cpu": "GitHub-hosted ubuntu-latest runner (avx2, no avx512f), 4 CPUs; Canary run 40960775877", + "date": "2026-10-29" + } +} diff --git a/canary/goldens/llama-cpu.json b/canary/goldens/llama-cpu.json new file mode 100644 index 00000000..c6544904 --- /dev/null +++ b/canary/goldens/llama-cpu.json @@ -0,0 +1,12 @@ +{ + "prompt": "The capital of France is", + "regex": "Paris", + "exact": " Paris. The capital of Italy is Rome", + "recorded_with": { + "engine": "ghcr.io/ggml-org/llama.cpp:server-b11960 (build b11960-8642541b2)", + "model": "ggml-org/Qwen3-0.6B-GGUF@13ac06624948e4d8589cb4b6f90100837342fcb8 Qwen3-0.6B-Q8_0.gguf", + "request": "POST /v1/completions temperature 0, top_k 1, seed 0, max_tokens 8", + "cpu": "Intel Core i9-13900T (avx2, no avx512f), 2 CPUs", + "date": "2026-10-29" + } +} diff --git a/canary/goldens/ovms-cpu.json b/canary/goldens/ovms-cpu.json new file mode 100644 index 00000000..1107f633 --- /dev/null +++ b/canary/goldens/ovms-cpu.json @@ -0,0 +1,12 @@ +{ + "prompt": "The capital of France is", + "regex": "Paris", + "exact": " Paris. The capital of Italy is Rome", + "recorded_with": { + "engine": "openvino/model_server:2026.4.1-gpu (2026.4.1.cfc9ac721), CPU plugin", + "model": "OpenVINO/Qwen3-0.6B-int8-ov@ac8e917fb8350f528ada41b02c60ea3a2853955f", + "request": "POST /v3/completions temperature 0, top_k 1, seed 0, max_tokens 8", + "cpu": "Intel Core i9-13900T (avx2, no avx512f), 2 CPUs", + "date": "2026-10-29" + } +} diff --git a/canary/goldens/vllm-cpu.json b/canary/goldens/vllm-cpu.json new file mode 100644 index 00000000..c3abd780 --- /dev/null +++ b/canary/goldens/vllm-cpu.json @@ -0,0 +1,12 @@ +{ + "prompt": "The capital of France is", + "regex": "Paris", + "exact": " Paris. The capital of France is also", + "recorded_with": { + "engine": "vllm/vllm-openai-cpu:v0.32.0 (0.32.0), --dtype=bfloat16, VLLM_CPU_KVCACHE_SPACE=1", + "model": "Qwen/Qwen3-0.6B@c1899de289a04d12100db370d81485cdf75e47ca", + "request": "POST /v1/completions temperature 0, top_k 1, seed 0, max_tokens 8", + "cpu": "GitHub-hosted ubuntu-latest runner (avx2, no avx512f), 4 CPUs; Canary run 40960775877", + "date": "2026-10-29" + } +} diff --git a/canary/node.sh b/canary/node.sh new file mode 100755 index 00000000..9206ebcf --- /dev/null +++ b/canary/node.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# The node canary: build the node image from the root Dockerfile (Ubuntu 26.04, the +# default variant), start a k3d cluster on it with the k3d release packaging/install.sh +# pins (sha256-checked) and the arguments the launcher's cluster create passes on a host +# without NVIDIA (NODE_ARGS), and within CANARY_NODE_TIMEOUT seconds (default 300) see: +# - the node Ready; +# - every pod, in every namespace (kube-system, NFD, the device plugins), Running with +# its containers ready, or Succeeded. A plugin that needs hardware has no pod here +# (its node selector keeps it off a hosted runner), so a pod that exists must run: +# one still Pending at the timeout (an image that does not pull) fails; +# - every manifest the Dockerfile bakes into /var/lib/rancher/k3s/server/manifests; +# - every DaemonSet those baked manifests define (the device plugins, NFD); +# and then the pods hold for CANARY_NODE_SETTLE seconds (default 20) with no container +# restarting, so a container that crashes after it started is caught. Then delete the +# cluster. kubectl runs inside the node (k3s ships it), so nothing but docker and curl is +# needed on the host. CANARY_NODE_POLL (default 5) is the seconds between looks. +# +# Writes ${CANARY_OUT_DIR:-./canary-out}/node.json (kind node; pinned_ref the k3s pin) +# and node.log, and exits non-zero unless the verdict is pass. + +set -uo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "${HERE}/.." && pwd)" +OUT_DIR=${CANARY_OUT_DIR:-${PWD}/canary-out} +TIMEOUT=${CANARY_NODE_TIMEOUT:-300} +SETTLE=${CANARY_NODE_SETTLE:-20} +POLL=${CANARY_NODE_POLL:-5} +CLUSTER=canary +NODE=k3d-${CLUSTER}-server-0 +IMAGE_TAG=k3d-gpu-canary:node +# scripts/k3d-gpu's cluster create on a host without NVIDIA, as a hosted runner is: no +# load balancer, no traefik, and the node labelled nvidia.com/gpu.present=false, which +# keeps the baked NVIDIA plugin off it (with no driver it crash-loops). Both Intel stacks +# stay enabled, so every baked manifest is applied. tests/canary_spec_test.sh checks that +# the launcher still passes each of these. +NODE_ARGS=(--no-lb "--k3s-arg=--disable=traefik@server:0" "--k3s-node-label=nvidia.com/gpu.present=false@server:*;agent:*") +mkdir -p "${OUT_DIR}" +LOG=${OUT_DIR}/node.log +WORK=$(mktemp -d "${TMPDIR:-/tmp}/canary-node.XXXXXX") +: > "${LOG}" +K3D="" +cleanup() { + [ -z "${K3D}" ] || "${K3D}" cluster delete "${CLUSTER}" >> "${LOG}" 2>&1 + rm -rf "${WORK}" +} +trap cleanup EXIT +say() { echo "canary node: $*" >&2; } + +R_CPU="" +cpuflags=" $(grep -m1 '^flags' "${CANARY_CPUINFO:-/proc/cpuinfo}" | cut -d: -f2) " +for f in avx2 avx512f amx_tile; do [[ "${cpuflags}" == *" ${f} "* ]] && R_CPU+="${f} "; done +REF=$(python3 "${HERE}/plan.py" ref node 2>> "${LOG}") + +finish() { # finish + R_VERDICT=$1 R_DETAIL=$2 R_REF=${REF} R_CPU=${R_CPU} python3 - "${OUT_DIR}/node.json" <<'PY' +import json, os, sys +e = os.environ +ok = e["R_VERDICT"] == "pass" +res = {"kind": "node", "pinned_ref": e["R_REF"], "verdict": e["R_VERDICT"], "hard": ok, "exact": ok, + "cpu_flags": e["R_CPU"].split(), "detail": e["R_DETAIL"]} +with open(sys.argv[1], "w") as f: + json.dump(res, f, indent=1) + f.write("\n") +print("canary node: %s: %s" % (e["R_VERDICT"], e["R_DETAIL"])) +PY + [ "$1" = pass ] + exit +} +[ -n "${REF}" ] || finish fail "plan: the node's k3s pin does not resolve (see node.log)" + +# The node image, as the release job builds its default variant. +start=${SECONDS} +say "docker buildx build --load -t ${IMAGE_TAG} ." +docker buildx build --load -t "${IMAGE_TAG}" "${ROOT}" >> "${LOG}" 2>&1 \ + || finish fail "the node image did not build: $(grep -E 'ERROR|error:' "${LOG}" | tail -3 | tr '\n' ' ')" +built=$((SECONDS - start)) + +# k3d at the version and sha256 packaging/install.sh pins. +ver=$(sed -n 's/^K3D_VERSION=//p' "${ROOT}/packaging/install.sh") +sha=$(sed -n 's/^K3D_SHA256_AMD64=//p' "${ROOT}/packaging/install.sh") +[[ "${ver}" =~ ^v[0-9.]+$ && "${sha}" =~ ^[0-9a-f]{64}$ ]] || finish fail "packaging/install.sh pins no K3D_VERSION and K3D_SHA256_AMD64" +K3D=${WORK}/k3d +curl -fsSL --retry 3 -o "${K3D}" "https://github.com/k3d-io/k3d/releases/download/${ver}/k3d-linux-amd64" 2>> "${LOG}" \ + || { K3D=""; finish fail "k3d ${ver} did not download"; } +echo "${sha} ${K3D}" | sha256sum -c - >> "${LOG}" 2>&1 || { K3D=""; finish fail "k3d ${ver}: sha256 mismatch"; } +chmod +x "${K3D}" + +start=${SECONDS} +say "k3d ${ver} cluster create ${CLUSTER} --image ${IMAGE_TAG} --wait ${NODE_ARGS[*]}" +"${K3D}" cluster create "${CLUSTER}" --image "${IMAGE_TAG}" --wait --timeout "${TIMEOUT}s" "${NODE_ARGS[@]}" \ + --kubeconfig-update-default=false >> "${LOG}" 2>&1 \ + || finish fail "k3d cluster create failed: $(tail -3 "${LOG}" | tr '\n' ' ')" +created=$((SECONDS - start)) + +kc() { docker exec "${NODE}" kubectl "$@"; } +# The manifests the Dockerfile bakes, and the DaemonSets they define (namespace/name). +manifests=$(sed -nE 's#^COPY share/[^ ]+ /var/lib/rancher/k3s/server/manifests/([^ ]+)$#\1#p' "${ROOT}/Dockerfile" | sort) +daemonsets=$(sed -nE 's#^COPY (share/[^ ]+) /var/lib/rancher/k3s/server/manifests/[^ ]+$#\1#p' "${ROOT}/Dockerfile" \ + | (cd "${ROOT}" && xargs python3 -c ' +import re, sys +for path in sys.argv[1:]: + for doc in re.split(r"^---\s*$", open(path).read(), flags=re.M): + if re.search(r"^kind: DaemonSet\s*$", doc, re.M): + meta = re.search(r"^metadata:\n((?:[ ]+.*\n)+)", doc, re.M).group(1) + name = re.search(r"^ name: (\S+)", meta, re.M).group(1) + ns = re.search(r"^ namespace: (\S+)", meta, re.M) + print("%s/%s" % (ns.group(1) if ns else "default", name)) +') | sort -u) + +problems="" steady="" steady_since=${SECONDS} +deadline=$((SECONDS + TIMEOUT)) +while :; do + problems="" + ready=$(kc get nodes -o jsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>> "${LOG}") + [ "${ready}" = True ] || problems+="node not Ready (${ready:-no node}); " + have=$(docker exec "${NODE}" ls /var/lib/rancher/k3s/server/manifests 2>> "${LOG}") + for m in ${manifests}; do + grep -qx "${m}" <<< "${have}" || problems+="manifest ${m} not baked; " + done + ds=$(kc get ds -A -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' 2>> "${LOG}") + for d in ${daemonsets}; do + grep -qx "${d}" <<< "${ds}" || problems+="DaemonSet ${d} missing; " + done + # Every pod: Running with every container ready, or Succeeded. One line per pod: + # / =:,... + pods=$(kc get pods -A -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name} {.status.phase} {range .status.containerStatuses[*]}{.name}={.ready}:{.restartCount},{end}{"\n"}{end}' 2>> "${LOG}") + [ -n "${pods}" ] || problems+="no pods; " + while read -r name phase states; do + [ -n "${name}" ] || continue + case "${phase}" in + Succeeded) continue ;; + Running) [ -n "${states}" ] && [[ "${states}" != *=false:* ]] && continue ;; + esac + problems+="pod ${name} ${phase}${states:+ ${states%,}}; " + done <<< "${pods}" + if [ -z "${problems}" ]; then + # Up: it must now hold, the same pods with the same restart counts, for SETTLE. + [ "${pods}" = "${steady}" ] || { steady=${pods}; steady_since=${SECONDS}; } + [ $((SECONDS - steady_since)) -lt "${SETTLE}" ] || break + [ "${SECONDS}" -lt $((deadline + SETTLE)) ] || break + else + steady="" + [ "${SECONDS}" -lt "${deadline}" ] || break + fi + sleep "${POLL}" +done +if [ -z "${problems}" ] && [ $((SECONDS - steady_since)) -lt "${SETTLE}" ]; then + problems="the pods did not hold ${SETTLE}s without a restart: $(tr '\n' ' ' <<< "${pods}"); " +fi +waited=$((SECONDS - start)) +{ kc get nodes -o wide; kc get pods -A -o wide; kc get ds -A; } >> "${LOG}" 2>&1 +if [ -n "${problems}" ]; then + finish fail "after ${waited}s: ${problems%; }" +fi +finish pass "image built in ${built}s, cluster created in ${created}s, ready in ${waited}s; $(wc -w <<< "${manifests}") baked manifests, $(wc -w <<< "${daemonsets}") DaemonSets, $(grep -c . <<< "${pods}") pods up and steady for ${SETTLE}s" diff --git a/canary/plan.py b/canary/plan.py new file mode 100755 index 00000000..c920aec4 --- /dev/null +++ b/canary/plan.py @@ -0,0 +1,485 @@ +#!/usr/bin/env python3 +"""The canary's engine table (canary/engines.toml): where each engine's image is pinned, +what to check, which engines a change selects, and the merged report. + + plan.py ids the engine ids + plan.py ref ID the image reference ID's locator matches (exactly one) + plan.py show ID the engine as JSON: its table, the resolved pin (ref, + image, tag, digest), the version the tag implies, the + flags canary/flags.sh names for it, its derived + constants and its golden + plan.py shell ID the same as shell assignments, for canary/run.sh + plan.py model-dir ID the model cache directory (CANARY_MODELS, default + ~/.cache/k3d-gpu-canary) + plan.py matrix [--mode auto|all] [--engines CSV] [--base REF | --base-root DIR] + the engines to run and the workflow's job matrices + plan.py size REF the linux/amd64 image's compressed size, in bytes + plan.py report --selected JSON --parts DIR --sha SHA --ref REF --out FILE + canary-result.json from the per-engine results; exits + non-zero unless every selected engine is green + +Every subcommand takes --root DIR (default: this repo). In auto mode an engine is selected +when, between the base tree and this one, its resolved image reference, its table in +engines.toml, its golden, one of its derived constants or one of the flags canary/flags.sh +names for it changed (the node engine: also any file under its paths). A change to the +harness itself (canary/ outside engines.toml and goldens/, the defaults table, or +.github/workflows/canary.yml) selects every engine. --base REF compares with +`git merge-base REF HEAD`; --base-root compares with a directory. + +Python standard library only (tomllib: Python 3.11+). +""" + +import argparse +import hashlib +import json +import os +import re +import shlex +import shutil +import subprocess +import sys +import tempfile +import tomllib + +HERE = os.path.dirname(os.path.abspath(__file__)) +REPO = os.path.dirname(HERE) +TOML = "canary/engines.toml" +WORKFLOW = ".github/workflows/canary.yml" +KINDS = ("runtime", "surface", "node") +VERDICTS = ("pass", "fail", "isa-skip", "too-big", "unparseable") +RESULT_KEYS = ("kind", "pinned_ref", "verdict", "hard", "exact", "cpu_flags", "detail") +# Files every plan reads besides the locators, derived files and node paths the table names. +BASE_PATHS = ("canary", WORKFLOW, "scripts/k3d-gpu", "console/model_opt.py", "console/fit.py", "bench/h54/h54.py") + + +class PlanError(Exception): + pass + + +def load(root): + path = os.path.join(root, TOML) + if not os.path.exists(path): + return {}, {} + with open(path, "rb") as f: + doc = tomllib.load(f) + engines = {} + for e in doc.get("engine", []): + if e.get("id") in engines: + raise PlanError("%s: engine %s is listed twice" % (TOML, e["id"])) + if e.get("kind") not in KINDS: + raise PlanError("%s: engine %s has kind %r, not one of %s" % (TOML, e.get("id"), e.get("kind"), ", ".join(KINDS))) + engines[e["id"]] = e + return doc.get("defaults", {}), engines + + +def read(root, rel): + with open(os.path.join(root, rel), encoding="utf-8") as f: + return f.read() + + +def split_ref(ref): + """name, tag ('' when none) and digest of name[:tag]@sha256:.""" + m = re.fullmatch(r"([^@\s]+)@(sha256:[0-9a-f]{64})", ref) + if not m: + raise PlanError("%r is not name[:tag]@sha256:<64 hex>" % ref) + name, digest = m.groups() + tag = "" + if ":" in name.rsplit("/", 1)[-1]: + name, tag = name.rsplit(":", 1) + return name, tag, digest + + +def resolve(eng, root): + """The engine's pin: {ref, image (its name), tag, digest}, from exactly one locator match.""" + loc = eng["locator"] + try: + text = read(root, loc["file"]) + except FileNotFoundError: + raise PlanError("%s: locator file %s does not exist" % (eng["id"], loc["file"])) + matches = list(re.finditer(loc["regex"], text, re.M)) + if len(matches) != 1: + raise PlanError("%s: locator %r matches %d image references in %s, not exactly one" + % (eng["id"], loc["regex"], len(matches), loc["file"])) + g = matches[0].groupdict() + if g.get("ref"): + name, tag, digest = split_ref(g["ref"]) + else: + name, tag, digest = loc["name"], g.get("tag") or "", g["digest"] + ref = "%s%s@%s" % (name, ":" + tag if tag else "", digest) + return {"ref": ref, "image": name, "tag": tag, "digest": digest} + + +def expected_version(probe, tag): + """The version the pinned tag names, read with version_probe.tag; '' when the pin + has no tag or the probe names no tag regex.""" + rx = (probe or {}).get("tag") + if not rx or not tag: + return "" + m = re.search(rx, tag) + if not m: + raise PlanError("tag %r does not match version_probe.tag %r" % (tag, rx)) + return m.group("v") + + +_flags_cache = {} + + +def flags_lines(root): + if root not in _flags_cache: + out = subprocess.run([os.path.join(HERE, "flags.sh"), "--root", root], capture_output=True, text=True) + if out.returncode != 0: + raise PlanError("canary/flags.sh failed on %s: %s" % (root, out.stderr.strip())) + _flags_cache[root] = [dict(zip(("family", "source", "kind", "name"), ln.split("\t"))) + for ln in out.stdout.splitlines() if ln] + return _flags_cache[root] + + +def flags_for(eng, root): + srcs = set(eng.get("flags_source", [])) + return [{"source": r["source"], "kind": r["kind"], "name": r["name"]} for r in flags_lines(root) + if r["family"] == eng.get("family") and r["source"] in srcs] + + +def derived_for(eng, root): + out = [] + for d in eng.get("derived", []): + try: + text = read(root, d["file"]) + except FileNotFoundError: + raise PlanError("%s: derived file %s does not exist" % (eng["id"], d["file"])) + ms = list(re.finditer(d["regex"], text, re.M)) + if len(ms) != 1: + raise PlanError("%s: derived %r matches %d times in %s, not exactly once" % (eng["id"], d["regex"], len(ms), d["file"])) + out.append({"file": d["file"], "regex": d["regex"], "of": d["of"], "value": ms[0].group("v")}) + return out + + +def model_dir(eng): + base = os.environ.get("CANARY_MODELS") or os.path.join(os.path.expanduser("~"), ".cache", "k3d-gpu-canary") + return os.path.join(base, model_slug(eng)) + + +def model_slug(eng): + return "%s@%s" % (eng["model"].replace("/", "__"), eng["revision"]) + + +def subst(s, eng, name): + mount = eng.get("mount", "").replace("{model}", eng.get("model", "")) + return (s.replace("{mount}", mount).replace("{model}", eng.get("model", "")).replace("{name}", name) + .replace("{threads}", str(os.cpu_count() or 1))) + + +def show(eid, root): + defaults, engines = load(root) + if eid not in engines: + raise PlanError("no engine %r in %s (engines: %s)" % (eid, TOML, ", ".join(sorted(engines)))) + eng = dict(engines[eid]) + doc = {"ready_timeout": defaults.get("ready_timeout", 600), "size_factor": defaults.get("size_factor", 3.5), + "name": defaults.get("name", "canary"), "informational": False} + doc.update(eng) + doc.update(resolve(eng, root)) + doc["pinned_ref"] = doc["ref"] + doc["expected_version"] = expected_version(eng.get("version_probe"), doc["tag"]) + if eng["kind"] != "node": + doc["flags"] = flags_for(eng, root) + doc["derived"] = derived_for(eng, root) + if eng["kind"] == "runtime": + doc["model_dir"] = model_dir(eng) + doc["golden_doc"] = json.loads(read(root, eng["golden"])) + name = doc["name"] + for k in ("args", "env", "prepare", "prepare_env"): + doc[k] = [subst(a, eng, name) for a in eng.get(k, [])] + doc["mount"] = subst(eng.get("mount", "/models"), eng, name) + return doc + + +def shell(doc): + def one(k, v): + return "%s=%s" % (k, shlex.quote(str(v))) + + def arr(k, vs): + return "%s=(%s)" % (k, " ".join(shlex.quote(str(v)) for v in vs)) + probe = doc.get("version_probe") or {} + lines = [one("E_ID", doc["id"]), one("E_KIND", doc["kind"]), one("E_INFORMATIONAL", "1" if doc["informational"] else ""), + one("E_REF", doc["ref"]), one("E_TAG", doc["tag"]), one("E_SIZE_FACTOR", doc["size_factor"]), + one("E_READY_TIMEOUT", doc["ready_timeout"]), one("E_SURFACE", (doc.get("surface") or {}).get("driver", "")), + one("E_ISA_SIGNAL", doc.get("isa_signal", "")), one("E_EXPECT_VERSION", doc["expected_version"])] + if doc["kind"] == "runtime": + lines += [one("E_MODEL", doc["model"]), one("E_REVISION", doc["revision"]), one("E_MODEL_DIR", doc["model_dir"]), + arr("E_FILES", doc["files"]), one("E_MOUNT", doc["mount"]), one("E_WRITABLE", doc.get("writable", "")), + arr("E_ARGS", doc["args"]), arr("E_ENV", doc["env"]), arr("E_PREPARE", doc["prepare"]), + arr("E_PREPARE_ENV", doc["prepare_env"]), arr("E_CAP_ADD", doc.get("cap_add", [])), + one("E_SHM_SIZE", doc.get("shm_size", "")), + one("E_API", doc["api"]), one("E_PORT", doc["port"]), one("E_NAME", doc["name"]), + one("E_GOLDEN", doc["golden"]), one("E_VERSION_PATH", probe.get("path", "")), + one("E_VERSION_KEY", probe.get("key", "")), one("E_VERSION_REGEX", probe.get("regex", ""))] + return "\n".join(lines) + + +# ---- planning ---------------------------------------------------------------------- + +def digest_paths(root, paths): + h = hashlib.sha256() + for p in paths: + full = os.path.join(root, p) + files = [] + if os.path.isdir(full): + for d, _, fs in os.walk(full): + files += [os.path.join(d, f) for f in fs] + elif os.path.exists(full): + files = [full] + for f in sorted(files): + h.update(os.path.relpath(f, root).encode() + b"\0") + with open(f, "rb") as fh: + h.update(fh.read()) + return h.hexdigest() + + +def fingerprint(eid, root): + """What an engine's canary depends on in a tree; an error is part of it (a broken + locator in the head tree is a change that must run, and fail).""" + defaults, engines = load(root) + eng = engines.get(eid) + if eng is None: + return None + fp = {"table": json.dumps(eng, sort_keys=True)} + for key, fn in (("pin", lambda: resolve(eng, root)["ref"]), + ("derived", lambda: [d["value"] for d in derived_for(eng, root)]), + ("flags", lambda: sorted(json.dumps(f, sort_keys=True) for f in flags_for(eng, root)) + if eng["kind"] != "node" else []), + ("golden", lambda: read(root, eng["golden"]) if eng.get("golden") else ""), + ("paths", lambda: digest_paths(root, eng.get("paths", [])))): + try: + fp[key] = fn() + except (PlanError, OSError) as e: + fp[key] = "error: %s" % e + return fp + + +def harness(root): + """The canary's own code and settings: a change here runs every engine.""" + h = hashlib.sha256() + canary = os.path.join(root, "canary") + for d, _, fs in sorted(os.walk(canary)) if os.path.isdir(canary) else []: + rel = os.path.relpath(d, root) + if rel == "canary/goldens" or rel.startswith("canary/goldens/") or "__pycache__" in rel: + continue + for f in sorted(fs): + if (rel == "canary" and f == "engines.toml") or f.endswith(".md"): + continue + h.update(os.path.join(rel, f).encode() + b"\0") + with open(os.path.join(d, f), "rb") as fh: + h.update(fh.read()) + defaults, _ = load(root) + h.update(json.dumps(defaults, sort_keys=True).encode()) + wf = os.path.join(root, WORKFLOW) + if os.path.exists(wf): + with open(wf, "rb") as fh: + h.update(fh.read()) + return h.hexdigest() + + +def git(root, *args): + out = subprocess.run(["git", "-C", root, *args], capture_output=True, text=True) + if out.returncode != 0: + raise PlanError("git %s: %s" % (" ".join(args), out.stderr.strip())) + return out.stdout.strip() + + +def base_tree(root, ref, dest): + """The files a plan reads, from `git merge-base REF HEAD`, written under dest.""" + commit = git(root, "merge-base", ref, "HEAD") + _, engines = load(root) + paths = set(BASE_PATHS) + for e in engines.values(): + paths.add(e["locator"]["file"]) + paths.update(d["file"] for d in e.get("derived", [])) + paths.update(p.rstrip("/") for p in e.get("paths", [])) + present = [p for p in sorted(paths) + if subprocess.run(["git", "-C", root, "cat-file", "-e", "%s:%s" % (commit, p)], capture_output=True).returncode == 0] + if present: + arc = subprocess.run(["git", "-C", root, "archive", "--format=tar", commit, "--", *present], capture_output=True, check=True) + subprocess.run(["tar", "-x", "-C", dest], input=arc.stdout, check=True) + return commit + + +def matrix(root, mode, engines_csv, base=None, base_root=None): + defaults, engines = load(root) + ids = list(engines) + if engines_csv and engines_csv != "auto": + want = [e.strip() for e in engines_csv.split(",") if e.strip()] + unknown = [e for e in want if e not in engines] + if unknown: + raise PlanError("unknown engine id(s): %s (engines: %s)" % (", ".join(unknown), ", ".join(ids))) + chosen = [e for e in ids if e in want] + elif mode == "all": + chosen = ids + else: + tmp = None + try: + if base_root is None: + tmp = tempfile.mkdtemp(prefix="canary-base.") + base_tree(root, base or "origin/main", tmp) + base_root = tmp + if harness(base_root) != harness(root): + chosen = ids + else: + chosen = [e for e in ids if fingerprint(e, base_root) != fingerprint(e, root)] + finally: + if tmp: + shutil.rmtree(tmp, ignore_errors=True) + selected, runtime, surface, node = [], [], [], False + for eid in chosen: + e = engines[eid] + try: + ref = resolve(e, root)["ref"] + except PlanError: + ref = "" + selected.append({"id": eid, "kind": e["kind"], "informational": bool(e.get("informational")), "pinned_ref": ref}) + if e["kind"] == "runtime": + slug = model_slug(e) + runtime.append({"id": eid, "model": e["model"], "revision": e["revision"], + "cache_key": "canary-model-" + slug, "cache_path": "~/.cache/k3d-gpu-canary/" + slug}) + elif e["kind"] == "surface": + surface.append({"id": eid}) + else: + node = True + return {"selected": selected, "runtime": runtime, "surface": surface, "node": node} + + +# ---- images -------------------------------------------------------------------------- + +def image_bytes(ref): + """Compressed bytes of the linux/amd64 image (layers and config), from the registry.""" + def raw(r): + out = subprocess.run(["docker", "buildx", "imagetools", "inspect", "--raw", r], capture_output=True, text=True) + if out.returncode != 0: + raise PlanError("cannot read the manifest of %s: %s" % (r, out.stderr.strip())) + return json.loads(out.stdout) + m = raw(ref) + if "manifests" in m: + amd = [x for x in m["manifests"] if (x.get("platform") or {}).get("os") == "linux" + and (x.get("platform") or {}).get("architecture") == "amd64"] + if not amd: + raise PlanError("%s has no linux/amd64 image" % ref) + m = raw("%s@%s" % (ref.split("@")[0], amd[0]["digest"])) + return sum(int(x.get("size", 0)) for x in m.get("layers", [])) + int((m.get("config") or {}).get("size", 0)) + + +# ---- report -------------------------------------------------------------------------- + +def report(selected, parts, sha, ref, out, summary=None, github_output=None): + engines, red, isa = {}, [], [] + for s in selected: + p = os.path.join(parts, s["id"] + ".json") + r = None + try: + with open(p, encoding="utf-8") as f: + r = json.load(f) + if not all(k in r for k in RESULT_KEYS) or r["verdict"] not in VERDICTS: + r = dict(r, verdict="fail", hard=False, exact=False, + detail="malformed result (%s): %s" % (p, json.dumps(r)[:200])) + except (OSError, ValueError): + r = {"kind": s.get("kind", "runtime"), "pinned_ref": s.get("pinned_ref", ""), "verdict": "fail", "hard": False, + "exact": False, "cpu_flags": [], "detail": "no result: the job failed or was cancelled before writing one"} + r = {k: r.get(k) for k in RESULT_KEYS} + r["kind"] = r["kind"] if r["kind"] in KINDS else s.get("kind", "runtime") + r["pinned_ref"] = str(r["pinned_ref"] or s.get("pinned_ref", "")) + r["hard"], r["exact"] = bool(r["hard"]), bool(r["exact"]) + r["cpu_flags"] = list(r["cpu_flags"] or []) + r["detail"] = str(r["detail"] or "") + engines[s["id"]] = r + green = r["verdict"] == "pass" and r["hard"] + if not green and not s.get("informational"): + red.append(s["id"]) + if r["verdict"] == "isa-skip" and not s.get("informational"): + isa.append(s["id"]) # the workflow re-dispatches these, hoping for another CPU + with open(out, "w", encoding="utf-8") as f: + json.dump({"sha": sha, "ref": ref, "engines": engines}, f, indent=1, sort_keys=True) + f.write("\n") + rows = ["| engine | kind | verdict | hard | exact | pinned | detail |", "|---|---|---|---|---|---|---|"] + info = {s["id"] for s in selected if s.get("informational")} + for eid, r in engines.items(): + rows.append("| %s%s | %s | %s | %s | %s | `%s` | %s |" % ( + eid, " (informational)" if eid in info else "", r["kind"], r["verdict"], r["hard"], r["exact"], + r["pinned_ref"].split("@")[0], r["detail"].replace("|", "/").replace("\n", " ")[:300])) + text = "## Canary %s\n\n%s\n\n%s\n" % ( + "green" if not red else "red: " + ", ".join(red), + "\n".join(rows) if engines else "No engine selected: nothing an engine depends on changed.", + "Green means verdict pass with hard true. exact false (a golden mismatch) stays green but is not auto-merge evidence.") + print(text) + if summary: + with open(summary, "a", encoding="utf-8") as f: + f.write(text) + if github_output: + with open(github_output, "a", encoding="utf-8") as f: + f.write("isa_skip=%s\ngreen=%s\n" % (",".join(isa), "false" if red else "true")) + return 1 if red else 0 + + +def main(argv=None): + p = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + p.add_argument("--root", default=REPO) + sub = p.add_subparsers(dest="cmd", required=True) + sub.add_parser("ids") + for c in ("ref", "show", "shell", "model-dir"): + s = sub.add_parser(c) + s.add_argument("id") + s.add_argument("--root", default=argparse.SUPPRESS) + m = sub.add_parser("matrix") + m.add_argument("--mode", choices=("auto", "all"), default="auto") + m.add_argument("--engines", default="auto") + m.add_argument("--base") + m.add_argument("--base-root") + m.add_argument("--root", default=argparse.SUPPRESS) + m.add_argument("--github-output") + s = sub.add_parser("size") + s.add_argument("ref") + r = sub.add_parser("report") + r.add_argument("--selected", required=True, help="JSON list, or @FILE") + r.add_argument("--parts", required=True) + r.add_argument("--sha", required=True) + r.add_argument("--ref", required=True) + r.add_argument("--out", required=True) + r.add_argument("--summary") + r.add_argument("--github-output") + a = p.parse_args(argv) + root = os.path.abspath(a.root) + try: + if a.cmd == "ids": + print("\n".join(load(root)[1])) + elif a.cmd == "ref": + _, engines = load(root) + if a.id not in engines: + raise PlanError("no engine %r" % a.id) + print(resolve(engines[a.id], root)["ref"]) + elif a.cmd == "show": + print(json.dumps(show(a.id, root), indent=1, sort_keys=True)) + elif a.cmd == "shell": + print(shell(show(a.id, root))) + elif a.cmd == "model-dir": + print(model_dir(load(root)[1][a.id])) + elif a.cmd == "matrix": + doc = matrix(root, a.mode, a.engines, a.base, os.path.abspath(a.base_root) if a.base_root else None) + print(json.dumps(doc, indent=1)) + if a.github_output: + with open(a.github_output, "a", encoding="utf-8") as f: + for k in ("runtime", "surface", "selected"): + f.write("%s=%s\n" % (k, json.dumps(doc[k], separators=(",", ":")))) + f.write("node=%s\n" % ("true" if doc["node"] else "false")) + elif a.cmd == "size": + print(image_bytes(a.ref)) + elif a.cmd == "report": + sel = a.selected + if sel.startswith("@"): + with open(sel[1:], encoding="utf-8") as f: + sel = f.read() + return report(json.loads(sel), a.parts, a.sha, a.ref, a.out, a.summary, a.github_output) + except PlanError as e: + print("plan.py: %s" % e, file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/canary/probe.py b/canary/probe.py new file mode 100755 index 00000000..15389f08 --- /dev/null +++ b/canary/probe.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""The canary's runtime checks against a serving engine (canary/run.sh starts it). + + probe.py --base URL --api /v1 --name NAME --golden FILE --timeout S --out FILE + [--container NAME] [--version-path P --version-key K --version-regex R] + [--expect-version V] + +1. Ready: GET {api}/models until it lists NAME, for up to --timeout seconds; with + --container, stop early once `docker inspect` says the container exited. +2. One completion: POST {api}/completions, the golden's prompt, temperature 0, top_k 1, + seed 0, max_tokens 8. Hard: HTTP 200, finish_reason stop or length, + usage.completion_tokens > 0 and the text matches the golden's regex. +3. One streamed completion: at least 2 chunks with choices, then data: [DONE] last. +4. The version the server reports (GET --version-path, JSON key --version-key, read with + --version-regex) equals --expect-version, when both are known. +5. Exact: the text equals the golden's exact text (null: none recorded). A mismatch keeps + hard and clears exact. + +Writes {ready, ready_s, exited, exit_code, hard, exact, text, version, stream_chunks, +failures, notes} to --out. Python standard library only. +""" + +import argparse +import json +import re +import subprocess +import time +import urllib.error +import urllib.request + +MAX_TOKENS = 8 + + +def http(url, body=None, timeout=60): + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request(url, data=data, headers={"Content-Type": "application/json"} if data else {}) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.status, r.read() + except urllib.error.HTTPError as e: + return e.code, e.read() + + +def container_state(name): + """(running, exit code) of a container, (True, None) when docker cannot tell.""" + try: + out = subprocess.run(["docker", "inspect", "-f", "{{.State.Running}} {{.State.ExitCode}}", name], + capture_output=True, text=True, timeout=30).stdout.split() + return out[0] == "true", int(out[1]) + except (OSError, ValueError, IndexError, subprocess.TimeoutExpired): + return True, None + + +def wait_ready(a, res): + start = time.monotonic() + while True: + try: + code, body = http(a.base + a.api + "/models", timeout=10) + if code == 200 and a.name in [m.get("id") for m in json.loads(body).get("data", [])]: + res["ready"], res["ready_s"] = True, round(time.monotonic() - start, 1) + return True + except (OSError, ValueError, AttributeError): + pass + if a.container: + running, rc = container_state(a.container) + if not running: + res["exited"], res["exit_code"] = True, rc + res["failures"].append("the engine exited (code %s) before it listed %s" % (rc, a.name)) + return False + if time.monotonic() - start >= a.timeout: + res["failures"].append("never ready: %s%s/models did not list %s within %ds" % (a.base, a.api, a.name, a.timeout)) + return False + time.sleep(min(2.0, max(0.2, a.timeout / 20))) + + +def request(a, golden, stream): + body = {"model": a.name, "prompt": golden["prompt"], "max_tokens": MAX_TOKENS, "temperature": 0, "top_k": 1, "seed": 0} + if stream: + body["stream"] = True + return body + + +def complete(a, golden, res): + code, raw = http(a.base + a.api + "/completions", request(a, golden, False), timeout=300) + if code != 200: + res["failures"].append("completion: HTTP %d %s" % (code, raw[:200].decode(errors="replace"))) + return + try: + doc = json.loads(raw) + choice = doc["choices"][0] + text = choice.get("text") or "" + fin = choice.get("finish_reason") + tokens = (doc.get("usage") or {}).get("completion_tokens") or 0 + except (ValueError, KeyError, IndexError, TypeError) as e: + res["failures"].append("completion: unreadable body (%s): %s" % (e, raw[:200].decode(errors="replace"))) + return + res["text"] = text + if fin not in ("stop", "length"): + res["failures"].append("completion: finish_reason %r, not stop or length" % fin) + if tokens <= 0: + res["failures"].append("completion: usage.completion_tokens is %r" % tokens) + if not re.search(golden["regex"], text): + res["failures"].append("completion: text %r does not match /%s/" % (text, golden["regex"])) + + +def stream(a, golden, res): + req = urllib.request.Request(a.base + a.api + "/completions", data=json.dumps(request(a, golden, True)).encode(), + headers={"Content-Type": "application/json"}) + events = [] + try: + with urllib.request.urlopen(req, timeout=300) as r: + for line in r: + line = line.decode(errors="replace").strip() + if line.startswith("data:"): + events.append(line[5:].strip()) + except urllib.error.HTTPError as e: + res["failures"].append("stream: HTTP %d" % e.code) + return + except OSError as e: + res["failures"].append("stream: %s" % e) + return + chunks = 0 + for ev in events: + if ev == "[DONE]": + continue + try: + doc = json.loads(ev) + except ValueError: + res["failures"].append("stream: a chunk is not JSON: %r" % ev[:120]) + return + if "error" in doc: + res["failures"].append("stream: error chunk %s" % json.dumps(doc["error"])[:200]) + return + if doc.get("choices"): + chunks += 1 + res["stream_chunks"] = chunks + if chunks < 2: + res["failures"].append("stream: %d chunks with choices, want at least 2" % chunks) + if not events or events[-1] != "[DONE]": + res["failures"].append("stream: did not end with data: [DONE]") + + +def version(a, res): + if not a.version_path: + return + code, raw = http(a.base + a.version_path, timeout=30) + try: + v = str(json.loads(raw)[a.version_key]) + except (ValueError, KeyError, TypeError): + res["failures"].append("version: GET %s answered %d without %s" % (a.version_path, code, a.version_key)) + return + m = re.search(a.version_regex, v) if a.version_regex else None + res["version"] = m.group("v") if m else v + if not a.expect_version: + res["notes"].append("server version %s (the pin has no tag to compare)" % v) + elif res["version"] != a.expect_version: + res["failures"].append("version: the server reports %s, the pinned tag says %s" % (v, a.expect_version)) + + +def main(): + p = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + p.add_argument("--base", required=True) + p.add_argument("--api", required=True) + p.add_argument("--name", required=True) + p.add_argument("--golden", required=True) + p.add_argument("--timeout", type=float, required=True) + p.add_argument("--out", required=True) + p.add_argument("--container") + p.add_argument("--version-path", default="") + p.add_argument("--version-key", default="version") + p.add_argument("--version-regex", default="") + p.add_argument("--expect-version", default="") + a = p.parse_args() + with open(a.golden, encoding="utf-8") as f: + golden = json.load(f) + res = {"ready": False, "ready_s": None, "exited": False, "exit_code": None, "hard": False, "exact": False, + "text": None, "version": None, "stream_chunks": 0, "failures": [], "notes": []} + if wait_ready(a, res): + complete(a, golden, res) + stream(a, golden, res) + version(a, res) + res["hard"] = not res["failures"] + res["exact"] = res["hard"] and golden.get("exact") is not None and res["text"] == golden["exact"] + if res["hard"] and golden.get("exact") is None: + res["notes"].append("no exact golden recorded") + elif res["hard"] and not res["exact"]: + res["notes"].append("exact: %r is not the golden %r" % (res["text"], golden["exact"])) + with open(a.out, "w", encoding="utf-8") as f: + json.dump(res, f, indent=1) + + +if __name__ == "__main__": + main() diff --git a/canary/run.sh b/canary/run.sh new file mode 100755 index 00000000..dfa13665 --- /dev/null +++ b/canary/run.sh @@ -0,0 +1,225 @@ +#!/usr/bin/env bash +# One engine's canary on this machine: a GitHub-hosted runner in .github/workflows/canary.yml. +# +# canary/run.sh ID a runtime engine: its surface check, then serve and probe it +# canary/run.sh --surface ID a surface engine: the surface check only +# +# In order: (1) free disk, when the image needs more than is free and CANARY_FREE_DISK=1 +# (it deletes the runner's preinstalled toolchains: never set it on a workstation); +# (2) record the CPU's avx2, avx512f and amx_tile flags; (3) the model at its pinned +# revision, from the cache (actions/cache restores it) or downloaded; (4) the image's +# size against the free disk (too-big), then docker pull, 3 attempts with backoff; +# (5) the surface check (canary/surface.py); (6) start the engine as the launcher would +# (canary/engines.toml) and probe it (canary/probe.py): ready within the timeout, one +# completion, one stream, its version; (7) an engine that cannot run on this ISA +# (isa_signal, no avx512f) is isa-skip, not a failure. +# +# Writes ${CANARY_OUT_DIR:-./canary-out}/.json (the canary-result.json entry: kind, +# pinned_ref, verdict, hard, exact, cpu_flags, detail) and .log (the engine's log), +# and exits non-zero unless the verdict is pass (an informational engine exits 0). +# +# Environment: CANARY_MODELS (model cache, default ~/.cache/k3d-gpu-canary), +# CANARY_READY_TIMEOUT (seconds, default the engine's), CANARY_PULL_BACKOFF (seconds, +# default 20), CANARY_CPUINFO (default /proc/cpuinfo), CANARY_FREE_BYTES (free disk, +# default df of docker's root), HF_ENDPOINT (default https://huggingface.co). + +set -uo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +OUT_DIR=${CANARY_OUT_DIR:-${PWD}/canary-out} +surface_only="" +if [ "${1:-}" = --surface ]; then surface_only=1; shift; fi +ID=${1:?usage: canary/run.sh [--surface] ID} +mkdir -p "${OUT_DIR}" +RESULT=${OUT_DIR}/${ID}.json +LOG=${OUT_DIR}/${ID}.log +WORK=$(mktemp -d "${TMPDIR:-/tmp}/canary-${ID}.XXXXXX") +NAME=canary-${ID} +: > "${LOG}" +E_KIND=${surface_only:+surface} E_REF="" E_INFORMATIONAL="" E_SURFACE="" +cleanup() { + docker rm -f "${NAME}" > /dev/null 2>&1 + # an engine writes as its own user into the copied model tree + rm -rf "${WORK}" 2>/dev/null || docker run --rm -v "${WORK}:/w" --entrypoint rm "${E_REF}" -rf /w/models > /dev/null 2>&1 + rm -rf "${WORK}" 2>/dev/null + return 0 +} +trap cleanup EXIT + +say() { echo "canary ${ID}: $*" >&2; } + +# finish : write the result and exit. A pass needs the surface check +# (when there was one) and the probe (for a runtime engine) to pass. +finish() { + R_VERDICT=$1 R_DETAIL=$2 python3 - "${RESULT}" "${WORK}" <<'PY' +import json, os, sys +out, work = sys.argv[1:] +env = os.environ +verdict, detail = env["R_VERDICT"], [env["R_DETAIL"]] if env["R_DETAIL"] else [] +hard = exact = False +def load(name): + try: + with open(os.path.join(work, name)) as f: + return json.load(f) + except (OSError, ValueError): + return None +surface, probe = load("surface.json"), load("probe.json") +if surface: + detail.append(surface["detail"]) +if probe: + detail += probe["failures"] + probe["notes"] + if probe["text"] is not None: + detail.append("text %r" % probe["text"]) + if probe["ready_s"] is not None: + detail.append("ready after %ss, %d stream chunks%s" % (probe["ready_s"], probe["stream_chunks"], + ", version %s" % probe["version"] if probe["version"] else "")) +if verdict == "pass": + if surface and surface["verdict"] != "pass": + verdict = surface["verdict"] + elif env.get("R_SERVED") and probe is None: + verdict = "fail" + detail.append("canary/probe.py wrote no result") + elif probe is not None and not probe["hard"]: + verdict = "fail" + else: + hard = True + exact = bool(probe["exact"]) if probe is not None else True +res = {"kind": env["E_KIND"] or "runtime", "pinned_ref": env["E_REF"], "verdict": verdict, "hard": hard, "exact": exact, + "cpu_flags": env["R_CPU"].split(), "detail": "; ".join(d for d in detail if d)} +with open(out, "w") as f: + json.dump(res, f, indent=1) + f.write("\n") +print("canary %s: %s (hard %s, exact %s): %s" % (os.path.basename(out)[:-5], verdict, hard, exact, res["detail"])) +sys.exit(0 if verdict == "pass" else 1) +PY + local rc=$? + [ -z "${E_INFORMATIONAL}" ] || rc=0 + exit "${rc}" +} + +# (2) CPU flags +R_CPU="" +cpuflags=" $(grep -m1 '^flags' "${CANARY_CPUINFO:-/proc/cpuinfo}" | cut -d: -f2) " +for f in avx2 avx512f amx_tile; do + [[ "${cpuflags}" == *" ${f} "* ]] && R_CPU+="${f} " +done +export R_CPU + +# The engine, from canary/engines.toml and the tree. +if ! cfg=$(python3 "${HERE}/plan.py" shell "${ID}" 2> "${WORK}/plan.err"); then + export E_KIND E_REF + finish fail "plan: $(tr '\n' ' ' < "${WORK}/plan.err")" +fi +eval "${cfg}" +export E_KIND E_REF +if [ -z "${surface_only}" ] && [ "${E_KIND}" != runtime ]; then finish fail "${ID} is a ${E_KIND} engine: canary/run.sh --surface ${ID}"; fi +cpu_model=$(sed -n 's/^model name[[:space:]]*: //p' "${CANARY_CPUINFO:-/proc/cpuinfo}" | head -1) +say "${E_KIND} ${E_REF}; cpu: ${cpu_model:-unknown}, $(grep -c '^processor' "${CANARY_CPUINFO:-/proc/cpuinfo}") CPUs, flags ${R_CPU:-none}" + +# (3) the model at its pinned revision +if [ -z "${surface_only}" ]; then + if [ "$(cat "${E_MODEL_DIR}/.complete" 2>/dev/null)" = "${E_REVISION}" ]; then + say "model ${E_MODEL}@${E_REVISION} from the cache" + else + hf=${HF_ENDPOINT:-https://huggingface.co} + mkdir -p "${E_MODEL_DIR}" + for f in "${E_FILES[@]}"; do + say "download ${E_MODEL}@${E_REVISION} ${f}" + curl -fsSL --retry 3 --retry-all-errors --retry-delay 5 -o "${E_MODEL_DIR}/${f}.partial" \ + "${hf}/${E_MODEL}/resolve/${E_REVISION}/${f}" 2>> "${LOG}" \ + || finish fail "model: ${hf}/${E_MODEL}/resolve/${E_REVISION}/${f} did not download" + mv -f "${E_MODEL_DIR}/${f}.partial" "${E_MODEL_DIR}/${f}" + done + echo "${E_REVISION}" > "${E_MODEL_DIR}/.complete" + fi +fi + +# (1, 4) size, free disk, pull +free_bytes() { + local root + if [ -n "${CANARY_FREE_BYTES:-}" ]; then echo "${CANARY_FREE_BYTES}"; return; fi + root=$(docker info -f '{{.DockerRootDir}}' 2>/dev/null) || root=/var/lib/docker + df -P -B1 "${root}" 2>/dev/null | awk 'NR == 2 {print $4}' +} +free_disk() { + say "freeing disk: the runner's preinstalled toolchains" + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /usr/local/.ghcup /opt/hostedtoolcache/CodeQL \ + /usr/local/share/boost /usr/share/swift /usr/local/share/powershell /opt/microsoft 2>> "${LOG}" + sudo docker image prune -af > /dev/null 2>&1 + return 0 +} +if bytes=$(python3 "${HERE}/plan.py" size "${E_REF}" 2>> "${LOG}"); then + need=$(python3 -c 'import sys; print(int(float(sys.argv[1]) * float(sys.argv[2])))' "${bytes}" "${E_SIZE_FACTOR}") + free=$(free_bytes) + if [ "${need}" -gt "${free:-0}" ] && [ "${CANARY_FREE_DISK:-0}" = 1 ]; then + free_disk + free=$(free_bytes) + fi + say "image ${bytes} bytes compressed, ~${need} on disk, ${free} free" + if [ "${need}" -gt "${free:-0}" ]; then + finish too-big "the image is ${bytes} bytes compressed, ~${need} bytes pulled (x${E_SIZE_FACTOR}), ${free} bytes free" + fi +else + say "the image's size is unreadable (see the log); pulling anyway" +fi +pulled="" +for attempt in 1 2 3; do + if docker pull "${E_REF}" > "${WORK}/pull.out" 2>&1; then pulled=1; break; fi + say "pull attempt ${attempt} failed: $(tail -1 "${WORK}/pull.out")" + cat "${WORK}/pull.out" >> "${LOG}" + [ "${attempt}" -eq 3 ] || sleep $(( ${CANARY_PULL_BACKOFF:-20} * attempt )) +done +[ -n "${pulled}" ] || finish fail "pull failed after 3 attempts: $(tail -1 "${WORK}/pull.out")" + +# (5) the surface check +if [ -n "${E_SURFACE}" ]; then + if ! python3 "${HERE}/plan.py" show "${ID}" > "${WORK}/engine.json" \ + || ! python3 "${HERE}/surface.py" check --engine "${WORK}/engine.json" --out "${WORK}/surface.json" >&2; then + finish fail "surface: canary/surface.py did not run" + fi +fi +[ -z "${surface_only}" ] || finish pass "" + +# (6) serve and probe: from here a pass needs the probe's result +export R_SERVED=1 +vols=() envs=() caps=() +if [ -n "${E_WRITABLE}" ]; then + # The engine writes next to the model (OVMS: graph.pbtxt): a copy, never the cache. + mkdir -p "${WORK}/models${E_MOUNT#"${E_WRITABLE}"}" + cp -r "${E_MODEL_DIR}/." "${WORK}/models${E_MOUNT#"${E_WRITABLE}"}/" + rm -f "${WORK}/models${E_MOUNT#"${E_WRITABLE}"}/.complete" + chmod -R a+rwX "${WORK}/models" + vols=(-v "${WORK}/models:${E_WRITABLE}") +else + vols=(-v "${E_MODEL_DIR}:${E_MOUNT}:ro") +fi +for e in ${E_ENV[@]+"${E_ENV[@]}"}; do envs+=(-e "${e}"); done +for c in ${E_CAP_ADD[@]+"${E_CAP_ADD[@]}"}; do caps+=(--cap-add "${c}"); done +[ -z "${E_SHM_SIZE}" ] || caps+=(--shm-size "${E_SHM_SIZE}") +if [ "${#E_PREPARE[@]}" -gt 0 ]; then + penv=() + for e in ${E_PREPARE_ENV[@]+"${E_PREPARE_ENV[@]}"}; do penv+=(-e "${e}"); done + say "prepare: ${E_PREPARE[*]}" + docker run --rm --name "${NAME}-prepare" "${vols[@]}" ${penv[@]+"${penv[@]}"} "${E_REF}" "${E_PREPARE[@]}" >> "${LOG}" 2>&1 \ + || finish fail "the prepare run (${E_PREPARE[*]}) failed: $(tail -3 "${LOG}" | tr '\n' ' ')" +fi +port=$(python3 -c 'import socket; s = socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])') +docker rm -f "${NAME}" > /dev/null 2>&1 +say "serve: ${E_ARGS[*]}" +docker run -d --name "${NAME}" -p "127.0.0.1:${port}:${E_PORT}" "${vols[@]}" ${envs[@]+"${envs[@]}"} \ + ${caps[@]+"${caps[@]}"} "${E_REF}" "${E_ARGS[@]}" > /dev/null 2>> "${LOG}" \ + || finish fail "docker run failed: $(tail -3 "${LOG}" | tr '\n' ' ')" +python3 "${HERE}/probe.py" --base "http://127.0.0.1:${port}" --api "${E_API}" --name "${E_NAME}" \ + --golden "${HERE}/../${E_GOLDEN}" --timeout "${CANARY_READY_TIMEOUT:-${E_READY_TIMEOUT}}" --container "${NAME}" \ + --version-path "${E_VERSION_PATH}" --version-key "${E_VERSION_KEY}" --version-regex "${E_VERSION_REGEX}" \ + --expect-version "${E_EXPECT_VERSION}" --out "${WORK}/probe.json" +docker logs "${NAME}" >> "${LOG}" 2>&1 +docker rm -f "${NAME}" > /dev/null 2>&1 + +# (7) an engine that cannot run on this ISA +ready=$(python3 -c 'import json, sys; d = json.load(open(sys.argv[1])); print("%s %s" % (d["ready"], d["exit_code"]))' "${WORK}/probe.json") +if [ "${ready%% *}" = False ] && [ -n "${E_ISA_SIGNAL}" ] && [[ " ${R_CPU}" != *" avx512f "* ]] \ + && { [ "${ready#* }" = 132 ] || grep -Eq -- "${E_ISA_SIGNAL}" "${LOG}"; }; then + finish isa-skip "this runner has no avx512f and the engine stopped on its ISA (exit ${ready#* }; /${E_ISA_SIGNAL}/ in its log)" +fi +finish pass "" diff --git a/canary/surface.py b/canary/surface.py new file mode 100755 index 00000000..c529bb9a --- /dev/null +++ b/canary/surface.py @@ -0,0 +1,495 @@ +#!/usr/bin/env python3 +"""The canary's surface check: every flag and env name k3d-gpu emits for an engine +(canary/flags.sh) exists in the pinned image, and the constants k3d-gpu derives from the +image (console/plan.py's vLLM versions, the NPU driver and firmware facts) still match it. +Runs with no device: nothing here needs a GPU, an NPU or a Gaudi. + + surface.py check --engine FILE --out FILE [--canary DIR] + host side: FILE is `canary/plan.py show `; runs the image with docker and + writes {verdict: pass|fail|unparseable, detail, method, version, missing} + surface.py inside --pkg NAME [--root DIR] + inside an image (docker run --entrypoint python3 -v canary:/canary:ro ...): finds the + installed package on sys.path without importing it and prints what its sources + define, as JSON {root, version, flags, envs}; exit 3 with {error} when unparseable + +Drivers (engines.toml surface.driver): + vllm `vllm serve --help=all` gives the flags when it runs on the CPU; otherwise + the static parse of vllm/engine/arg_utils.py (add_argument literals) and + every vllm/entrypoints/**/cli_args.py (add_argument literals and the + fields of its *Args dataclasses, which become --kebab-case flags), where + --no-X counts as present when --X is (BooleanOptionalAction). Env names + are always the keys of vllm/envs.py's environment_variables dict; the + version is the package's dist-info. Underscores and dashes are one flag + (vLLM's FlexibleArgumentParser). + llama-server llama-server --help + llama-convert convert_hf_to_gguf.py --help, and llama-quantize must answer --help + ovms ovms --help; with `deb`, that package's version (dpkg-query); with + `firmware`, the build date inside that firmware file + gst gst-inspect-1.0 per element: it must exist and have each property; + elements named in `device_only` (prefixes) register only on a device + sglang python3 -m sglang.launch_server --help, and the package version; a long + flag may be argparse's abbreviation of exactly one option (ABBREV_DRIVERS) +A help or parse that fails, or yields no flags, is unparseable: red, never skipped. + +Python 3.8+ standard library only (the `inside` mode runs on the image's Python). +""" + +import argparse +import ast +import glob +import json +import os +import re +import subprocess +import sys +import tempfile + +FLAG = re.compile(r"-{1,2}[A-Za-z][A-Za-z0-9_.-]*") +HELP_MIN = 3 # fewer flags than this in a help text is no help text + + +class Unparseable(Exception): + pass + + +# ---- help output --------------------------------------------------------------------- + +def help_flags(text): + """Flags an option listing defines: lines that start (after indentation) with an + option, as argparse, cxxopts and llama.cpp print them ("-a, --bb B", "--c, --no-c", + "-ngl, --gpu-layers, --n-gpu-layers N text"). Usage lines and prose are skipped.""" + flags = set() + for line in text.splitlines(): + s = line.strip() + # options sit at most 8 columns in; deeper lines continue a description + if not s.startswith("-") or s.startswith("---") or len(line) - len(line.lstrip()) > 8: + continue + # the option column ends where the description starts (two spaces after a word) + col = re.split(r"(?<=[^,\s])\s{2,}", s, maxsplit=1)[0] + for item in col.split(","): + word = item.strip().split(" ", 1)[0].split("=", 1)[0] + if not word: + continue + if not FLAG.fullmatch(word): + break + flags.add(word) + if len(flags) < HELP_MIN: + raise Unparseable("no option listing in the help output (%d flags)" % len(flags)) + return flags + + +def gst_properties(text): + """Property names of a gst-inspect-1.0 element page.""" + props, inside = set(), False + for line in text.splitlines(): + if re.match(r"^\S", line): + inside = line.startswith("Element Properties") + continue + m = re.match(r"^ ([a-z][a-z0-9-]*)\s*:", line) + if inside and m: + props.add(m.group(1)) + return props + + +# ---- static sources ------------------------------------------------------------------ + +def parse(path): + try: + with open(path, encoding="utf-8") as f: + return ast.parse(f.read(), path) + except FileNotFoundError: + raise Unparseable("%s is missing" % path) + except (SyntaxError, ValueError, UnicodeDecodeError) as e: + raise Unparseable("%s does not parse: %s" % (path, e)) + + +def add_argument_flags(tree): + out = set() + for n in ast.walk(tree): + if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute) and n.func.attr == "add_argument": + for a in n.args: + if isinstance(a, ast.Constant) and isinstance(a.value, str) and FLAG.fullmatch(a.value): + out.add(a.value) + return out + + +def args_class_fields(tree): + """--kebab-case of every annotated field of a class named *Args (vLLM registers each + field of FrontendArgs as a flag).""" + out = set() + for n in ast.walk(tree): + if isinstance(n, ast.ClassDef) and n.name.endswith("Args"): + for s in n.body: + if isinstance(s, ast.AnnAssign) and isinstance(s.target, ast.Name) and not s.target.id.startswith("_"): + out.add("--" + s.target.id.replace("_", "-")) + return out + + +def env_names(tree): + for n in tree.body: + target = n.target if isinstance(n, ast.AnnAssign) else (n.targets[0] if isinstance(n, ast.Assign) and len(n.targets) == 1 else None) + if isinstance(target, ast.Name) and target.id == "environment_variables" and isinstance(n.value, ast.Dict): + names = {k.value for k in n.value.keys if isinstance(k, ast.Constant) and isinstance(k.value, str)} + if names: + return names + raise Unparseable("envs.py has no environment_variables dict of env names") + + +def norm_dist(name): + return re.sub(r"[-_.]+", "_", name).lower() + + +def package_version(site, pkg): + for d in sorted(glob.glob(os.path.join(site, "*.dist-info"))): + meta = os.path.join(d, "METADATA") + if not os.path.exists(meta): + continue + with open(meta, encoding="utf-8", errors="replace") as f: + head = f.read(4096) + n = re.search(r"^Name: (.+)$", head, re.M) + v = re.search(r"^Version: (.+)$", head, re.M) + if n and v and norm_dist(n.group(1).strip()) == norm_dist(pkg): + return v.group(1).strip() + try: + import importlib.metadata as md # reads dist-info on sys.path; imports nothing of pkg + return md.version(pkg) + except Exception: + pass + for name in ("_version.py", "version.py"): + p = os.path.join(site, pkg, name) + if os.path.exists(p): + for n in ast.walk(parse(p)): + if isinstance(n, ast.Assign) and isinstance(n.value, ast.Constant) and isinstance(n.value.value, str) \ + and any(isinstance(t, ast.Name) and t.id in ("__version__", "version") for t in n.targets): + return n.value.value + return None + + +def static_package(site, pkg): + """What a package's sources define, read without importing it.""" + root = os.path.join(site, pkg) + if not os.path.isdir(root): + raise Unparseable("no package %s under %s" % (pkg, site)) + doc = {"root": root, "version": package_version(site, pkg), "flags": set(), "envs": set(), "files": []} + if pkg != "vllm": + return doc + engine = os.path.join(root, "engine", "arg_utils.py") + flags = add_argument_flags(parse(engine)) + if not flags: + raise Unparseable("%s has no add_argument flags" % engine) + clis = sorted(glob.glob(os.path.join(root, "entrypoints", "**", "cli_args.py"), recursive=True)) + if not clis: + raise Unparseable("no vllm/entrypoints/**/cli_args.py (the frontend flags) under %s" % root) + for p in clis: + t = parse(p) + flags |= add_argument_flags(t) | args_class_fields(t) + envs = env_names(parse(os.path.join(root, "envs.py"))) + doc.update(flags=flags, envs=envs, files=[engine] + clis + [os.path.join(root, "envs.py")]) + return doc + + +def find_site(pkg, paths): + """The directory holding the package: a sys.path entry, or for an editable install + whose finder maps the name, the parent of the location importlib finds. find_spec + on a top-level name runs the finders only; it does not execute the package.""" + for p in paths: + if p and os.path.isdir(os.path.join(p, pkg)) and (os.path.exists(os.path.join(p, pkg, "__init__.py")) + or os.path.isdir(os.path.join(p, pkg, "engine"))): + return p + try: + import importlib.util + spec = importlib.util.find_spec(pkg) + locs = list(spec.submodule_search_locations or []) if spec else [] + if locs and os.path.isdir(locs[0]): + return os.path.dirname(os.path.abspath(locs[0])) + except (ImportError, ValueError): + pass + raise Unparseable("package %s is not on sys.path (%s)" % (pkg, ":".join(x for x in paths if x))) + + +# ---- comparisons --------------------------------------------------------------------- + +def release(v): + """The X.Y[.Z] a version string starts with (v1.35.0, 0.32.0+cpu, 0.26.0b2, + 1.35.0.20260722-1~ubuntu24.04), or None. + + A setuptools-scm build of a tag with uncommitted changes is the tag's release. + Its default scheme (guess-next-dev, local node-and-date) names a tree at tag + X.Y.Z that has changes X.Y.(Z+1).dev0+g[.d]: dev0 is no commit + after the tag, and the last segment is bumped. intel/llm-scaler-vllm 1.5's + 0.26.1.dev0+g568afb3a1.d20260907.xpu is vLLM v0.26.0 (commit 568afb3a1) with + patches applied. A next version whose last segment is 0 has an unknown tag: None.""" + v = (v or "").strip() + m = re.match(r"v?(\d+\.\d+(?:\.\d+)?)", v) + if not m: + return None + scm = re.fullmatch(r"v?(\d+(?:\.\d+)+)\.dev0\+g[0-9a-f]{7,40}(?:\.[A-Za-z0-9]+)*", v) + if scm: + parts = scm.group(1).split(".") + if int(parts[-1]) == 0: + return None + parts[-1] = str(int(parts[-1]) - 1) + return ".".join(parts) + return m.group(1) + + +def fw_date(data): + """YYYYMMDD from an NPU firmware's version string ("20260630*MTL..." or + "Jun 30 2026*NPU40xx*", the scripts/k3d-gpu npu_fw_date rule).""" + m = re.search(rb"([0-9]{8}|[A-Z][a-z]{2} [ 0-9][0-9] [0-9]{4})\*(MTL|NPU)", data) + if not m: + return None + v = m.group(1).decode() + if v.isdigit(): + return v + mon, day, year = v.split() + i = "JanFebMarAprMayJunJulAugSepOctNovDec".find(mon) + return "%s%02d%02d" % (year, i // 3 + 1, int(day)) if i >= 0 else None + + +def norm_flag(name, driver): + name = name.split("=", 1)[0] + if driver == "vllm" and name.startswith("--"): + name = "--" + name[2:].replace("_", "-") + return name + + +# Drivers whose parser is argparse with allow_abbrev (its default), where k3d-gpu +# emits an abbreviation: bench/h54's sglang `--tp 1` is --tp-size (sglang's own docs +# write `--tp 2`). vLLM's parser allows them too, but the launcher spells every vLLM +# flag out, and an exact match catches a rename that a prefix match would hide. +ABBREV_DRIVERS = {"sglang"} + + +def abbreviation(name, have): + """The option string argparse expands a long option prefix to: the one option + string that begins with it (Python 3.12's argparse; two or more is "ambiguous + option"), else None. Single-dash names are matched exactly. `have` is what the help + lists, so an option hidden from help (argparse.SUPPRESS) cannot make it ambiguous.""" + if not name.startswith("--") or len(name) < 3: + return None + hits = [h for h in have if h.startswith(name)] + return hits[0] if len(hits) == 1 else None + + +def abbreviations(want, flags, driver): + """{name: option} for each wanted flag present only as an abbreviation.""" + if driver not in ABBREV_DRIVERS: + return {} + have = {norm_flag(f, driver) for f in flags} + out = {} + for w in want: + n = norm_flag(w["name"], driver) + if w["kind"] == "flag" and n not in have and abbreviation(n, have): + out[w["name"]] = abbreviation(n, have) + return out + + +def missing(want, flags, envs, driver, lenient_no=False): + have = {norm_flag(f, driver) for f in flags} + short = abbreviations(want, flags, driver) + out = [] + for w in want: + if w["kind"] == "env": + if w["name"] not in envs: + out.append(w) + continue + n = norm_flag(w["name"], driver) + if n in have or w["name"] in short or (lenient_no and n.startswith("--no-") and "--" + n[5:] in have): + continue + out.append(w) + return out + + +def derived_mismatch(derived, facts): + bad = [] + for d in derived: + fact = facts.get(d["of"]) + same = fact is not None and (fact == d["value"] if d["of"] == "firmware" else release(fact) == release(d["value"])) + if not same: + bad.append(dict(d, image=fact)) + return bad + + +# ---- host side ----------------------------------------------------------------------- + +def docker(args, timeout=600): + try: + p = subprocess.run(["docker", *args], capture_output=True, text=True, timeout=timeout) + return p.returncode, p.stdout + p.stderr + except subprocess.TimeoutExpired: + return 124, "timed out after %ds" % timeout + + +def run_image(image, args, entrypoint=None, extra=()): + opts = ["run", "--rm", "--network", "none", *extra] + if entrypoint: + opts += ["--entrypoint", entrypoint] + return docker(opts + [image, *args]) + + +def help_of(image, args, entrypoint=None): + rc, out = run_image(image, args, entrypoint) + if rc != 0: + raise Unparseable("%s%s exited %d: %s" % (entrypoint + " " if entrypoint else "", " ".join(args), rc, tail(out))) + return help_flags(out) + + +def tail(text, n=300): + text = " ".join(text.split()) + return text[-n:] + + +def inside(image, pkg, canary, python="python3"): + rc, out = run_image(image, ["/canary/surface.py", "inside", "--pkg", pkg], python, ("-v", "%s:/canary:ro" % canary)) + try: + doc = json.loads(out[out.index("{"):]) + except ValueError: + raise Unparseable("surface.py inside exited %d: %s" % (rc, tail(out))) + if rc != 0 or "error" in doc: + raise Unparseable(doc.get("error") or "surface.py inside exited %d" % rc) + return doc + + +def check(doc, canary): + image, sv = doc["ref"], doc.get("surface") or {} + driver = sv.get("driver") + want = doc.get("flags", []) + facts, notes, method = {}, [], "help" + flags, envs, lenient = set(), set(), False + try: + if driver == "vllm": + st = inside(image, "vllm", canary, sv.get("python", "python3")) + envs, facts["package"] = set(st["envs"]), st["version"] + try: + flags = help_of(image, ["serve", "--help=all"], "vllm") + except Unparseable as e: + flags, lenient, method = set(st["flags"]), True, "static" + notes.append("vllm serve --help=all did not run here (%s), flags from the sources" % e) + elif driver == "llama-server": + flags = help_of(image, ["--help"]) + elif driver == "llama-convert": + flags = help_of(image, ["/app/convert_hf_to_gguf.py", "--help"], "python3") + rc, out = run_image(image, ["--help"], "/app/llama-quantize") + if "usage:" not in out: + raise Unparseable("llama-quantize --help printed no usage (rc %d): %s" % (rc, tail(out))) + elif driver == "ovms": + flags = help_of(image, ["--help"]) + if sv.get("deb"): + rc, out = run_image(image, ["-W", "-f", "${Version}", sv["deb"]], "dpkg-query") + if rc != 0 or not out.strip(): + raise Unparseable("dpkg-query %s exited %d: %s" % (sv["deb"], rc, tail(out))) + facts["deb"] = out.strip().splitlines()[-1] + if sv.get("firmware"): + facts["firmware"] = firmware_date(image, sv["firmware"]) + elif driver == "gst": + method = "gst-inspect" + device_only = tuple(sv.get("device_only", [])) + for w in want: + if w["kind"] != "element": + continue + el = w["name"] + if el.startswith(device_only): + notes.append("%s registers only on a device: not checked" % el) + flags.add(el) + flags.update(x["name"] for x in want if x["name"].startswith(el + ".")) + continue + rc, out = run_image(image, [el], "gst-inspect-1.0") + if rc == 0: + flags.add(el) + flags.update("%s.%s" % (el, p) for p in gst_properties(out)) + elif driver == "sglang": + flags = help_of(image, ["-m", "sglang.launch_server", "--help"], "python3") + facts["package"] = inside(image, "sglang", canary)["version"] + else: + raise Unparseable("unknown surface driver %r" % driver) + version = probe_version(doc, facts) + except Unparseable as e: + return {"verdict": "unparseable", "detail": "surface: %s" % e, "method": method, "version": None, "missing": []} + lost = missing(want, flags, envs, driver, lenient_no=lenient) + notes += ["%s is argparse's abbreviation of %s" % kv for kv in sorted(abbreviations(want, flags, driver).items())] + bad = derived_mismatch(doc.get("derived", []), facts) + problems = [] + if lost: + problems.append("not in the image: " + ", ".join("%s (%s)" % (m["name"], m["source"]) for m in lost)) + for d in bad: + problems.append("%s says %s, the image's %s is %s" % (d["file"], d["value"], d["of"], d["image"])) + exp = doc.get("expected_version") + if exp and version and (release(version) or version) != (release(exp) or exp): + problems.append("the image reports version %s, its tag %s says %s" % (version, doc.get("tag"), exp)) + ok = "%d names checked (%s)%s" % (len(want), method, ", version %s" % version if version else "") + if notes: + ok += "; " + "; ".join(notes) + if problems: + return {"verdict": "fail", "detail": "surface: " + "; ".join(problems), "method": method, "version": version, + "missing": [m["name"] for m in lost]} + return {"verdict": "pass", "detail": "surface: " + ok, "method": method, "version": version, "missing": []} + + +def probe_version(doc, facts): + probe = doc.get("version_probe") or {} + if "package" in probe: + v = facts.get("package") + if not v: + raise Unparseable("no version for package %s" % probe["package"]) + return v + if "args" in probe: + rc, out = run_image(doc["ref"], probe["args"], probe.get("entrypoint")) + m = re.search(probe["regex"], out) + if not m: + raise Unparseable("%s printed no version matching %r (rc %d): %s" % (" ".join(probe["args"]), probe["regex"], rc, tail(out))) + return m.group("v") + return None # a server endpoint: canary/probe.py reads it + + +def firmware_date(image, path): + name = "canary-fw-%d" % os.getpid() + rc, out = docker(["create", "--name", name, image]) + if rc != 0: + raise Unparseable("docker create: %s" % tail(out)) + try: + with tempfile.TemporaryDirectory() as d: + rc, out = docker(["cp", "-L", "%s:%s" % (name, path), os.path.join(d, "fw")]) + if rc != 0: + raise Unparseable("no %s in the image: %s" % (path, tail(out))) + with open(os.path.join(d, "fw"), "rb") as f: + date = fw_date(f.read()) + finally: + docker(["rm", "-f", name]) + if not date: + raise Unparseable("no build date in %s" % path) + return date + + +def main(): + p = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + sub = p.add_subparsers(dest="cmd", required=True) + c = sub.add_parser("check") + c.add_argument("--engine", required=True) + c.add_argument("--out", required=True) + c.add_argument("--canary", default=os.path.dirname(os.path.abspath(__file__))) + i = sub.add_parser("inside") + i.add_argument("--pkg", required=True) + i.add_argument("--root") + a = p.parse_args() + if a.cmd == "inside": + try: + site = a.root or find_site(a.pkg, sys.path) + doc = static_package(site, a.pkg) + except Unparseable as e: + print(json.dumps({"error": str(e)})) + return 3 + print(json.dumps({"root": doc["root"], "version": doc["version"], "flags": sorted(doc["flags"]), + "envs": sorted(doc["envs"])})) + return 0 + with open(a.engine, encoding="utf-8") as f: + doc = json.load(f) + res = check(doc, os.path.abspath(a.canary)) + with open(a.out, "w", encoding="utf-8") as f: + json.dump(res, f, indent=1) + print(res["detail"]) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/canary_node_test.sh b/tests/canary_node_test.sh new file mode 100755 index 00000000..0517da65 --- /dev/null +++ b/tests/canary_node_test.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +# canary/node.sh against a docker, k3d and curl shim: the cluster's pods come from +# scenario files, one per `kubectl get pods` poll (the last repeats), so each case reads +# the verdict node.sh writes for a cluster whose pods behave a known way. No image is +# built, no cluster is started, nothing is downloaded. +# +# Usage: tests/canary_node_test.sh + +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +T=$(mktemp -d "${TMPDIR:-/tmp}/canary_node_test.XXXXXX") +trap 'rm -rf "$T"' EXIT +PASS=0 FAILS=0 +ok() { PASS=$((PASS + 1)); echo "ok $1"; } +fail() { FAILS=$((FAILS + 1)); echo "FAIL $1"; } + +# A tree node.sh reads: the canary, the Dockerfile and the manifests it bakes, and an +# install.sh whose k3d sha256 is the shim's. +R=${T}/root +mkdir -p "${R}/packaging" "${T}/bin" "${T}/state" +cp -r "${ROOT}/canary" "${ROOT}/share" "${ROOT}/Dockerfile" "${R}/" +printf '#!/bin/sh\necho "k3d $*" >> "${STUB_STATE}/calls"\ncase "$1 $2" in "cluster create") exit "${STUB_CREATE_RC:-0}" ;; esac\nexit 0\n' \ + > "${T}/k3d.src" +printf 'K3D_VERSION=v5.9.0\nK3D_SHA256_AMD64=%s\n' "$(sha256sum "${T}/k3d.src" | cut -d' ' -f1)" > "${R}/packaging/install.sh" + +# What the node holds when everything applied: the baked manifests and their +# DaemonSets, read here with a YAML parser (node.sh reads them its own way). +grep -oE '/var/lib/rancher/k3s/server/manifests/[^ ]+$' "${ROOT}/Dockerfile" | xargs -n1 basename > "${T}/manifests" +sed -nE 's#^COPY (share/[^ ]+) /var/lib/rancher/k3s/server/manifests/.*#\1#p' "${ROOT}/Dockerfile" \ + | (cd "${ROOT}" && xargs python3 -c ' +import sys, yaml +for p in sys.argv[1:]: + for d in yaml.safe_load_all(open(p)): + if d and d.get("kind") == "DaemonSet": + print("%s/%s" % (d["metadata"].get("namespace", "default"), d["metadata"]["name"])) +') | sort -u > "${T}/daemonsets" +[ -s "${T}/daemonsets" ] && grep -qx 'node-feature-discovery/nfd-worker' "${T}/daemonsets" \ + && ok "fixture: the baked manifests define $(wc -l < "${T}/daemonsets") DaemonSets, NFD's among them" \ + || fail "fixture: no DaemonSets read from the baked manifests" + +cat > "${T}/bin/curl" <<'SHIM' +#!/usr/bin/env bash +out="" +while [ $# -gt 0 ]; do case "$1" in -o) out=$2; shift 2 ;; *) shift ;; esac; done +cp "${STUB_K3D_SRC:?}" "${out:?}" +SHIM +cat > "${T}/bin/docker" <<'SHIM' +#!/usr/bin/env bash +S=${STUB_STATE:?} +echo "docker $*" >> "${S}/calls" +case "$1" in + buildx) exit "${STUB_BUILD_RC:-0}" ;; + exec) shift 2 ;; + *) exit 0 ;; +esac +case "$*" in + "ls /var/lib/rancher/k3s/server/manifests") cat "${STUB_MANIFESTS:?}" ;; + "kubectl get nodes -o jsonpath="*) echo True ;; + "kubectl get ds -A -o jsonpath="*) grep -vxF "${STUB_DS_DROP:-none}" "${STUB_DAEMONSETS:?}" ;; + "kubectl get pods -A -o jsonpath="*) + n=$(( $(cat "${S}/polls" 2>/dev/null || echo 0) + 1 )) + echo "${n}" > "${S}/polls" + f=$(ls "${STUB_PODS:?}" | sort -n | awk -v n="${n}" '$1 <= n' | tail -1) + cat "${STUB_PODS}/${f}" ;; + "kubectl get pods"*) echo "kubectl get pods: node.sh asks for all namespaces with -A" >&2; exit 1 ;; +esac +exit 0 +SHIM +chmod +x "${T}/bin/curl" "${T}/bin/docker" +export PATH="${T}/bin:${PATH}" STUB_STATE="${T}/state" STUB_K3D_SRC="${T}/k3d.src" +export STUB_MANIFESTS="${T}/manifests" STUB_DAEMONSETS="${T}/daemonsets" +export CANARY_NODE_TIMEOUT=4 CANARY_NODE_SETTLE=2 CANARY_NODE_POLL=0.2 + +UP=$'kube-system/coredns-1 Running coredns=true:0,\nkube-system/helm-install-x Succeeded helm=false:0,\nkube-system/intel-gpu-plugin-a Running intel-gpu-plugin=true:0,\nnode-feature-discovery/nfd-master-1 Running nfd-master=true:0,\nnode-feature-discovery/nfd-worker-a Running nfd-worker=true:0,' + +# node_case