From bf68b697603733380ea918f7f652ffa28b4cbd3e Mon Sep 17 00:00:00 2001 From: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> Date: Fri, 11 Sep 2026 05:06:26 +0000 Subject: [PATCH 01/11] [None][infra] Add PrimTS vendor promotion maintainer tool Automate promotion of immutable reviewed FlashInfer revisions and creation of lock-only follow-up PRs. Preserve upstream provenance in signed-off commits and squash messages, validate canonical destinations, and resume interrupted operations safely. Add CPU-only lifecycle coverage and document the developer, promotion, and periodic-refresh workflow with a Mermaid diagram. Signed-off-by: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> --- 3rdparty/vendor-sources.md | 198 ++++ AGENTS.md | 12 + scripts/maintain_prims_ts.py | 965 ++++++++++++++++ .../unittest/others/test_maintain_prims_ts.py | 1002 +++++++++++++++++ 4 files changed, 2177 insertions(+) create mode 100644 scripts/maintain_prims_ts.py create mode 100644 tests/unittest/others/test_maintain_prims_ts.py diff --git a/3rdparty/vendor-sources.md b/3rdparty/vendor-sources.md index c37cbb1de2d8..a4ebfe140ac2 100644 --- a/3rdparty/vendor-sources.md +++ b/3rdparty/vendor-sources.md @@ -289,6 +289,204 @@ local repository, and URL or commit changes require vendor CODEOWNER review. Never put credentials in a lock URL. Run checks that use internal credentials only in a trusted environment, not with pull-request-controlled scripts. +## Promote a reviewed PrimTS revision + +`scripts/maintain_prims_ts.py promote` automates the maintainer follow-up after +a TRT-LLM PR has merged a PrimTS code update from a temporary FlashInfer branch. +It does not import new kernel code or implement periodic upstream refreshes. +Use a trusted checkout of this tool; do not execute a PR-supplied copy with +maintainer credentials. + +### Maintenance workflow + +The source-update PR reviews code changes; the promotion PR only restores the +canonical lock location. The diagram shows publication with `--auto-merge`; +the CLI remains read-only unless `--publish` is supplied. + +```mermaid +flowchart TD + canonical["Canonical FlashInfer dev branch"] + temporary["Developer temporary branch
PrimTS code changes"] + source_pr["TRT-LLM source-update PR
vendor code and pin temporary SHA"] + pairing["Paired FlashInfer PR
or explicit unpaired reason"] + verify["promote: verify merged PR, immutable SHA,
canonical target and fast-forward range"] + + canonical -->|fork| temporary + temporary --> source_pr + source_pr -->|merged| verify + pairing --> verify + + subgraph publish["Maintainer: --publish --auto-merge"] + prepare["Verify source materialization
commit lock-only change with provenance and DCO"] + advance["Fast-forward canonical branch
to the reviewed source SHA"] + followup["Create or reuse lock-only TRT-LLM PR
same source SHA, patch, digests and files"] + merge_request["Post /bot skip for no code change
enable squash auto-merge with original message"] + prepare --> advance --> followup --> merge_request + end + + verify -->|publish requested| prepare + merge_request -->|required checks and reviews| merged["Follow-up merged
provenance preserved in squash commit"] + merged --> rebase["Other developers rebase their temporary branches
onto the promoted canonical revision"] + rebase --> temporary + merged -.-> refresh["Periodic refresh: manual / future tooling
new upstream main plus outstanding changes
new dated canonical branch"] + refresh -.-> refresh_pr["TRT-LLM refresh PR merged
all developers rebase to the dated branch"] + refresh_pr -.-> canonical +``` + +Solid arrows show the development/promotion cycle; dotted arrows show periodic +refresh, which this command does not implement. A paired upstream PR can merge +before or after the TRT-LLM source-update PR. Recording its URL and observed +commits preserves provenance; it is not permission to drop a change merely +because the upstream PR later merges. Unpaired changes explicitly retain their +reason and `refresh_policy: retain`. + +After a promotion follow-up merges, rebase both the pending TRT-LLM PR onto +latest main and its temporary FlashInfer branch onto the promoted canonical +revision before refreshing its vendor pin. For a periodic refresh, the +maintainer identifies already-integrated changes, carries forward outstanding +changes on newer FlashInfer main, and publishes a dated canonical branch and a +separate TRT-LLM refresh PR. That refresh may change vendor code and needs its +own validation; the promotion's no-code-change CI request does not apply. + +If validation fails or another maintainer advances the relevant pins, stop and +inspect the reported state. Retrying an interrupted promotion reuses verified +worktrees, commits, PRs, and CI comments; it does not overwrite unrelated work. + +### Plan and publish a promotion + +Start with a dry-run (GitHub reads only): + +```bash +python scripts/maintain_prims_ts.py promote \ + --trtllm-pr "$MERGED_TRTLLM_PR" \ + --canonical-repo yuxianq/flashinfer \ + --upstream-pr https://github.com/flashinfer-ai/flashinfer/pull/4829 +``` + +Normally an upstream PR is required and may be open, draft, or merged; it must +target FlashInfer `main`. Always specify the maintainer's canonical FlashInfer +fork with `--canonical-repo OWNER/REPO`; the tool never infers the destination +fork from a previous lock that might still point to a developer's fork. +`--canonical-branch` defaults to `trtllm-prims-ts-dev`; specify it explicitly +for a dated `trtllm-prims-ts-dev-YYYYMMDD` branch. Both must match the lock in +the parent of the TRT-LLM merge commit. The tool derives the old canonical +commit from that parent and the reviewed source SHA from the merge commit's +lock. It never promotes the temporary branch's current tip. Source changes +must form a linear, fast-forward range. Finish the preceding promotion before +merging another vendor update. A developer fork may use the same branch name +as the canonical fork: the repository URL is also part of its identity. + +For an existing change with no paired upstream PR, explicitly use +`--unpaired-reason` instead of `--upstream-pr`: + +```bash +python scripts/maintain_prims_ts.py promote \ + --trtllm-pr 18808 \ + --canonical-repo yuxianq/flashinfer \ + --unpaired-reason "No paired FlashInfer PR for the CUTLASS DSL 4.8 API migration." +``` + +This records the source commits with `upstream_pr: null`, the reason, and +`refresh_policy: retain` in the commit message. A future refresh must retain +these changes until an upstream pairing or equivalence is explicitly +established; missing PR metadata must never be interpreted as permission to +drop them. This option cannot be combined with PR mappings. It does not change +the fast-forward, immutable-pin, or lock-only requirements. + +One upstream PR assigns the whole newly promoted range to that PR. For multiple +PRs, repeat `--upstream-pr` and provide `--map-upstream COMMIT=PR` for each source +commit (full SHA or unambiguous prefix of at least seven characters). A terminal +invocation prompts for missing assignments; noninteractive use fails instead +of guessing. This mapping is a maintainer assertion of provenance, not proof +that a later upstream revision preserves the change's behavior. + +To publish the promotion and enable squash auto-merge for its follow-up PR: + +```bash +python scripts/maintain_prims_ts.py promote \ + --trtllm-pr "$MERGED_TRTLLM_PR" \ + --canonical-repo yuxianq/flashinfer \ + --upstream-pr https://github.com/flashinfer-ai/flashinfer/pull/4829 \ + --flashinfer-repo /path/to/flashinfer \ + --publish --auto-merge --wait +``` + +`--flashinfer-repo` is optional. If supplied, it must contain the previous and +reviewed immutable source commits; neither its checked-out branch nor dirty +files are used. Otherwise the tool fetches the reviewed source into a temporary +repository. `--repo` selects the local TRT-LLM repository. The personal TRT-LLM +fork defaults to its `fork` remote; use `--fork OWNER/REPO` to override it. +`GH_CONFIG_DIR` is respected, defaulting to `~/.config/gh`. + +Publishing performs the following checks and actions: + +1. Verify that current TRT-LLM main still contains the reviewed vendor entry and + the canonical branch has not been superseded. +2. Create a separate follow-up worktree, verify source materialization, and use + `vendor_sources.py pin` to change only the PrimTS lock URL/branch. The commit, + compatibility patch, digest, selected source, and destination stay unchanged. +3. Make a signed-off commit with the maintainer's Git identity. Its versioned + `PRIMTS PROMOTION V1` JSON block records both source SHAs, upstream baseline, + originating TRT-LLM PR/merge, and upstream PR/commit mappings and snapshots. +4. Fast-forward the canonical FlashInfer branch with a non-force push, push the + lock-only commit to the TRT-LLM personal fork, and create the follow-up PR. +5. Reverify the open PR's lock-only commit and post + `/bot skip --comment "skip CI since no code change"` to request the bot's + no-code-change CI path. PR creation can also start automatic GitHub checks; + this command does not disable those checks or waive required owner reviews. +6. With `--auto-merge`, supply the signed-off commit's title/body as the final squash + message and verify GitHub retained them. Required checks and reviews are not + bypassed. If already eligible, the PR may merge immediately. Merge queues are + currently rejected because this workflow requires a custom squash message. + +The generated PR checklist contains only assertions the tool has verified, not +an unchecked placeholder for future approvals or checks. Those remain enforced +by GitHub. The tool does not delete branches or rewrite history. Without +`--auto-merge`, the PR description includes the required squash message for +manual merging. Preserve that complete message: it is the durable provenance +record, with no extra tracking file. Initial history predating these records +will still need classification when a future refresh tool is introduced. + +`--wait` verifies the final merged commit, polling for up to `--timeout` seconds +(default 3600). Without it, the tool verifies the saved auto-merge request and +returns; there is no background monitor. If GitHub merges immediately, the tool +verifies and prints the final squash SHA. Do not change the follow-up branch +after requesting the CI skip or enabling auto-merge. A head check at enablement +does not permanently freeze the branch. + +Publication spans two repositories and is not atomic. Completed pushes and PRs +are preserved if a later step fails. Repeating the same invocation recognizes +the deterministic follow-up branch/PR, verifies its lock-only change and +signed-off metadata, and resumes without creating duplicate PRs. Before posting +the CI command, retries search all PR comments for the identical command from +the authenticated user, including when a previous posting request timed out +after succeeding. A posting failure stops before enabling auto-merge. Serialize +maintainer invocations; comment deduplication is not a cross-process lock. + +Interrupted worktree preparation can resume if the branch and repository match, +with either an unchanged base, the expected lock-only edit (staged or unstaged), +or a verified promotion commit. Unexpected tracked, staged, or untracked edits +are preserved and require inspection instead of automatic overwrite. Existing +upstream snapshots are preserved even if their PR heads have since advanced; +different mappings on an unfinished promotion are rejected. After a promotion +merges, reruns verify its historical squash commit and recorded provenance and +return without remote writes, even if current main or canonical branches have +advanced or the original temporary fork disappeared. The completed record is +authoritative; rerunning does not revise its provenance. + +`--worktree PATH` selects the follow-up worktree; by default it is a sibling of +`--repo` named `prims-ts-promote--`. Git checkout skips LFS +asset downloads for this lock-only worktree without changing repository-wide +configuration. Fetch, checkout, commit, and push progress is streamed. + +Hermetic CPU-only validation, without loading TRT-LLM's GPU test configuration: + +```bash +PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 pytest --noconftest -c /dev/null \ + -p no:cacheprovider -o 'markers=cpu_only: CPU-only test' \ + tests/unittest/others/test_maintain_prims_ts.py +``` + ## License and attribution The vendor lock is a reproducibility record, not a license manifest. Before diff --git a/AGENTS.md b/AGENTS.md index bb41fdcbf983..cbfe22c8ab96 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -193,6 +193,18 @@ The bot-to-bot semantic review workflow is disabled. See [.github/semantic-review.md](.github/semantic-review.md) for its current status and historical implementation details. +### PrimTS vendor promotion + +Use `scripts/maintain_prims_ts.py promote` for the lock-only maintainer follow-up +after a PrimTS source-update PR merges. It defaults to dry-run; publication and +auto-merge require explicit flags. For maintainer promotions, publish with +`--publish --auto-merge`. Specify the maintainer fork explicitly with +`--canonical-repo OWNER/REPO`. Publication posts +`/bot skip --comment "skip CI since no code change"` after verifying the lock-only +PR, before enabling optional auto-merge. See `3rdparty/vendor-sources.md`. +Preserve its structured promotion metadata and DCO sign-off in the final squash +message. + ### Triggering CI CI is triggered by posting comments on the PR. Basic commands: diff --git a/scripts/maintain_prims_ts.py b/scripts/maintain_prims_ts.py new file mode 100644 index 000000000000..16d3fbb4d5e1 --- /dev/null +++ b/scripts/maintain_prims_ts.py @@ -0,0 +1,965 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Promote reviewed PrimTS source revisions; dry-run unless --publish is supplied. + +Only Git, gh, and PyYAML are required. GitHub reads and temporary source fetches +are allowed in dry-run; remote writes and consumer worktrees require --publish. +Provenance lives in signed-off commits, not in a separate tracking file. +""" + +from __future__ import annotations + +import argparse +import base64 +import contextlib +import json +import os +import re +import subprocess +import sys +import tempfile +import time +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import urlencode + +import yaml + +if __package__: + from . import vendor_sources as vendor +else: + import vendor_sources as vendor + +_TRTLLM = "NVIDIA/TensorRT-LLM" +_FLASHINFER = "flashinfer-ai/flashinfer" +_VENDOR = "flashinfer-prims-ts" +_LOCK = "3rdparty/vendor_sources.lock.yaml" +_BEGIN = "----- BEGIN PRIMTS PROMOTION V1 -----" +_END = "----- END PRIMTS PROMOTION V1 -----" +_SHA = re.compile(r"[0-9a-f]{40}\Z") +_CANONICAL = re.compile(r"trtllm-prims-ts-dev(?:-[0-9]{8})?\Z") +_CI_SKIP_COMMAND = '/bot skip --comment "skip CI since no code change"' + + +def _run( + command: list[str], + *, + cwd: Path | None = None, + text: str | None = None, + timeout: int = 300, + stream: bool = False, +) -> str: + # Do not use vendor_sources._run_git here: its synthetic identity is for + # patch generation. Real promotion commits must use the maintainer's identity. + env = os.environ.copy() + for key in vendor._GIT_LOCAL_ENVIRONMENT_VARIABLES: + env.pop(key, None) + env.setdefault("GH_CONFIG_DIR", str(Path.home() / ".config/gh")) + env.update(GIT_TERMINAL_PROMPT="0", GH_PROMPT_DISABLED="1", GIT_NO_REPLACE_OBJECTS="1") + if command[0] == "git": + # Lock-only worktrees need source text, not unrelated LFS assets. + env["GIT_LFS_SKIP_SMUDGE"] = "1" + result = subprocess.run( + command, + cwd=cwd, + env=env, + input=text, + text=True, + capture_output=not stream, + timeout=timeout, + ) + if result.returncode: + detail = (result.stderr or result.stdout or "See streamed output above.").strip() + raise RuntimeError(f"{command[0]} failed: {detail}") + return (result.stdout or "").strip() + + +def _git(repo: Path, *args: str, text: str | None = None) -> str: + return _run( + ["git", "-C", str(repo), *args], + text=text, + timeout=1800 if args[:2] == ("worktree", "add") else 300, + stream=args[:2] == ("worktree", "add") or args[0] in ("fetch", "commit", "push"), + ) + + +class GitHub: + """Small gh-backed client; respects GH_CONFIG_DIR without handling tokens.""" + + def api( + self, endpoint: str, *, method: str = "GET", payload: dict | None = None + ) -> dict | list: + command = ["gh", "api", "--hostname", "github.com", "--method", method, endpoint] + if payload is not None: + command += ["--input", "-"] + result = json.loads(_run(command, text=None if payload is None else json.dumps(payload))) + if not isinstance(result, (dict, list)): + raise ValueError(f"Unexpected GitHub response for {endpoint}") + return result + + def get(self, endpoint: str) -> dict: + result = self.api(endpoint) + if not isinstance(result, dict): + raise ValueError(f"Expected an object from {endpoint}") + return result + + def optional(self, endpoint: str) -> dict | None: + try: + return self.get(endpoint) + except RuntimeError as error: + if "HTTP 404" not in str(error): + raise + return None + + def compare(self, repo: str, base: str, head: str, *, all_commits: bool = False) -> dict: + first = self.get(f"repos/{repo}/compare/{base}...{head}?per_page=100&page=1") + if all_commits: + count = int(first["total_commits"]) + for page in range(2, (count + 99) // 100 + 1): + more = self.get(f"repos/{repo}/compare/{base}...{head}?per_page=100&page={page}") + first["commits"].extend(more["commits"]) + if len(first["commits"]) != count: + raise ValueError("Incomplete GitHub comparison; refusing partial provenance.") + return first + + +def _sha(value: str) -> str: + if not isinstance(value, str) or not _SHA.fullmatch(value): + raise ValueError(f"Expected a full immutable commit SHA, got {value!r}.") + return value + + +def _repo_name(url: str) -> str: + match = re.fullmatch( + r"(?:https://github\.com/|git@github\.com:)([\w.-]+/[\w.-]+?)(?:\.git)?/?", url + ) + if not match: + raise ValueError(f"Expected a credential-free GitHub repository URL, got {url!r}.") + return match[1] + + +def _pr_number(value: str, repo: str) -> int: + for prefix in (f"https://github.com/{repo}/pull/", f"{repo}#"): + if value.startswith(prefix): + value = value[len(prefix) :] + break + if not re.fullmatch(r"[1-9][0-9]*", value): + raise ValueError(f"Expected a PR number or PR URL in {repo}: {value!r}.") + return int(value) + + +def _lock_document_at(gh: GitHub, repo: str, commit: str) -> dict: + response = gh.get(f"repos/{repo}/contents/{_LOCK}?ref={_sha(commit)}") + if response.get("encoding") != "base64": + raise ValueError("GitHub did not return the complete vendor lock.") + data = yaml.load(base64.b64decode(response["content"]), Loader=vendor._UniqueKeyLoader) + if not isinstance(data, dict) or data.get("schema_version") != 1: + raise ValueError("Unsupported vendor lock schema.") + return data + + +def _lock_at(gh: GitHub, repo: str, commit: str) -> vendor.Vendor: + data = _lock_document_at(gh, repo, commit) + return vendor._validate_vendor(_VENDOR, data["vendors"][_VENDOR]) + + +def _main_sha(gh: GitHub) -> str: + return _sha(gh.get(f"repos/{_TRTLLM}/git/ref/heads/main")["object"]["sha"]) + + +def _check_fork(gh: GitHub, repo: str, upstream: str) -> None: + info = gh.get(f"repos/{repo}") + if ( + repo.lower() == upstream.lower() + or info.get("source", {}).get("full_name", "").lower() != upstream.lower() + ): + raise ValueError(f"{repo} must be a personal fork of {upstream}, not upstream itself.") + + +def _record_from_message(message: str) -> dict: + if message.count(_BEGIN) != 1 or message.count(_END) != 1: + raise ValueError("Missing or ambiguous PrimTS promotion metadata in commit message.") + record = json.loads(message.split(_BEGIN, 1)[1].split(_END, 1)[0]) + if not isinstance(record, dict) or record.get("version") != 1: + raise ValueError("Unsupported PrimTS promotion metadata version.") + return record + + +def _record_text(record: dict) -> str: + return f"{_BEGIN}\n{json.dumps(record, indent=2, sort_keys=True)}\n{_END}" + + +def _map_commits(commits: list[dict], prs: dict[int, dict], mappings: list[str]) -> dict[str, int]: + assignments: dict[str, int] = {} + for mapping in mappings: + prefix, separator, reference = mapping.partition("=") + matches = [item["sha"] for item in commits if item["sha"].startswith(prefix)] + if not separator or not re.fullmatch(r"[0-9a-f]{7,40}", prefix) or len(matches) != 1: + raise ValueError(f"Ambiguous or invalid --map-upstream {mapping!r}.") + number = _pr_number(reference, _FLASHINFER) + if number not in prs or matches[0] in assignments: + raise ValueError(f"Duplicate mapping or PR missing from --upstream-pr: {mapping!r}.") + assignments[matches[0]] = number + for item in commits: + sha = item["sha"] + if sha in assignments: + continue + if len(prs) == 1: + assignments[sha] = next(iter(prs)) + elif sys.stdin.isatty(): + title = item["commit"]["message"].splitlines()[0] + answer = input(f"Upstream PR for {sha[:12]} ({title}), choose {sorted(prs)}: ") + number = _pr_number(answer, _FLASHINFER) + if number not in prs: + raise ValueError("Answer must identify a supplied upstream PR.") + assignments[sha] = number + else: + raise ValueError(f"Provide --map-upstream {sha}=PR for every unmapped commit.") + if set(assignments.values()) != set(prs): + raise ValueError("Every supplied upstream PR must map to at least one new source commit.") + return assignments + + +@dataclass +class Promotion: + """An immutable source revision and the expected consumer/canonical states.""" + + number: int + main_sha: str + previous: vendor.Vendor + reviewed: vendor.Vendor + canonical_repo: str + fork: str + record: dict + completed_pr: dict | None = None + + @property + def branch(self) -> str: + return f"chore/prims-ts-promote-{self.number}-{self.reviewed.commit[:12]}" + + @property + def title(self) -> str: + return f"[None][chore] promote PrimTS source from TRT-LLM #{self.number}" + + @property + def promoted(self) -> dict: + result = self.reviewed.to_mapping() + result.update(url=self.previous.url, branch=self.previous.branch) + return result + + +def _check_current(gh: GitHub, plan: Promotion) -> None: + current = _lock_at(gh, _TRTLLM, _main_sha(gh)).to_mapping() + if current not in (plan.reviewed.to_mapping(), plan.promoted): + raise ValueError("TRT-LLM main's vendor lock has been superseded; do not promote this PR.") + ref = gh.get(f"repos/{plan.canonical_repo}/git/ref/heads/{plan.previous.branch}") + if ref["object"]["sha"] not in (plan.previous.commit, plan.reviewed.commit): + raise ValueError( + "Canonical branch moved unexpectedly; coordinate with the other maintainer." + ) + + +def _make_plan(args: argparse.Namespace, gh: GitHub) -> Promotion: + if args.unpaired_reason is not None: + if not args.unpaired_reason.strip() or args.upstream_pr or args.map_upstream: + raise ValueError( + "--unpaired-reason requires a nonempty reason and cannot be combined " + "with --upstream-pr or --map-upstream." + ) + elif not args.upstream_pr: + raise ValueError("Provide --upstream-pr or an explicit --unpaired-reason.") + number = _pr_number(args.trtllm_pr, _TRTLLM) + pr = gh.get(f"repos/{_TRTLLM}/pulls/{number}") + if not pr.get("merged") or pr["base"]["ref"] != "main": + raise ValueError("--trtllm-pr must identify a PR already merged into TRT-LLM main.") + merge_sha = _sha(pr["merge_commit_sha"]) + merge = gh.get(f"repos/{_TRTLLM}/commits/{merge_sha}") + previous = _lock_at(gh, _TRTLLM, merge["parents"][0]["sha"]) + reviewed = _lock_at(gh, _TRTLLM, merge_sha) + canonical_repo = _repo_name(previous.url) + source_repo = _repo_name(reviewed.url) + expected_repo = _repo_name(f"https://github.com/{args.canonical_repo}") + if not _CANONICAL.fullmatch(args.canonical_branch): + raise ValueError("--canonical-branch must name a supported canonical branch.") + if canonical_repo.lower() != expected_repo.lower() or previous.branch != args.canonical_branch: + raise ValueError( + "Previous lock does not match the explicit canonical repository/branch; " + "finish the preceding promotion first." + ) + if ( + previous.commit == reviewed.commit + or not reviewed.branch + or (canonical_repo.lower() == source_repo.lower() and _CANONICAL.fullmatch(reviewed.branch)) + ): + raise ValueError("The PR must introduce a new source SHA on a temporary branch.") + fork = args.fork or _repo_name(_git(args.repo, "remote", "get-url", "fork")) + record = { + "version": 1, + "kind": "promotion", + "vendor": _VENDOR, + "trtllm_pr": f"https://github.com/{_TRTLLM}/pull/{number}", + "trtllm_merge": merge_sha, + "canonical_url": previous.url, + "canonical_branch": previous.branch, + "previous": previous.commit, + "promoted": reviewed.commit, + "source_url": reviewed.url, + "source_branch": reviewed.branch, + } + plan = Promotion(number, _main_sha(gh), previous, reviewed, canonical_repo, fork, record) + # A completed operation is historical: later promotions, deleted temporary + # forks, or changed upstream PR state must not make it run again or fail. + existing = _find_followup(gh, plan) + if existing: + existing = gh.get(f"repos/{_TRTLLM}/pulls/{existing['number']}") + if existing.get("merged"): + _verify_remote_commit(gh, plan, _TRTLLM, existing["merge_commit_sha"]) + plan.completed_pr = existing + return plan + _check_fork(gh, canonical_repo, _FLASHINFER) + _check_fork(gh, source_repo, _FLASHINFER) + _check_fork(gh, fork, _TRTLLM) + comparison = gh.compare(canonical_repo, previous.commit, reviewed.commit, all_commits=True) + if comparison["merge_base_commit"]["sha"] != previous.commit: + raise ValueError( + "Reviewed source is not a fast-forward of the previous canonical revision." + ) + commits = comparison["commits"] + if not commits or any(len(item["parents"]) != 1 for item in commits): + raise ValueError( + "Promote requires a nonempty, linear source commit range; rebase merges first." + ) + prs = {} + for reference in args.upstream_pr: + upstream_number = _pr_number(reference, _FLASHINFER) + if upstream_number in prs: + raise ValueError("Duplicate --upstream-pr.") + upstream = gh.get(f"repos/{_FLASHINFER}/pulls/{upstream_number}") + if upstream["base"]["ref"] != "main" or ( + upstream["state"] == "closed" and not upstream.get("merged") + ): + raise ValueError("Upstream PR must target main and be open (draft allowed) or merged.") + prs[upstream_number] = upstream + assignments = _map_commits(commits, prs, args.map_upstream) if prs else {} + upstream_main = _sha(gh.get(f"repos/{_FLASHINFER}/git/ref/heads/main")["object"]["sha"]) + upstream_base = gh.compare(canonical_repo, previous.commit, upstream_main)["merge_base_commit"][ + "sha" + ] + groups = [] + if args.unpaired_reason is not None: + groups.append( + { + "commits": [item["sha"] for item in commits], + "upstream_pr": None, + "upstream_base": None, + "upstream_head": None, + "upstream_merge": None, + "unpaired_reason": args.unpaired_reason.strip(), + "refresh_policy": "retain", + } + ) + for upstream_number, upstream in prs.items(): + groups.append( + { + "commits": [ + item["sha"] for item in commits if assignments[item["sha"]] == upstream_number + ], + "upstream_pr": f"https://github.com/{_FLASHINFER}/pull/{upstream_number}", + "upstream_base": _sha(upstream["base"]["sha"]), + "upstream_head": _sha(upstream["head"]["sha"]), + "upstream_merge": upstream.get("merge_commit_sha") + if upstream.get("merged") + else None, + } + ) + record.update( + { + "upstream_base": _sha(upstream_base), + "upstream_observed_main": upstream_main, + "changes": groups, + } + ) + _check_current(gh, plan) + return plan + + +def _find_followup(gh: GitHub, plan: Promotion) -> dict | None: + query = urlencode( + {"state": "all", "base": "main", "head": f"{plan.fork.split('/')[0]}:{plan.branch}"} + ) + prs = gh.api(f"repos/{_TRTLLM}/pulls?{query}") + if not isinstance(prs, list) or len(prs) > 1: + raise ValueError("Ambiguous existing promotion PRs.") + if prs and prs[0]["state"] == "closed" and not prs[0].get("merged_at"): + raise ValueError("The promotion PR was closed without merging; inspect it before retrying.") + return prs[0] if prs else None + + +def _adopt_record(plan: Promotion, message: str) -> None: + """Keep the original upstream snapshot when resuming an already-published run.""" + existing = _record_from_message(message) + stable_keys = set(plan.record) - {"upstream_base", "upstream_observed_main", "changes"} + if any(existing.get(key) != plan.record[key] for key in stable_keys): + raise ValueError("Existing promotion metadata does not describe this source update.") + _validate_snapshot(existing) + if "changes" not in plan.record: + # Only the read-only, already-merged path restores an entire snapshot. + plan.record = existing + return + expected = { + group["upstream_pr"]: ( + group["commits"], + group.get("unpaired_reason"), + group.get("refresh_policy"), + ) + for group in plan.record["changes"] + } + actual = { + group["upstream_pr"]: ( + group["commits"], + group.get("unpaired_reason"), + group.get("refresh_policy"), + ) + for group in existing["changes"] + } + if actual != expected: + raise ValueError("Existing promotion has different upstream PR/commit mappings.") + plan.record = existing + + +def _validate_snapshot(record: dict) -> None: + _sha(record["upstream_base"]) + _sha(record["upstream_observed_main"]) + commits = [] + if not isinstance(record["changes"], list) or not record["changes"]: + raise ValueError("Promotion metadata must contain source changes.") + for group in record["changes"]: + if not isinstance(group["commits"], list) or not group["commits"]: + raise ValueError("Promotion change group has no commits.") + commits.extend(_sha(sha) for sha in group["commits"]) + if group["upstream_pr"] is None: + if ( + group.get("refresh_policy") != "retain" + or not group.get("unpaired_reason", "").strip() + ): + raise ValueError( + "Unpaired promotion changes must retain an explicit reason/policy." + ) + else: + _pr_number(group["upstream_pr"], _FLASHINFER) + _sha(group["upstream_base"]) + _sha(group["upstream_head"]) + if group["upstream_merge"] is not None: + _sha(group["upstream_merge"]) + if len(set(commits)) != len(commits) or record["promoted"] not in commits: + raise ValueError("Promotion metadata has duplicate or missing source commits.") + + +def _verify_remote_commit(gh: GitHub, plan: Promotion, repo: str, sha: str) -> str: + commit = gh.get(f"repos/{repo}/commits/{_sha(sha)}") + if len(commit["parents"]) != 1 or [item["filename"] for item in commit["files"]] != [_LOCK]: + raise ValueError("Promotion commit must change only the vendor lock and have one parent.") + parent = commit["parents"][0]["sha"] + comparison = gh.compare(_TRTLLM, parent, _main_sha(gh)) + if comparison["merge_base_commit"]["sha"] != parent: + raise ValueError( + "Promotion parent is not on TRT-LLM main; unrelated commits may be present." + ) + before = _lock_document_at(gh, repo, parent) + after = _lock_document_at(gh, repo, sha) + _check_document_change(before, after, plan) + message = commit["commit"]["message"] + _adopt_record(plan, message) + if not re.search(r"^Signed-off-by: .+ <.+>$", message, re.MULTILINE): + raise ValueError("Promotion commit has no DCO sign-off.") + return message + + +def _check_document_change(before: dict, after: dict, plan: Promotion) -> None: + if before["vendors"][_VENDOR] != plan.reviewed.to_mapping(): + raise ValueError("Promotion parent does not contain the reviewed temporary pin.") + expected = {**before, "vendors": {**before["vendors"], _VENDOR: plan.promoted}} + if expected != after: + raise ValueError("Promotion commit is not the expected branch/URL-only lock change.") + + +@contextlib.contextmanager +def _source_repo(args: argparse.Namespace, plan: Promotion) -> Iterator[Path]: + with tempfile.TemporaryDirectory(prefix="prims-ts-promote-source-") as temporary: + repo = args.flashinfer_repo or Path(temporary) + if args.flashinfer_repo is None: + print(f"Fetching reviewed FlashInfer source {plan.reviewed.commit}...", flush=True) + _git(repo, "init", "--quiet") + _git(repo, "fetch", "--quiet", "--no-tags", plan.reviewed.url, plan.reviewed.commit) + _git(repo, "cat-file", "-e", f"{plan.reviewed.commit}^{{commit}}") + _git(repo, "merge-base", "--is-ancestor", plan.previous.commit, plan.reviewed.commit) + yield repo + + +def _local_lock_document(worktree: Path, revision: str | None = None) -> dict: + content = ( + (worktree / _LOCK).read_text() + if revision is None + else _git(worktree, "show", f"{revision}:{_LOCK}") + ) + return yaml.load(content, Loader=vendor._UniqueKeyLoader) + + +def _pending_paths(worktree: Path) -> set[str]: + # A file restored in the working tree can still have an unrelated staged edit. + return set(_git(worktree, "diff", "--name-only", "HEAD").splitlines()) | set( + _git(worktree, "diff", "--cached", "--name-only", "HEAD").splitlines() + ) + + +def _check_lock_only(worktree: Path, before: dict, plan: Promotion) -> None: + _check_document_change(before, _local_lock_document(worktree), plan) + if _pending_paths(worktree) != {_LOCK}: + raise ValueError("Promotion must change only the vendor lock.") + if _git(worktree, "ls-files", "--others", "--exclude-standard"): + raise ValueError("Unexpected untracked files in the promotion worktree.") + + +def _prepare_commit(args: argparse.Namespace, plan: Promotion, source: Path) -> tuple[Path, str]: + print(f"Fetching TRT-LLM base {plan.main_sha}...", flush=True) + _git( + args.repo, + "fetch", + "--quiet", + "--no-tags", + f"https://github.com/{_TRTLLM}.git", + plan.main_sha, + ) + worktree = ( + args.worktree + or args.repo.parent / f"prims-ts-promote-{plan.number}-{plan.reviewed.commit[:12]}" + ) + worktree = worktree.resolve() + if worktree.exists(): + if _git(worktree, "branch", "--show-current") != plan.branch or _git( + worktree, "rev-parse", "--path-format=absolute", "--git-common-dir" + ) != _git(args.repo, "rev-parse", "--path-format=absolute", "--git-common-dir"): + raise ValueError( + f"Existing worktree is not this repository's promotion branch: {worktree}" + ) + head = _git(worktree, "rev-parse", "HEAD") + if _git(worktree, "merge-base", head, plan.main_sha) != head: + if _git(worktree, "status", "--porcelain"): + raise ValueError( + f"Completed promotion worktree contains unexpected edits: {worktree}" + ) + _adopt_record(plan, _git(worktree, "log", "-1", "--format=%B")) + parent = _git(worktree, "rev-parse", "HEAD^") + _git(worktree, "merge-base", "--is-ancestor", parent, plan.main_sha) + before = _local_lock_document(worktree, "HEAD^") + _check_committed_lock(worktree, before, plan) + lock = vendor._load_lock(worktree / _LOCK) + vendor._verify_source(lock, lock.vendors[_VENDOR], source) + print(f"Reusing completed promotion commit {head} in {worktree}.", flush=True) + return worktree, head + print(f"Resuming incomplete promotion in {worktree}...", flush=True) + else: + print(f"Creating promotion worktree {worktree} (LFS downloads disabled)...", flush=True) + _git(args.repo, "worktree", "add", "-b", plan.branch, str(worktree), plan.main_sha) + before = _local_lock_document(worktree, "HEAD") + if before["vendors"][_VENDOR] != plan.reviewed.to_mapping(): + raise ValueError("Worktree base no longer contains the reviewed temporary pin.") + # Recover only the two states this tool writes. Preserve arbitrary user or + # hook edits, including staged-only changes, instead of overwriting them. + for revision in (None, ""): + pending = _local_lock_document(worktree, revision) + if pending != before: + _check_document_change(before, pending, plan) + if _pending_paths(worktree) - {_LOCK}: + raise ValueError( + "Incomplete promotion has changes outside the vendor lock; inspect before retrying." + ) + if _git(worktree, "ls-files", "--others", "--exclude-standard"): + raise ValueError("Incomplete promotion has untracked files; inspect before retrying.") + lock = vendor._load_lock(worktree / _LOCK) + print("Verifying source materialization and the lock-only change...", flush=True) + vendor._verify_source(lock, lock.vendors[_VENDOR], source) + result = vendor.main( + [ + "--lock", + str(worktree / _LOCK), + "pin", + _VENDOR, + "--url", + plan.previous.url, + "--branch", + plan.previous.branch, + "--commit", + plan.reviewed.commit, + "--repo", + str(source), + ] + ) + if result: + raise ValueError("Vendor pin failed; worktree preserved for inspection.") + _check_lock_only(worktree, before, plan) + _git(worktree, "add", "--", _LOCK) + print("Creating signed-off promotion commit...", flush=True) + _git(worktree, "commit", "-s", "-F", "-", text=f"{plan.title}\n\n{_record_text(plan.record)}\n") + if _git(worktree, "status", "--porcelain"): + raise ValueError(f"Commit hooks left changes in {worktree}; inspect and retry.") + _check_committed_lock(worktree, before, plan) + return worktree, _git(worktree, "rev-parse", "HEAD") + + +def _check_committed_lock(worktree: Path, before: dict, plan: Promotion) -> None: + _check_document_change(before, _local_lock_document(worktree), plan) + if len(_git(worktree, "rev-list", "--parents", "-n", "1", "HEAD").split()) != 2: + raise ValueError("Promotion commit must have exactly one parent.") + if _git(worktree, "diff", "--name-only", "HEAD^", "HEAD") != _LOCK: + raise ValueError("Commit hooks added changes outside the vendor lock.") + message = _git(worktree, "log", "-1", "--format=%B") + if _record_from_message(message) != plan.record: + raise ValueError("Commit hooks changed the promotion provenance.") + if not re.search(r"^Signed-off-by: .+ <.+>$", message, re.MULTILINE): + raise ValueError("Promotion commit has no DCO sign-off.") + + +def _check_auto_merge(gh: GitHub) -> None: + settings = gh.get(f"repos/{_TRTLLM}") + if not settings.get("allow_auto_merge") or not settings.get("allow_squash_merge"): + raise ValueError("Repository must allow auto-merge and squash merging.") + rules = gh.api(f"repos/{_TRTLLM}/rules/branches/main") + if not isinstance(rules, list) or any(rule["type"] == "merge_queue" for rule in rules): + raise ValueError( + "Merge queues are unsupported: custom squash-message retention is required." + ) + + +def _disable_auto_merge(gh: GitHub, pr: dict) -> None: + gh.api( + "graphql", + method="POST", + payload={ + "query": ( + "mutation($id:ID!){disablePullRequestAutoMerge(" + "input:{pullRequestId:$id}){clientMutationId}}" + ), + "variables": {"id": pr["node_id"]}, + }, + ) + + +def _report_merged(pr: dict) -> None: + print( + f"Promotion merged: {pr['html_url']}\n" + f"Squash commit: {pr['merge_commit_sha']}\n" + "Final lock-only change, provenance, and DCO sign-off verified.", + flush=True, + ) + + +def _arm_auto_merge(gh: GitHub, plan: Promotion, pr: dict, message: str) -> None: + _check_auto_merge(gh) + _check_current(gh, plan) + number, head = pr["number"], pr["head"]["sha"] + state = gh.api( + "graphql", + method="POST", + payload={ + "query": "query($id:ID!){node(id:$id){... on PullRequest{headRefOid mergeStateStatus}}}", + "variables": {"id": pr["node_id"]}, + }, + ) + node = state["data"]["node"] + if node["headRefOid"] != head: + raise ValueError("Follow-up PR head changed; revalidate before enabling auto-merge.") + title, _, body = message.partition("\n") + saved = pr.get("auto_merge") + if saved: + if ( + saved.get("merge_method") == "squash" + and saved.get("commit_title") == title + and saved.get("commit_message", "").strip() == body.strip() + ): + print(f"Existing squash auto-merge message verified: {pr['html_url']}") + return + _disable_auto_merge(gh, pr) + raise ValueError("Existing auto-merge message differs; disabled. Inspect before retrying.") + # Query only necessary fields: gh pr merge's broader PR query can require + # read:org even when the token already has permission to merge this PR. + operation = ( + "mergePullRequest" if node["mergeStateStatus"] == "CLEAN" else "enablePullRequestAutoMerge" + ) + input_type = ( + "MergePullRequestInput" + if operation == "mergePullRequest" + else "EnablePullRequestAutoMergeInput" + ) + gh.api( + "graphql", + method="POST", + payload={ + "query": f"mutation($input:{input_type}!){{{operation}(input:$input){{clientMutationId}}}}", + "variables": { + "input": { + "pullRequestId": pr["node_id"], + "expectedHeadOid": head, + "mergeMethod": "SQUASH", + "commitHeadline": title, + "commitBody": body.strip(), + } + }, + }, + ) + updated = gh.get(f"repos/{_TRTLLM}/pulls/{number}") + if updated.get("merged"): + _verify_remote_commit(gh, plan, _TRTLLM, updated["merge_commit_sha"]) + _report_merged(updated) + else: + saved = updated.get("auto_merge") or {} + if ( + updated["head"]["sha"] != head + or saved.get("merge_method") != "squash" + or saved.get("commit_title") != title + or saved.get("commit_message", "").strip() != body.strip() + ): + _disable_auto_merge(gh, pr) + raise ValueError("Auto-merge message/head verification failed; auto-merge disabled.") + print(f"Squash auto-merge enabled and message verified: {updated['html_url']}") + + +def _pr_body(plan: Promotion, message: str) -> str: + unpaired = "" + if any(group["upstream_pr"] is None for group in plan.record["changes"]): + unpaired = ( + "No paired FlashInfer PR exists for this source update. Its provenance " + "explicitly records it as unpaired, with `refresh_policy: retain`. A future " + "refresh must carry it forward until an upstream pairing or equivalence " + "is explicitly established.\n\n" + ) + return ( + "@coderabbitai summary\n\n## Description\n\n" + f"Promote the exact source revision reviewed in {plan.record['trtllm_pr']}.\n" + "Only the PrimTS lock URL/branch changes. The source SHA, selected vendor files, " + "compatibility patch, and digests are unchanged.\n\n" + f"{unpaired}" + "## Test Coverage\n\nSource materialization and lock-only diff verified by the promotion tool. " + "No new kernel behavior. The tool posts " + f"`{_CI_SKIP_COMMAND}` to request the bot's no-code-change CI path. " + "Automatic GitHub checks and required reviews still apply.\n\n" + "## PR Checklist\n\n" + "- [x] No runtime/API/dependency changes.\n" + "- [x] Immutable source pin and vendor content preserved.\n" + "- [x] Please check this after reviewing the above items as appropriate for this PR.\n\n" + "## Required squash commit message\n\n" + "Preserve the following message, including the provenance block and DCO sign-off.\n\n" + f"```text\n{message.strip()}\n```\n" + ) + + +def _ensure_ci_skip(gh: GitHub, plan: Promotion, pr: dict) -> dict: + """Request the no-code-change bot path once across sequential retries.""" + current = gh.get(f"repos/{_TRTLLM}/pulls/{pr['number']}") + if current.get("merged"): + _verify_remote_commit(gh, plan, _TRTLLM, current["merge_commit_sha"]) + _report_merged(current) + return current + if current["state"] != "open" or current["head"]["sha"] != pr["head"]["sha"]: + raise ValueError("Follow-up PR closed or changed before the CI request; inspect it.") + _check_current(gh, plan) + _verify_remote_commit(gh, plan, _TRTLLM, current["head"]["sha"]) + actor = gh.get("user")["login"] + endpoint = f"repos/{_TRTLLM}/issues/{pr['number']}/comments" + page = 1 + while True: + comments = gh.api(f"{endpoint}?per_page=100&page={page}") + if not isinstance(comments, list): + raise ValueError("Unexpected issue-comment response; refusing a duplicate CI request.") + if any( + comment.get("body", "").strip() == _CI_SKIP_COMMAND + and comment.get("user", {}).get("login") == actor + for comment in comments + ): + print(f"Existing CI skip request found: {current['html_url']}", flush=True) + return current + if len(comments) < 100: + break + page += 1 + comment = gh.api(endpoint, method="POST", payload={"body": _CI_SKIP_COMMAND}) + print(f"Posted CI skip request: {comment['html_url']}", flush=True) + return current + + +def _publish(args: argparse.Namespace, gh: GitHub, plan: Promotion) -> dict: + if plan.completed_pr is not None: + _report_merged(plan.completed_pr) + return plan.completed_pr + existing = _find_followup(gh, plan) + remote = gh.optional(f"repos/{plan.fork}/git/ref/heads/{plan.branch}") + if existing: + existing = gh.get(f"repos/{_TRTLLM}/pulls/{existing['number']}") + sha = existing["merge_commit_sha"] if existing.get("merged") else existing["head"]["sha"] + message = _verify_remote_commit(gh, plan, _TRTLLM, sha) + if existing.get("merged"): + _report_merged(existing) + return existing + if remote is None or remote["object"]["sha"] != sha: + raise ValueError("Follow-up branch is missing or changed; inspect before resuming.") + elif remote: + sha = remote["object"]["sha"] + message = _verify_remote_commit(gh, plan, plan.fork, sha) + else: + message = "" + if args.auto_merge: + _check_auto_merge(gh) + _check_current(gh, plan) + with _source_repo(args, plan) as source: + if remote is None: + worktree, sha = _prepare_commit(args, plan, source) + message = _git(worktree, "log", "-1", "--format=%B") + _check_current(gh, plan) + # No force push: Git also enforces the fast-forward constraint remotely. + print(f"Ensuring canonical branch points to {plan.reviewed.commit}...", flush=True) + _git( + source, + "push", + plan.previous.url, + f"{plan.reviewed.commit}:refs/heads/{plan.previous.branch}", + ) + print(f"Canonical branch now contains reviewed revision {plan.reviewed.commit}.") + if remote is None: + print(f"Publishing follow-up branch {plan.fork}:{plan.branch}...", flush=True) + _git( + worktree, + "push", + f"https://github.com/{plan.fork}.git", + f"{sha}:refs/heads/{plan.branch}", + ) + _check_current(gh, plan) + if existing is None: + print("Creating the lock-only follow-up PR...", flush=True) + existing = gh.api( + f"repos/{_TRTLLM}/pulls", + method="POST", + payload={ + "title": plan.title, + "body": _pr_body(plan, message), + "base": "main", + "head": f"{plan.fork.split('/')[0]}:{plan.branch}", + "head_repo": plan.fork.split("/")[1], + }, + ) + print(f"Lock-only promotion PR: {existing['html_url']}") + existing = _ensure_ci_skip(gh, plan, existing) + if existing.get("merged"): + return existing + if args.auto_merge: + _arm_auto_merge(gh, plan, existing, message) + if args.wait: + deadline = time.monotonic() + args.timeout + while True: + current = gh.get(f"repos/{_TRTLLM}/pulls/{existing['number']}") + if current.get("merged"): + _verify_remote_commit(gh, plan, _TRTLLM, current["merge_commit_sha"]) + _report_merged(current) + break + if current["state"] == "closed" or current["head"]["sha"] != existing["head"]["sha"]: + if args.auto_merge and current.get("auto_merge"): + _disable_auto_merge(gh, current) + raise ValueError( + "Follow-up PR closed or changed while waiting; inspect before proceeding." + ) + if time.monotonic() >= deadline: + raise TimeoutError( + "Promotion PR is still pending; rerun promote to verify after merge." + ) + time.sleep(min(15, max(0, deadline - time.monotonic()))) + return existing + + +def main(argv: list[str] | None = None) -> int: + """Run the maintainer CLI; dry-run unless publication is explicitly requested.""" + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + promote = commands.add_parser("promote", help="Promote a merged TRT-LLM source update.") + promote.add_argument("--trtllm-pr", required=True) + promote.add_argument( + "--canonical-repo", + required=True, + help="Explicit canonical FlashInfer fork OWNER/REPO (not the developer's temporary fork).", + ) + promote.add_argument( + "--canonical-branch", + default="trtllm-prims-ts-dev", + help="Expected canonical branch; specify a dated branch when applicable.", + ) + provenance = promote.add_mutually_exclusive_group(required=True) + provenance.add_argument("--upstream-pr", action="append", default=[]) + provenance.add_argument( + "--unpaired-reason", + help="Explicit exception for changes with no upstream PR; retain them during refresh.", + ) + promote.add_argument("--map-upstream", action="append", default=[], metavar="COMMIT=PR") + promote.add_argument("--repo", type=Path, default=Path(__file__).resolve().parents[1]) + promote.add_argument( + "--fork", help="TRT-LLM personal fork OWNER/REPO; defaults to remote 'fork'." + ) + promote.add_argument( + "--flashinfer-repo", + type=Path, + help="Local source repository containing both immutable SHAs.", + ) + promote.add_argument("--worktree", type=Path, help="New/preserved follow-up worktree path.") + promote.add_argument( + "--publish", + action="store_true", + help="Publish the lock-only PR and post the no-code-change /bot skip request.", + ) + promote.add_argument("--auto-merge", action="store_true") + promote.add_argument( + "--wait", action="store_true", help="Wait for merge and verify final commit metadata." + ) + promote.add_argument( + "--timeout", type=int, default=3600, help="Maximum --wait duration in seconds." + ) + args = parser.parse_args(argv) + if (args.auto_merge or args.wait) and not args.publish: + parser.error("--auto-merge and --wait require --publish") + if args.timeout <= 0: + parser.error("--timeout must be positive") + args.repo = args.repo.resolve() + try: + gh = GitHub() + plan = _make_plan(args, gh) + mode = ( + "ALREADY COMPLETED" if plan.completed_pr else "PUBLISH" if args.publish else "DRY RUN" + ) + print(f"{mode}: {plan.canonical_repo}:{plan.previous.branch}", flush=True) + print(f"Source: {plan.previous.commit} -> {plan.reviewed.commit}") + print(f"Follow-up: {plan.fork}:{plan.branch}") + print(_record_text(plan.record)) + if args.publish: + _publish(args, gh, plan) + elif plan.completed_pr is not None: + _report_merged(plan.completed_pr) + else: + print(f"On publication: post {_CI_SKIP_COMMAND}") + print(f"Squash auto-merge: {'enabled' if args.auto_merge else 'off'}") + print("No remote writes or consumer worktree changes. Use --publish to proceed.") + return 0 + except ( + RuntimeError, + ValueError, + OSError, + KeyError, + subprocess.TimeoutExpired, + vendor.VendorError, + ) as error: + print(f"Promotion stopped: {error}", file=sys.stderr) + print( + "Any completed pushes/PRs are preserved. Inspect the error before retrying.", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/unittest/others/test_maintain_prims_ts.py b/tests/unittest/others/test_maintain_prims_ts.py new file mode 100644 index 000000000000..77d6d3cb9777 --- /dev/null +++ b/tests/unittest/others/test_maintain_prims_ts.py @@ -0,0 +1,1002 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Promotion lifecycle tests: real local Git, fake GitHub, no network or GPUs.""" + +from __future__ import annotations + +import argparse +import base64 +import copy +import dataclasses +import importlib.util +import runpy +import subprocess +import sys +from pathlib import Path + +import pytest + +# tests/unittest/scripts shadows the repository's scripts namespace. Load the +# standalone tool as an isolated package, without editing sys.path. +_SCRIPTS = Path(__file__).resolve().parents[3] / "scripts" +_SPEC = importlib.util.spec_from_file_location( + "prims_ts_maintenance_test", + _SCRIPTS / "maintain_prims_ts.py", + submodule_search_locations=[str(_SCRIPTS)], +) +m = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = m +_SPEC.loader.exec_module(m) + +pytestmark = pytest.mark.cpu_only + + +def _git(repo: Path, *args: str, text: str | None = None) -> str: + result = subprocess.run( + ["git", "-C", str(repo), *args], input=text, text=True, capture_output=True, timeout=30 + ) + assert result.returncode == 0, result.stderr + return result.stdout.strip() + + +def _init(path: Path) -> Path: + path.mkdir() + _git(path, "init", "-q", "-b", "main") + _git(path, "config", "user.name", "Promotion Test") + _git(path, "config", "user.email", "promotion@example.invalid") + _git(path, "config", "core.hooksPath", str(path / "empty-hooks")) + return path + + +def _write(repo: Path, name: str, content: str) -> None: + file = repo / name + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text(content) + + +def _commit(repo: Path, message: str) -> str: + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-s", "-m", message) + return _git(repo, "rev-parse", "HEAD") + + +class World(m.GitHub): + """GitHub facade whose immutable data comes from temporary Git repositories.""" + + def __init__(self, root: Path) -> None: + self.root = root + self.source = _init(root / "flashinfer") + _write(self.source, "prims_ts/kernel.py", "VALUE = 0\n") + self.upstream = _commit(self.source, "upstream baseline") + _write(self.source, "prims_ts/kernel.py", "VALUE = 1\n") + self.previous = _commit(self.source, "previous downstream fix") + _write(self.source, "prims_ts/kernel.py", "VALUE = 2\n") + self.first = _commit(self.source, "first fix") + _write(self.source, "prims_ts/kernel.py", "VALUE = 3\n") + self.reviewed = _commit(self.source, "second fix") + self.consumer = _init(root / "trtllm") + assert ( + m.vendor.main( + [ + "--lock", + str(self.consumer / m._LOCK), + "create", + m._VENDOR, + "--url", + "https://github.com/maintainer/flashinfer.git", + "--branch", + "trtllm-prims-ts-dev", + "--commit", + self.previous, + "--source", + "prims_ts", + "--destination", + "vendored/prims_ts", + "--include", + "**/*.py", + "--repo", + str(self.source), + ] + ) + == 0 + ) + self.base = _commit(self.consumer, "previous vendor pin") + _write(self.consumer, "vendored/prims_ts/kernel.py", "VALUE = 3\n") + assert ( + m.vendor.main( + [ + "--lock", + str(self.consumer / m._LOCK), + "pin", + m._VENDOR, + "--url", + "https://github.com/dev/flashinfer.git", + "--branch", + "tmp-fix", + "--commit", + self.reviewed, + "--repo", + str(self.source), + ] + ) + == 0 + ) + self.merged = _commit(self.consumer, "merged source update") + self.main = self.merged + self.canonical = root / "canonical.git" + self.fork = root / "trtllm-fork.git" + _git(root, "clone", "-q", "--bare", str(self.source), str(self.canonical)) + _git(root, "clone", "-q", "--bare", str(self.consumer), str(self.fork)) + _git(self.canonical, "update-ref", "refs/heads/trtllm-prims-ts-dev", self.previous) + _git( + self.consumer, "remote", "add", "fork", "https://github.com/maintainer/TensorRT-LLM.git" + ) + self.prs: dict[int, dict] = {} + self.calls: list[tuple[str, str, dict | None]] = [] + self.overrides: dict[str, dict | list] = {} + self.state = "BLOCKED" + self.tamper_auto = False + self.fail_create = False + self.comments: list[dict] = [] + self.fail_comment_before = False + self.fail_comment_after = False + self.git_calls: list[tuple[str, ...]] = [] + + def args(self, **overrides: object) -> argparse.Namespace: + values = dict( + trtllm_pr="17", + canonical_repo="maintainer/flashinfer", + canonical_branch="trtllm-prims-ts-dev", + upstream_pr=["4829"], + unpaired_reason=None, + map_upstream=[], + repo=self.consumer, + fork=None, + flashinfer_repo=self.source, + worktree=self.root / "followup", + publish=False, + auto_merge=False, + wait=False, + timeout=30, + ) + values.update(overrides) + return argparse.Namespace(**values) + + def commit_data(self, repo: Path, sha: str) -> dict: + return { + "sha": sha, + "parents": [{"sha": s} for s in _git(repo, "show", "-s", "--format=%P", sha).split()], + "commit": {"message": _git(repo, "show", "-s", "--format=%B", sha)}, + "files": [ + {"filename": f} + for f in _git( + repo, "diff-tree", "--no-commit-id", "--name-only", "-r", sha + ).splitlines() + ], + } + + def compare(self, repo: str, base: str, head: str, *, all_commits: bool = False) -> dict: + source = self.consumer if repo == m._TRTLLM else self.source + commits = _git(source, "rev-list", "--reverse", f"{base}..{head}").splitlines() + return { + "merge_base_commit": {"sha": _git(source, "merge-base", base, head)}, + "commits": [self.commit_data(source, sha) for sha in commits], + "total_commits": len(commits), + } + + def routed_git(self, repo: Path, *args: str, text: str | None = None) -> str: + self.git_calls.append(args) + replacements = { + "https://github.com/NVIDIA/TensorRT-LLM.git": str(self.consumer), + "https://github.com/maintainer/TensorRT-LLM.git": str(self.fork), + "https://github.com/maintainer/flashinfer.git": str(self.canonical), + "https://github.com/dev/flashinfer.git": str(self.source), + } + if args[:3] == ("remote", "get-url", "fork"): + return "https://github.com/maintainer/TensorRT-LLM.git" + return _git(repo, *[replacements.get(arg, arg) for arg in args], text=text) + + def api( + self, endpoint: str, *, method: str = "GET", payload: dict | None = None + ) -> dict | list: + self.calls.append((method, endpoint, payload)) + if method == "GET" and endpoint in self.overrides: + return copy.deepcopy(self.overrides[endpoint]) + if endpoint == "user": + return {"login": "maintainer"} + if endpoint == "graphql": + query = payload["query"] + pr = self.prs[99] + if query.startswith("query"): + return { + "data": { + "node": {"headRefOid": pr["head"]["sha"], "mergeStateStatus": self.state} + } + } + if "disablePullRequestAutoMerge" in query: + pr["auto_merge"] = None + return {"data": {}} + data = payload["variables"]["input"] + assert data["expectedHeadOid"] == pr["head"]["sha"] + if "enablePullRequestAutoMerge" in query: + pr["auto_merge"] = { + "merge_method": "squash", + "commit_title": data["commitHeadline"], + "commit_message": "lost metadata" if self.tamper_auto else data["commitBody"], + } + else: + message = f"{data['commitHeadline']}\n\n{data['commitBody']}\n" + tree = _git(self.consumer, "rev-parse", f"{pr['head']['sha']}^{{tree}}") + sha = _git(self.consumer, "commit-tree", tree, "-p", self.main, text=message) + self.main = sha + pr.update(merged=True, merged_at="today", state="closed", merge_commit_sha=sha) + return {"data": {}} + prefix = "repos/" + assert endpoint.startswith(prefix), endpoint + owner, name, *rest = endpoint[len(prefix) :].split("/") + repository = f"{owner}/{name}" + route = "/".join(rest) + if route.startswith("issues/99/comments"): + if method == "GET": + page = int(route.rsplit("page=", 1)[1]) + return copy.deepcopy(self.comments[(page - 1) * 100 : page * 100]) + if self.fail_comment_before: + raise RuntimeError("simulated comment failure before posting") + comment = { + "body": payload["body"], + "user": {"login": "maintainer"}, + "html_url": f"https://github.com/NVIDIA/TensorRT-LLM/pull/99#issuecomment-{len(self.comments) + 1}", + } + self.comments.append(comment) + if self.fail_comment_after: + raise RuntimeError("simulated comment failure after posting") + return copy.deepcopy(comment) + if not route: + root = m._FLASHINFER if name == "flashinfer" else m._TRTLLM + return { + "source": {"full_name": root}, + "allow_auto_merge": True, + "allow_squash_merge": True, + } + if route == "rules/branches/main": + return [{"type": "required_status_checks"}] + if route.startswith("git/ref/heads/"): + branch = route[len("git/ref/heads/") :] + if repository == m._TRTLLM: + return {"object": {"sha": self.main}} + if repository == m._FLASHINFER: + return {"object": {"sha": self.upstream}} + remote = self.canonical if name == "flashinfer" else self.fork + result = subprocess.run( + ["git", "-C", str(remote), "rev-parse", "--verify", f"refs/heads/{branch}"], + capture_output=True, + text=True, + ) + if result.returncode: + raise RuntimeError("HTTP 404") + return {"object": {"sha": result.stdout.strip()}} + if route.startswith("contents/"): + path, sha = route[len("contents/") :].split("?ref=") + content = _git(self.consumer, "show", f"{sha}:{path}") + "\n" + return {"encoding": "base64", "content": base64.b64encode(content.encode()).decode()} + if route.startswith("commits/"): + return self.commit_data(self.consumer, route.split("/")[1]) + if route.startswith("pulls?"): + return copy.deepcopy(list(self.prs.values())) + if route == "pulls" and method == "POST": + if self.fail_create: + raise RuntimeError("simulated PR creation failure") + branch = payload["head"].split(":")[1] + head = _git(self.fork, "rev-parse", f"refs/heads/{branch}") + self.prs[99] = dict( + number=99, + node_id="PR_test", + state="open", + merged=False, + head={"sha": head}, + html_url="https://github.com/NVIDIA/TensorRT-LLM/pull/99", + body=payload["body"], + auto_merge=None, + ) + return copy.deepcopy(self.prs[99]) + if route.startswith("pulls/"): + number = int(route.split("/")[1]) + if repository == m._FLASHINFER: + return { + "number": number, + "base": {"ref": "main", "sha": self.upstream}, + "head": {"sha": self.reviewed}, + "state": "open", + "merged": False, + "draft": True, + } + if number == 17: + return { + "number": 17, + "merged": True, + "base": {"ref": "main"}, + "merge_commit_sha": self.merged, + "head": {"sha": "a" * 40}, + } + return copy.deepcopy(self.prs[number]) + raise AssertionError((method, endpoint, payload)) + + +@pytest.fixture +def world(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> World: + instance = World(tmp_path) + monkeypatch.setattr(m, "_git", instance.routed_git) + monkeypatch.setattr(m, "GitHub", lambda: instance) + return instance + + +def test_dry_run_has_no_writes_and_pins_merged_not_live_head( + world: World, capsys: pytest.CaptureFixture +) -> None: + assert ( + m.main( + [ + "promote", + "--trtllm-pr", + "17", + "--canonical-repo", + "maintainer/flashinfer", + "--upstream-pr", + "4829", + "--repo", + str(world.consumer), + ] + ) + == 0 + ) + assert "DRY RUN" in capsys.readouterr().out + assert all(method == "GET" for method, _, _ in world.calls) + assert not any(args[0] in ("push", "commit", "worktree") for args in world.git_calls) + plan = m._make_plan(world.args(), world) + assert plan.reviewed.commit == world.reviewed + assert plan.record["changes"][0]["commits"] == [world.first, world.reviewed] + assert plan.record["upstream_base"] == world.upstream + + +def test_publish_signed_lock_only_and_repeat(world: World) -> None: + args = world.args(publish=True) + plan = m._make_plan(args, world) + pr = m._publish(args, world, plan) + assert _git(world.canonical, "rev-parse", "trtllm-prims-ts-dev") == world.reviewed + head = pr["head"]["sha"] + assert _git(world.consumer, "diff", "--name-only", f"{head}^", head) == m._LOCK + assert m._lock_at(world, m._TRTLLM, head).to_mapping() == plan.promoted + message = _git(world.consumer, "show", "-s", "--format=%B", head) + assert m._record_from_message(message) == plan.record + assert "Signed-off-by: Promotion Test " in message + assert not any("graphql" == endpoint for _, endpoint, _ in world.calls) + m._publish(args, world, m._make_plan(args, world)) + assert ( + sum(method == "POST" and endpoint.endswith("/pulls") for method, endpoint, _ in world.calls) + == 1 + ) + + +@pytest.mark.parametrize("state", ["BLOCKED", "CLEAN"]) +def test_auto_merge_message_and_immediate_merge(world: World, state: str) -> None: + world.state = state + args = world.args(publish=True, auto_merge=True) + plan = m._make_plan(args, world) + m._publish(args, world, plan) + if state == "CLEAN": + assert world.prs[99]["merged"] + m._verify_remote_commit(world, plan, m._TRTLLM, world.main) + m._publish(args, world, m._make_plan(args, world)) + else: + saved = world.prs[99]["auto_merge"] + assert saved["merge_method"] == "squash" + assert m._record_from_message(saved["commit_message"]) == plan.record + assert "Signed-off-by:" in saved["commit_message"] + + +def test_resume_after_upstream_head_moves_retains_snapshot(world: World) -> None: + args = world.args(publish=True) + original = m._make_plan(args, world) + m._publish(args, world, original) + changed = world.get(f"repos/{m._FLASHINFER}/pulls/4829") + changed["head"]["sha"] = "b" * 40 + world.overrides[f"repos/{m._FLASHINFER}/pulls/4829"] = changed + resumed = m._make_plan(args, world) + assert resumed.record != original.record + m._publish(args, world, resumed) + assert resumed.record == original.record + + +def test_resume_after_push_before_pr_creation(world: World) -> None: + args = world.args(publish=True) + world.fail_create = True + with pytest.raises(RuntimeError, match="creation failure"): + m._publish(args, world, m._make_plan(args, world)) + assert _git(world.canonical, "rev-parse", "trtllm-prims-ts-dev") == world.reviewed + world.fail_create = False + m._publish(args, world, m._make_plan(args, world)) + assert len(world.prs) == 1 + + +def test_auto_merge_tampering_disables_request(world: World) -> None: + world.tamper_auto = True + args = world.args(publish=True, auto_merge=True) + with pytest.raises(ValueError, match="verification failed"): + m._publish(args, world, m._make_plan(args, world)) + assert world.prs[99]["auto_merge"] is None + + +@pytest.mark.parametrize("field,value", [("merged", False), ("base", {"ref": "release"})]) +def test_unmerged_or_wrong_base_rejected(world: World, field: str, value: object) -> None: + endpoint = f"repos/{m._TRTLLM}/pulls/17" + changed = world.get(endpoint) + changed[field] = value + world.overrides[endpoint] = changed + with pytest.raises(ValueError, match="already merged"): + m._make_plan(world.args(), world) + + +@pytest.mark.parametrize("target", ["main", "canonical"]) +def test_concurrent_updates_block_before_publication(world: World, target: str) -> None: + plan = m._make_plan(world.args(), world) + if target == "main": + world.main = world.base + else: + _git(world.canonical, "update-ref", "refs/heads/trtllm-prims-ts-dev", world.first) + with pytest.raises(ValueError, match="superseded|moved unexpectedly"): + m._publish(world.args(publish=True), world, plan) + assert not any(method != "GET" for method, _, _ in world.calls) + assert not any(args[0] == "push" for args in world.git_calls) + + +def test_divergent_source_rejected(world: World, monkeypatch: pytest.MonkeyPatch) -> None: + original = world.compare + + def divergent(*args: object, **kwargs: object) -> dict: + result = original(*args, **kwargs) + result["merge_base_commit"]["sha"] = world.upstream + return result + + monkeypatch.setattr(world, "compare", divergent) + with pytest.raises(ValueError, match="not a fast-forward"): + m._make_plan(world.args(), world) + + +def test_upstream_closed_unmerged_rejected(world: World) -> None: + endpoint = f"repos/{m._FLASHINFER}/pulls/4829" + changed = world.get(endpoint) + changed.update(state="closed", merged=False) + world.overrides[endpoint] = changed + with pytest.raises(ValueError, match="Upstream PR"): + m._make_plan(world.args(), world) + + +@pytest.mark.parametrize("setting", ["auto", "squash", "queue"]) +def test_unsupported_auto_merge_settings_fail_before_push(world: World, setting: str) -> None: + if setting == "queue": + world.overrides[f"repos/{m._TRTLLM}/rules/branches/main"] = [{"type": "merge_queue"}] + else: + data = world.get(f"repos/{m._TRTLLM}") + data[f"allow_{'auto_merge' if setting == 'auto' else 'squash_merge'}"] = False + world.overrides[f"repos/{m._TRTLLM}"] = data + args = world.args(publish=True, auto_merge=True) + with pytest.raises(ValueError, match="allow|Merge queues"): + m._publish(args, world, m._make_plan(args, world)) + assert not any(args[0] == "push" for args in world.git_calls) + + +def test_multiple_upstream_pr_mapping(world: World, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(m.sys.stdin, "isatty", lambda: False) + args = world.args(upstream_pr=["4829", "4830"]) + with pytest.raises(ValueError, match="--map-upstream"): + m._make_plan(args, world) + args.map_upstream = [f"{world.first}=4829", f"{world.reviewed}=4830"] + plan = m._make_plan(args, world) + assert [group["commits"] for group in plan.record["changes"]] == [ + [world.first], + [world.reviewed], + ] + + +@pytest.mark.parametrize("mapping", ["1234567=4829", "bad=4829", "abcdefg=4829"]) +def test_bad_mapping_rejected(world: World, mapping: str) -> None: + with pytest.raises(ValueError, match="invalid"): + m._make_plan(world.args(map_upstream=[mapping]), world) + + +def test_modified_vendor_files_block_before_push( + world: World, monkeypatch: pytest.MonkeyPatch +) -> None: + original = world.routed_git + + def tamper(repo: Path, *args: str, text: str | None = None) -> str: + result = original(repo, *args, text=text) + if args[:2] == ("worktree", "add"): + _write(world.root / "followup", "vendored/prims_ts/kernel.py", "UNREVIEWED = True\n") + return result + + monkeypatch.setattr(m, "_git", tamper) + args = world.args(publish=True) + with pytest.raises(ValueError, match="outside the vendor lock"): + m._publish(args, world, m._make_plan(args, world)) + assert not any(args[0] == "push" for args in world.git_calls) + + +def test_commit_hook_adds_unrelated_file_rejected( + world: World, monkeypatch: pytest.MonkeyPatch +) -> None: + original = world.routed_git + + def tamper(repo: Path, *args: str, text: str | None = None) -> str: + if args[0] == "commit": + _write(repo, "unrelated.txt", "unexpected hook edit\n") + _git(repo, "add", "unrelated.txt") + return original(repo, *args, text=text) + + monkeypatch.setattr(m, "_git", tamper) + args = world.args(publish=True) + with pytest.raises(ValueError, match="outside the vendor lock"): + m._publish(args, world, m._make_plan(args, world)) + assert not any(args[0] == "push" for args in world.git_calls) + + +@pytest.mark.parametrize( + "value", + [ + "http://github.com/user/repo", + "https://token@github.com/a/b", + "file:///tmp/source", + "https://example.com/a/b", + ], +) +def test_non_github_or_credential_urls_rejected(value: str) -> None: + with pytest.raises(ValueError, match="credential-free"): + m._repo_name(value) + + +@pytest.mark.parametrize("suffix", ["", "/", ".git"]) +def test_github_repo_url_parsing(suffix: str) -> None: + assert m._repo_name(f"https://github.com/owner/repo{suffix}") == "owner/repo" + + +@pytest.mark.parametrize("flags", [["--auto-merge"], ["--wait"], ["--timeout", "0"]]) +def test_cli_invalid_flags(flags: list[str]) -> None: + with pytest.raises(SystemExit) as raised: + m.main( + [ + "promote", + "--trtllm-pr", + "17", + "--canonical-repo", + "maintainer/flashinfer", + "--upstream-pr", + "4829", + *flags, + ] + ) + assert raised.value.code == 2 + + +def test_command_preserves_identity_and_config(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GH_CONFIG_DIR", "/tmp/test-gh-config") + monkeypatch.setenv("GIT_AUTHOR_NAME", "Real Maintainer") + monkeypatch.setenv("GIT_DIR", "/unrelated/git") + + def run(command: list[str], **kwargs: object) -> subprocess.CompletedProcess: + env = kwargs["env"] + assert env["GH_CONFIG_DIR"] == "/tmp/test-gh-config" + assert env["GIT_AUTHOR_NAME"] == "Real Maintainer" + assert env["GIT_LFS_SKIP_SMUDGE"] == "1" + assert "GIT_DIR" not in env + assert "shell" not in kwargs + return subprocess.CompletedProcess(command, 0, "ok\n", "") + + monkeypatch.setattr(m.subprocess, "run", run) + assert m._run(["git", "status"]) == "ok" + + +def test_metadata_missing_and_duplicate_rejected() -> None: + with pytest.raises(ValueError, match="Missing or ambiguous"): + m._record_from_message("ordinary commit") + record = m._record_text({"version": 1}) + with pytest.raises(ValueError, match="Missing or ambiguous"): + m._record_from_message(record + record) + + +def test_compare_paginates_all_commits(monkeypatch: pytest.MonkeyPatch) -> None: + gh = object.__new__(World) + + def get(endpoint: str) -> dict: + page = int(endpoint.rsplit("=", 1)[1]) + return {"total_commits": 101, "commits": list(range(100)) if page == 1 else [100]} + + monkeypatch.setattr(gh, "get", get) + result = m.GitHub.compare(gh, "a/b", "a" * 40, "b" * 40, all_commits=True) + assert result["commits"] == list(range(101)) + + +def test_existing_auto_merge_is_not_reenabled(world: World) -> None: + args = world.args(publish=True, auto_merge=True) + m._publish(args, world, m._make_plan(args, world)) + m._publish(args, world, m._make_plan(args, world)) + enables = [ + payload + for _, endpoint, payload in world.calls + if endpoint == "graphql" and "enablePullRequestAutoMerge" in payload["query"] + ] + assert len(enables) == 1 + + +def test_remote_lock_cannot_change_other_vendor_entries( + world: World, monkeypatch: pytest.MonkeyPatch +) -> None: + args = world.args(publish=True) + plan = m._make_plan(args, world) + pr = m._publish(args, world, plan) + original = m._lock_document_at + + def tampered(gh: m.GitHub, repo: str, sha: str) -> dict: + data = original(gh, repo, sha) + if sha == pr["head"]["sha"]: + data["vendors"]["unexpected"] = {} + return data + + monkeypatch.setattr(m, "_lock_document_at", tampered) + with pytest.raises(ValueError, match="branch/URL-only"): + m._verify_remote_commit(world, plan, m._TRTLLM, pr["head"]["sha"]) + + +def test_unmerged_parent_cannot_hide_unrelated_commits( + world: World, monkeypatch: pytest.MonkeyPatch +) -> None: + args = world.args(publish=True) + plan = m._make_plan(args, world) + pr = m._publish(args, world, plan) + original = world.compare + + def unrelated(repo: str, base: str, head: str, **kwargs: object) -> dict: + result = original(repo, base, head, **kwargs) + if repo == m._TRTLLM: + result["merge_base_commit"]["sha"] = world.base + return result + + monkeypatch.setattr(world, "compare", unrelated) + with pytest.raises(ValueError, match="not on TRT-LLM main"): + m._verify_remote_commit(world, plan, m._TRTLLM, pr["head"]["sha"]) + + +def test_wait_checks_final_commit(world: World) -> None: + world.state = "CLEAN" + args = world.args(publish=True, auto_merge=True, wait=True) + m._publish(args, world, m._make_plan(args, world)) + assert world.prs[99]["merged"] + + +def test_wait_timeout_preserves_enabled_auto_merge( + world: World, monkeypatch: pytest.MonkeyPatch +) -> None: + ticks = iter([0, 2]) + monkeypatch.setattr(m.time, "monotonic", lambda: next(ticks)) + args = world.args(publish=True, auto_merge=True, wait=True, timeout=1) + with pytest.raises(TimeoutError, match="still pending"): + m._publish(args, world, m._make_plan(args, world)) + assert world.prs[99]["auto_merge"] + + +def test_source_fetch_uses_reviewed_sha_when_no_local_checkout(world: World) -> None: + args = world.args(publish=True, flashinfer_repo=None) + m._publish(args, world, m._make_plan(args, world)) + assert any(args[0] == "fetch" and args[-1] == world.reviewed for args in world.git_calls) + + +def test_incomplete_local_worktree_is_preserved(world: World) -> None: + args = world.args(publish=True) + args.worktree.mkdir() + marker = args.worktree / "important.txt" + marker.write_text("preserve\n") + with pytest.raises((RuntimeError, AssertionError)): + m._publish(args, world, m._make_plan(args, world)) + assert marker.read_text() == "preserve\n" + + +def test_unpaired_promotion_publishes_and_resumes(world: World) -> None: + args = world.args( + publish=True, upstream_pr=[], unpaired_reason="No upstream PR for the DSL API migration." + ) + plan = m._make_plan(args, world) + change = plan.record["changes"][0] + assert change["upstream_pr"] is None + assert change["upstream_head"] is None + assert change["refresh_policy"] == "retain" + assert change["unpaired_reason"] == args.unpaired_reason + assert change["commits"] == [world.first, world.reviewed] + m._publish(args, world, plan) + m._publish(args, world, m._make_plan(args, world)) + assert len(world.prs) == 1 + assert "No paired FlashInfer PR" in world.prs[99]["body"] + assert not any(f"repos/{m._FLASHINFER}/pulls/" in call[1] for call in world.calls) + changed = copy.deepcopy(plan.record) + changed["changes"][0]["refresh_policy"] = "drop" + with pytest.raises(ValueError, match="reason/policy"): + m._adopt_record(plan, m._record_text(changed)) + + +@pytest.mark.parametrize( + "options", + [ + {"upstream_pr": [], "unpaired_reason": None}, + {"upstream_pr": [], "unpaired_reason": " "}, + {"upstream_pr": ["4829"], "unpaired_reason": "No pairing"}, + {"upstream_pr": [], "unpaired_reason": "No pairing", "map_upstream": ["abcdef0=4829"]}, + ], +) +def test_unpaired_options_require_explicit_unambiguous_reason(world: World, options: dict) -> None: + with pytest.raises(ValueError, match="unpaired-reason"): + m._make_plan(world.args(**options), world) + assert not world.calls + + +@pytest.mark.parametrize("same_fork", [False, True]) +def test_canonical_branch_name_on_developer_fork(world: World, same_fork: bool) -> None: + lock = world.consumer / m._LOCK + text = lock.read_text().replace("branch: tmp-fix", "branch: trtllm-prims-ts-dev") + if same_fork: + text = text.replace("github.com/dev/", "github.com/maintainer/") + lock.write_text(text) + _git(world.consumer, "add", m._LOCK) + _git(world.consumer, "commit", "--amend", "--no-edit", "-s") + world.main = world.merged = _git(world.consumer, "rev-parse", "HEAD") + args = world.args(publish=True) + if same_fork: + with pytest.raises(ValueError, match="temporary branch"): + m._make_plan(args, world) + return + plan = m._make_plan(args, world) + m._publish(args, world, plan) + actual = m.vendor._load_lock(args.worktree / m._LOCK).vendors[m._VENDOR].to_mapping() + assert actual == {**plan.reviewed.to_mapping(), "url": plan.previous.url} + + +def test_unpaired_cli_dry_run(world: World, capsys: pytest.CaptureFixture) -> None: + assert ( + m.main( + [ + "promote", + "--trtllm-pr", + "17", + "--canonical-repo", + "maintainer/flashinfer", + "--repo", + str(world.consumer), + "--unpaired-reason", + "No paired upstream PR.", + ] + ) + == 0 + ) + assert '"refresh_policy": "retain"' in capsys.readouterr().out + assert not any(args[0] == "push" for args in world.git_calls) + + +def test_previous_developer_fork_cannot_become_canonical( + world: World, monkeypatch: pytest.MonkeyPatch +) -> None: + original = m._lock_at + + def lock_at(gh: m.GitHub, repo: str, sha: str) -> m.vendor.Vendor: + entry = original(gh, repo, sha) + if sha == world.base: + return dataclasses.replace(entry, url="https://github.com/another-dev/flashinfer.git") + return entry + + monkeypatch.setattr(m, "_lock_at", lock_at) + with pytest.raises(ValueError, match="explicit canonical"): + m._make_plan(world.args(), world) + assert not any(call[0] == "push" for call in world.git_calls) + + +@pytest.mark.parametrize("branch", ["tmp-fix", "trtllm-prims-ts-dev-20260910"]) +def test_canonical_branch_must_be_explicit_and_match(world: World, branch: str) -> None: + with pytest.raises(ValueError, match="canonical"): + m._make_plan(world.args(canonical_branch=branch), world) + + +def test_completed_promotion_ignores_later_live_state( + world: World, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture +) -> None: + world.state = "CLEAN" + args = world.args(publish=True, auto_merge=True) + plan = m._make_plan(args, world) + m._publish(args, world, plan) + merged = world.main + assert f"Squash commit: {merged}" in capsys.readouterr().out + _git(world.consumer, "switch", "--detach", world.main) + _write(world.consumer, "later.txt", "later main commit\n") + world.main = _commit(world.consumer, "later main update") + world.overrides[f"repos/{m._TRTLLM}"] = {"allow_auto_merge": False} + world.overrides["repos/maintainer/flashinfer/git/ref/heads/trtllm-prims-ts-dev"] = { + "object": {"sha": "b" * 40} + } + world.overrides["repos/dev/flashinfer"] = {} + world.overrides[f"repos/{m._FLASHINFER}/pulls/4829"] = {"state": "closed", "merged": False} + original_compare = world.compare + + def compare(repo: str, base: str, head: str, **kwargs: object) -> dict: + assert repo == m._TRTLLM, "Completed retry must not query old source history" + return original_compare(repo, base, head, **kwargs) + + monkeypatch.setattr(world, "compare", compare) + world.calls.clear() + world.git_calls.clear() + args.flashinfer_repo = world.root / "missing-source" + resumed = m._make_plan(args, world) + assert resumed.completed_pr["merge_commit_sha"] == merged + assert resumed.record == plan.record + m._publish(args, world, resumed) + assert all(method == "GET" for method, _, _ in world.calls) + assert not any(call[0] in ("fetch", "push", "commit", "worktree") for call in world.git_calls) + assert len(world.comments) == 1 + + +@pytest.mark.parametrize("phase", ["worktree_created", "before_commit", "after_commit"]) +@pytest.mark.parametrize("main_advances", [False, True]) +def test_interrupted_preparation_resumes_without_extra_commits( + world: World, monkeypatch: pytest.MonkeyPatch, phase: str, main_advances: bool +) -> None: + def interrupted(repo: Path, *args: str, text: str | None = None) -> str: + if phase == "before_commit" and args[0] == "commit": + raise RuntimeError("simulated interruption") + result = world.routed_git(repo, *args, text=text) + if (phase == "worktree_created" and args[:2] == ("worktree", "add")) or ( + phase == "after_commit" and args[0] == "commit" + ): + raise RuntimeError("simulated interruption") + return result + + args = world.args(publish=True) + monkeypatch.setattr(m, "_git", interrupted) + with pytest.raises(RuntimeError, match="simulated interruption"): + m._publish(args, world, m._make_plan(args, world)) + assert not any(call[0] == "push" for call in world.git_calls) + monkeypatch.setattr(m, "_git", world.routed_git) + if main_advances: + _write(world.consumer, "later.txt", "later main commit\n") + world.main = _commit(world.consumer, "later main update") + pr = m._publish(args, world, m._make_plan(args, world)) + head = pr["head"]["sha"] + assert _git(world.consumer, "rev-list", "--count", f"{world.merged}..{head}") == "1" + assert sum(call[:2] == ("worktree", "add") for call in world.git_calls) == 1 + assert len(world.comments) == 1 + + +@pytest.mark.parametrize("edit", ["tracked", "untracked", "index_only", "index_only_other"]) +def test_incomplete_recovery_preserves_unexpected_edits( + world: World, monkeypatch: pytest.MonkeyPatch, edit: str +) -> None: + def interrupted(repo: Path, *args: str, text: str | None = None) -> str: + if args[0] == "commit": + raise RuntimeError("simulated interruption") + return world.routed_git(repo, *args, text=text) + + args = world.args(publish=True) + monkeypatch.setattr(m, "_git", interrupted) + with pytest.raises(RuntimeError, match="interruption"): + m._publish(args, world, m._make_plan(args, world)) + monkeypatch.setattr(m, "_git", world.routed_git) + if edit.startswith("index_only"): + file = m._LOCK if edit == "index_only" else "vendored/prims_ts/kernel.py" + target = args.worktree / file + original = target.read_text() + target.write_text( + original.replace(world.reviewed, world.previous) + if edit == "index_only" + else "preserve this staged edit\n" + ) + _git(args.worktree, "add", file) + target.write_text(original) + preserved = _git(args.worktree, "show", f":{file}") + else: + file = "vendored/prims_ts/kernel.py" if edit == "tracked" else "important.txt" + _write(args.worktree, file, "preserve this edit\n") + with pytest.raises(ValueError): + m._publish(args, world, m._make_plan(args, world)) + if edit.startswith("index_only"): + assert _git(args.worktree, "show", f":{file}") == preserved + else: + assert (args.worktree / file).read_text() == "preserve this edit\n" + assert not any(call[0] == "push" for call in world.git_calls) + assert _git(args.worktree, "rev-parse", "HEAD") == world.merged + + +def test_ci_skip_is_exact_once_and_precedes_auto_merge(world: World) -> None: + args = world.args(publish=True, auto_merge=True) + m._publish(args, world, m._make_plan(args, world)) + m._publish(args, world, m._make_plan(args, world)) + assert [comment["body"] for comment in world.comments] == [m._CI_SKIP_COMMAND] + create_index = next( + i + for i, (verb, route, _) in enumerate(world.calls) + if verb == "POST" and route.endswith("/pulls") + ) + comment_index = next( + i + for i, (verb, route, _) in enumerate(world.calls) + if verb == "POST" and route.endswith("/comments") + ) + auto_index = next( + i + for i, (_, route, payload) in enumerate(world.calls) + if route == "graphql" and "enablePullRequestAutoMerge" in payload["query"] + ) + assert create_index < comment_index < auto_index + + +@pytest.mark.parametrize("after", [False, True]) +def test_ci_comment_failure_is_resumable_without_duplicates(world: World, after: bool) -> None: + world.fail_comment_before = not after + world.fail_comment_after = after + args = world.args(publish=True, auto_merge=True) + with pytest.raises(RuntimeError, match="comment failure"): + m._publish(args, world, m._make_plan(args, world)) + assert len(world.prs) == 1 + assert not world.prs[99]["auto_merge"] + world.fail_comment_before = world.fail_comment_after = False + m._publish(args, world, m._make_plan(args, world)) + assert len(world.comments) == 1 + assert world.prs[99]["auto_merge"] + + +@pytest.mark.parametrize("same_actor", [False, True]) +def test_ci_comment_search_paginates_and_checks_author(world: World, same_actor: bool) -> None: + world.comments = [{"body": "unrelated", "user": {"login": "maintainer"}}] * 100 + world.comments.append( + { + "body": m._CI_SKIP_COMMAND, + "user": {"login": "maintainer" if same_actor else "someone-else"}, + } + ) + args = world.args(publish=True) + m._publish(args, world, m._make_plan(args, world)) + assert len(world.comments) == (101 if same_actor else 102) + assert any("comments?per_page=100&page=2" in route for _, route, _ in world.calls) + + +@pytest.mark.parametrize("change", ["closed", "head"]) +def test_ci_request_rejects_changed_pr(world: World, change: str) -> None: + world.fail_comment_before = True + args = world.args(publish=True) + plan = m._make_plan(args, world) + with pytest.raises(RuntimeError, match="comment failure"): + m._publish(args, world, plan) + verified = copy.deepcopy(world.prs[99]) + if change == "closed": + world.prs[99]["state"] = "closed" + else: + world.prs[99]["head"]["sha"] = "b" * 40 + world.calls.clear() + with pytest.raises(ValueError, match="closed or changed"): + m._ensure_ci_skip(world, plan, verified) + assert all(method == "GET" for method, _, _ in world.calls) + assert not world.comments + + +def test_generated_body_passes_repository_checklist(world: World) -> None: + checker = runpy.run_path(str(_SCRIPTS.parent / ".github/scripts/pr_checklist_check.py")) + plan = m._make_plan(world.args(), world) + body = m._pr_body(plan, f"{plan.title}\n\n{m._record_text(plan.record)}") + assert checker["find_unresolved_tasks"](body) == [] + assert checker["check_pr_checklist_section"](body) == (True, "") + assert "Required owner reviews and checks completed" not in body + assert m._CI_SKIP_COMMAND in body + assert "Automatic GitHub checks" in body + + +def test_mutating_git_commands_stream_output(monkeypatch: pytest.MonkeyPatch) -> None: + def run(command: list[str], **kwargs: object) -> subprocess.CompletedProcess: + assert kwargs["capture_output"] is False + assert kwargs["env"]["GIT_LFS_SKIP_SMUDGE"] == "1" + return subprocess.CompletedProcess(command, 0, None, None) + + monkeypatch.setattr(m.subprocess, "run", run) + assert m._git(Path("/test/repository"), "worktree", "add", "new") == "" + assert m._git(Path("/test/repository"), "commit", "-s", "-F", "-", text="test") == "" From 93bccecb3b6f76cfdc39c22c3323b5828d63e9c1 Mon Sep 17 00:00:00 2001 From: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> Date: Wed, 23 Sep 2026 08:39:35 +0000 Subject: [PATCH 02/11] [None][feat] add generic local vendor promotion bot Group vendoring and promotion tools under scripts/vendor while retaining compatibility entry points. Add credential-independent polling and daemon modes with private workdir state, author-owned upstream attribution, and restartable lock-only promotion with squash provenance. Signed-off-by: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> --- .github/CODEOWNERS | 1 + .pre-commit-config.yaml | 2 +- 3rdparty/vendor-sources.md | 249 +-- AGENTS.md | 26 +- scripts/maintain_prims_ts.py | 970 +--------- scripts/vendor/README.md | 227 +++ scripts/vendor/__init__.py | 3 + scripts/vendor/bot.py | 587 ++++++ scripts/vendor/manage.py | 1644 +++++++++++++++++ scripts/vendor/metadata.py | 302 +++ scripts/vendor/promote.py | 1105 +++++++++++ scripts/vendor_sources.py | 1642 +--------------- .../unittest/others/test_maintain_prims_ts.py | 177 +- tests/unittest/others/test_vendor_bot.py | 560 ++++++ tests/unittest/others/test_vendor_sources.py | 2 +- 15 files changed, 4644 insertions(+), 2853 deletions(-) create mode 100644 scripts/vendor/README.md create mode 100644 scripts/vendor/__init__.py create mode 100644 scripts/vendor/bot.py create mode 100644 scripts/vendor/manage.py create mode 100644 scripts/vendor/metadata.py create mode 100644 scripts/vendor/promote.py create mode 100644 tests/unittest/others/test_vendor_bot.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 77bc45f49e75..a72a658cf868 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -476,6 +476,7 @@ /requirements-openengine.txt @NVIDIA/trt-llm-oss-compliance /requirements.txt @NVIDIA/trt-llm-oss-compliance /scripts/vendor_sources.py @NVIDIA/trt-llm-infra-devs @NVIDIA/trt-llm-oss-compliance +/scripts/vendor/ @NVIDIA/trt-llm-infra-devs @NVIDIA/trt-llm-oss-compliance /setup.py @NVIDIA/trt-llm-oss-compliance /tensorrt_llm/_torch/attention/backends/prims_ts/** @NVIDIA/trt-llm-torch-attention-devs @NVIDIA/trt-llm-oss-compliance /tests/unittest/api_stability/ @NVIDIA/trt-llm-noncommitted-api-review-committee diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bac469873fd0..c93ef9233a39 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1597,7 +1597,7 @@ repos: pass_filenames: false - id: vendor-sources-check name: verify vendored sources are in sync - entry: python scripts/vendor_sources.py check + entry: python scripts/vendor/manage.py check language: python additional_dependencies: [PyYAML] always_run: true diff --git a/3rdparty/vendor-sources.md b/3rdparty/vendor-sources.md index a4ebfe140ac2..9fccaa06fee7 100644 --- a/3rdparty/vendor-sources.md +++ b/3rdparty/vendor-sources.md @@ -14,7 +14,7 @@ the selected files, any persistent compatibility patch and its content digest, and a digest of the materialized destination. A short branch or tag may explain where the commit came from, but the full commit is authoritative. -Use `scripts/vendor_sources.py` for every lock or vendor-state change. Do not +Use `scripts/vendor/manage.py` for every lock or vendor-state change. Do not edit the YAML, generated patches, or digests by hand. All examples below use the default lock. For an isolated test or another consumer repository, place `--lock PATH` before the subcommand. @@ -91,10 +91,10 @@ existing compatibility patch, then use `pin`. List entries, or run the offline integrity status for all or one vendor: ```bash -python scripts/vendor_sources.py list -python scripts/vendor_sources.py status -python scripts/vendor_sources.py status VENDOR -python scripts/vendor_sources.py check VENDOR +python scripts/vendor/manage.py list +python scripts/vendor/manage.py status +python scripts/vendor/manage.py status VENDOR +python scripts/vendor/manage.py check VENDOR ``` `status` and the default `check` exit unsuccessfully if the destination has a @@ -107,7 +107,7 @@ When neither the lock entry nor destination exists, create both from an immutable commit and a local upstream checkout: ```bash -python scripts/vendor_sources.py create VENDOR \ +python scripts/vendor/manage.py create VENDOR \ --url https://example.com/organization/repository.git \ --branch main \ --commit FULL_COMMIT \ @@ -124,7 +124,7 @@ If the destination already exists but has no lock entry, adopt it. Use `exact` to require an exact upstream match: ```bash -python scripts/vendor_sources.py create VENDOR \ +python scripts/vendor/manage.py create VENDOR \ --url https://example.com/organization/repository.git \ --commit FULL_COMMIT \ --source path/in/upstream \ @@ -143,7 +143,7 @@ Discard destination edits and reproduce the currently locked upstream commit plus its persistent patch: ```bash -python scripts/vendor_sources.py sync VENDOR --repo /path/to/upstream +python scripts/vendor/manage.py sync VENDOR --repo /path/to/upstream ``` This overwrites the selected destination files. It does not update the lock, @@ -155,21 +155,21 @@ After editing an exact destination for a change that must remain downstream, create its persistent patch: ```bash -python scripts/vendor_sources.py patch VENDOR create --repo /path/to/upstream +python scripts/vendor/manage.py patch VENDOR create --repo /path/to/upstream ``` After intentionally changing an already patched destination, regenerate the patch: ```bash -python scripts/vendor_sources.py patch VENDOR refresh --repo /path/to/upstream +python scripts/vendor/manage.py patch VENDOR refresh --repo /path/to/upstream ``` Drop a no-longer-needed patch only after the destination exactly matches the currently locked upstream selection: ```bash -python scripts/vendor_sources.py patch VENDOR drop --repo /path/to/upstream +python scripts/vendor/manage.py patch VENDOR drop --repo /path/to/upstream ``` Generated patches live under `3rdparty/vendor_patches/`. Review them, but update @@ -184,7 +184,7 @@ branch at the currently locked commit **before** exporting: ```bash git -C /path/to/upstream switch -c trtllm-vendor-fix LOCKED_FULL_COMMIT -python scripts/vendor_sources.py export VENDOR --repo /path/to/upstream +python scripts/vendor/manage.py export VENDOR --repo /path/to/upstream ``` The upstream checkout's selected source must be clean before export and its @@ -205,7 +205,7 @@ git -C /path/to/upstream push -u origin trtllm-vendor-fix Finally, pin the committed revision from that checkout: ```bash -python scripts/vendor_sources.py pin VENDOR \ +python scripts/vendor/manage.py pin VENDOR \ --url https://example.com/my-fork/repository.git \ --branch trtllm-vendor-fix \ --commit NEW_FULL_COMMIT \ @@ -236,7 +236,7 @@ Remove a lock entry and its generated compatibility patch while preserving the destination: ```bash -python scripts/vendor_sources.py remove VENDOR +python scripts/vendor/manage.py remove VENDOR ``` The preserved destination is no longer protected by the lock. Delete or move @@ -247,8 +247,8 @@ it separately as part of the reviewed migration that removes the vendor. The default check is deliberately offline: ```bash -python scripts/vendor_sources.py check -python scripts/vendor_sources.py check --offline +python scripts/vendor/manage.py check +python scripts/vendor/manage.py check --offline ``` It validates the lock schema and path safety, patch metadata, and the checked-in @@ -260,7 +260,7 @@ When network access is available, attempt verification against every recorded upstream: ```bash -python scripts/vendor_sources.py check --upstream +python scripts/vendor/manage.py check --upstream ``` An inaccessible repository is reported as unavailable rather than failing. If @@ -268,14 +268,14 @@ a commit can be obtained, a source, patch, or destination mismatch is an error. Trusted maintainer CI can require access to every source: ```bash -python scripts/vendor_sources.py check --upstream --require-access +python scripts/vendor/manage.py check --upstream --require-access ``` To verify one vendor against an existing checkout without contacting the recorded URL, provide it explicitly: ```bash -python scripts/vendor_sources.py check VENDOR --repo /path/to/upstream +python scripts/vendor/manage.py check VENDOR --repo /path/to/upstream ``` The checkout's configured remote may differ from the lock URL; it only needs to @@ -289,203 +289,22 @@ local repository, and URL or commit changes require vendor CODEOWNER review. Never put credentials in a lock URL. Run checks that use internal credentials only in a trusted environment, not with pull-request-controlled scripts. -## Promote a reviewed PrimTS revision +## Promotion and local monitoring -`scripts/maintain_prims_ts.py promote` automates the maintainer follow-up after -a TRT-LLM PR has merged a PrimTS code update from a temporary FlashInfer branch. -It does not import new kernel code or implement periodic upstream refreshes. -Use a trusted checkout of this tool; do not execute a PR-supplied copy with -maintainer credentials. +The generic toolkit under `scripts/vendor/` supports every vendor key in this +lock. `promote.py` prepares a verified lock-only follow-up after a source-update +PR merges; `bot.py` can monitor PRs locally, request missing contributor metadata, +match upstream commits, and publish promotions with squash auto-merge. -### Maintenance workflow +See the [toolkit guide](../scripts/vendor/README.md) for the workflow diagram, +contributor description format, author resolution of ambiguous matches, +authentication, configurable private workdirs, daemon launch/stop, and recovery. +No GitHub App or systemd setup is required. All remote writes are opt-in with +`--publish`; operator settings and credentials are not committed to this repo. -The source-update PR reviews code changes; the promotion PR only restores the -canonical lock location. The diagram shows publication with `--auto-merge`; -the CLI remains read-only unless `--publish` is supplied. - -```mermaid -flowchart TD - canonical["Canonical FlashInfer dev branch"] - temporary["Developer temporary branch
PrimTS code changes"] - source_pr["TRT-LLM source-update PR
vendor code and pin temporary SHA"] - pairing["Paired FlashInfer PR
or explicit unpaired reason"] - verify["promote: verify merged PR, immutable SHA,
canonical target and fast-forward range"] - - canonical -->|fork| temporary - temporary --> source_pr - source_pr -->|merged| verify - pairing --> verify - - subgraph publish["Maintainer: --publish --auto-merge"] - prepare["Verify source materialization
commit lock-only change with provenance and DCO"] - advance["Fast-forward canonical branch
to the reviewed source SHA"] - followup["Create or reuse lock-only TRT-LLM PR
same source SHA, patch, digests and files"] - merge_request["Post /bot skip for no code change
enable squash auto-merge with original message"] - prepare --> advance --> followup --> merge_request - end - - verify -->|publish requested| prepare - merge_request -->|required checks and reviews| merged["Follow-up merged
provenance preserved in squash commit"] - merged --> rebase["Other developers rebase their temporary branches
onto the promoted canonical revision"] - rebase --> temporary - merged -.-> refresh["Periodic refresh: manual / future tooling
new upstream main plus outstanding changes
new dated canonical branch"] - refresh -.-> refresh_pr["TRT-LLM refresh PR merged
all developers rebase to the dated branch"] - refresh_pr -.-> canonical -``` - -Solid arrows show the development/promotion cycle; dotted arrows show periodic -refresh, which this command does not implement. A paired upstream PR can merge -before or after the TRT-LLM source-update PR. Recording its URL and observed -commits preserves provenance; it is not permission to drop a change merely -because the upstream PR later merges. Unpaired changes explicitly retain their -reason and `refresh_policy: retain`. - -After a promotion follow-up merges, rebase both the pending TRT-LLM PR onto -latest main and its temporary FlashInfer branch onto the promoted canonical -revision before refreshing its vendor pin. For a periodic refresh, the -maintainer identifies already-integrated changes, carries forward outstanding -changes on newer FlashInfer main, and publishes a dated canonical branch and a -separate TRT-LLM refresh PR. That refresh may change vendor code and needs its -own validation; the promotion's no-code-change CI request does not apply. - -If validation fails or another maintainer advances the relevant pins, stop and -inspect the reported state. Retrying an interrupted promotion reuses verified -worktrees, commits, PRs, and CI comments; it does not overwrite unrelated work. - -### Plan and publish a promotion - -Start with a dry-run (GitHub reads only): - -```bash -python scripts/maintain_prims_ts.py promote \ - --trtllm-pr "$MERGED_TRTLLM_PR" \ - --canonical-repo yuxianq/flashinfer \ - --upstream-pr https://github.com/flashinfer-ai/flashinfer/pull/4829 -``` - -Normally an upstream PR is required and may be open, draft, or merged; it must -target FlashInfer `main`. Always specify the maintainer's canonical FlashInfer -fork with `--canonical-repo OWNER/REPO`; the tool never infers the destination -fork from a previous lock that might still point to a developer's fork. -`--canonical-branch` defaults to `trtllm-prims-ts-dev`; specify it explicitly -for a dated `trtllm-prims-ts-dev-YYYYMMDD` branch. Both must match the lock in -the parent of the TRT-LLM merge commit. The tool derives the old canonical -commit from that parent and the reviewed source SHA from the merge commit's -lock. It never promotes the temporary branch's current tip. Source changes -must form a linear, fast-forward range. Finish the preceding promotion before -merging another vendor update. A developer fork may use the same branch name -as the canonical fork: the repository URL is also part of its identity. - -For an existing change with no paired upstream PR, explicitly use -`--unpaired-reason` instead of `--upstream-pr`: - -```bash -python scripts/maintain_prims_ts.py promote \ - --trtllm-pr 18808 \ - --canonical-repo yuxianq/flashinfer \ - --unpaired-reason "No paired FlashInfer PR for the CUTLASS DSL 4.8 API migration." -``` - -This records the source commits with `upstream_pr: null`, the reason, and -`refresh_policy: retain` in the commit message. A future refresh must retain -these changes until an upstream pairing or equivalence is explicitly -established; missing PR metadata must never be interpreted as permission to -drop them. This option cannot be combined with PR mappings. It does not change -the fast-forward, immutable-pin, or lock-only requirements. - -One upstream PR assigns the whole newly promoted range to that PR. For multiple -PRs, repeat `--upstream-pr` and provide `--map-upstream COMMIT=PR` for each source -commit (full SHA or unambiguous prefix of at least seven characters). A terminal -invocation prompts for missing assignments; noninteractive use fails instead -of guessing. This mapping is a maintainer assertion of provenance, not proof -that a later upstream revision preserves the change's behavior. - -To publish the promotion and enable squash auto-merge for its follow-up PR: - -```bash -python scripts/maintain_prims_ts.py promote \ - --trtllm-pr "$MERGED_TRTLLM_PR" \ - --canonical-repo yuxianq/flashinfer \ - --upstream-pr https://github.com/flashinfer-ai/flashinfer/pull/4829 \ - --flashinfer-repo /path/to/flashinfer \ - --publish --auto-merge --wait -``` - -`--flashinfer-repo` is optional. If supplied, it must contain the previous and -reviewed immutable source commits; neither its checked-out branch nor dirty -files are used. Otherwise the tool fetches the reviewed source into a temporary -repository. `--repo` selects the local TRT-LLM repository. The personal TRT-LLM -fork defaults to its `fork` remote; use `--fork OWNER/REPO` to override it. -`GH_CONFIG_DIR` is respected, defaulting to `~/.config/gh`. - -Publishing performs the following checks and actions: - -1. Verify that current TRT-LLM main still contains the reviewed vendor entry and - the canonical branch has not been superseded. -2. Create a separate follow-up worktree, verify source materialization, and use - `vendor_sources.py pin` to change only the PrimTS lock URL/branch. The commit, - compatibility patch, digest, selected source, and destination stay unchanged. -3. Make a signed-off commit with the maintainer's Git identity. Its versioned - `PRIMTS PROMOTION V1` JSON block records both source SHAs, upstream baseline, - originating TRT-LLM PR/merge, and upstream PR/commit mappings and snapshots. -4. Fast-forward the canonical FlashInfer branch with a non-force push, push the - lock-only commit to the TRT-LLM personal fork, and create the follow-up PR. -5. Reverify the open PR's lock-only commit and post - `/bot skip --comment "skip CI since no code change"` to request the bot's - no-code-change CI path. PR creation can also start automatic GitHub checks; - this command does not disable those checks or waive required owner reviews. -6. With `--auto-merge`, supply the signed-off commit's title/body as the final squash - message and verify GitHub retained them. Required checks and reviews are not - bypassed. If already eligible, the PR may merge immediately. Merge queues are - currently rejected because this workflow requires a custom squash message. - -The generated PR checklist contains only assertions the tool has verified, not -an unchecked placeholder for future approvals or checks. Those remain enforced -by GitHub. The tool does not delete branches or rewrite history. Without -`--auto-merge`, the PR description includes the required squash message for -manual merging. Preserve that complete message: it is the durable provenance -record, with no extra tracking file. Initial history predating these records -will still need classification when a future refresh tool is introduced. - -`--wait` verifies the final merged commit, polling for up to `--timeout` seconds -(default 3600). Without it, the tool verifies the saved auto-merge request and -returns; there is no background monitor. If GitHub merges immediately, the tool -verifies and prints the final squash SHA. Do not change the follow-up branch -after requesting the CI skip or enabling auto-merge. A head check at enablement -does not permanently freeze the branch. - -Publication spans two repositories and is not atomic. Completed pushes and PRs -are preserved if a later step fails. Repeating the same invocation recognizes -the deterministic follow-up branch/PR, verifies its lock-only change and -signed-off metadata, and resumes without creating duplicate PRs. Before posting -the CI command, retries search all PR comments for the identical command from -the authenticated user, including when a previous posting request timed out -after succeeding. A posting failure stops before enabling auto-merge. Serialize -maintainer invocations; comment deduplication is not a cross-process lock. - -Interrupted worktree preparation can resume if the branch and repository match, -with either an unchanged base, the expected lock-only edit (staged or unstaged), -or a verified promotion commit. Unexpected tracked, staged, or untracked edits -are preserved and require inspection instead of automatic overwrite. Existing -upstream snapshots are preserved even if their PR heads have since advanced; -different mappings on an unfinished promotion are rejected. After a promotion -merges, reruns verify its historical squash commit and recorded provenance and -return without remote writes, even if current main or canonical branches have -advanced or the original temporary fork disappeared. The completed record is -authoritative; rerunning does not revise its provenance. - -`--worktree PATH` selects the follow-up worktree; by default it is a sibling of -`--repo` named `prims-ts-promote--`. Git checkout skips LFS -asset downloads for this lock-only worktree without changing repository-wide -configuration. Fetch, checkout, commit, and push progress is streamed. - -Hermetic CPU-only validation, without loading TRT-LLM's GPU test configuration: - -```bash -PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 pytest --noconftest -c /dev/null \ - -p no:cacheprovider -o 'markers=cpu_only: CPU-only test' \ - tests/unittest/others/test_maintain_prims_ts.py -``` +The old `scripts/vendor_sources.py` and `scripts/maintain_prims_ts.py` paths are +compatibility entry points. Existing PrimTS promotions can still be resumed +through their original command and branch naming. ## License and attribution @@ -508,6 +327,6 @@ files remain exact upstream copies. Use the normal commands with `flashinfer-prims-ts`, for example: ```bash -python scripts/vendor_sources.py status flashinfer-prims-ts -python scripts/vendor_sources.py check flashinfer-prims-ts +python scripts/vendor/manage.py status flashinfer-prims-ts +python scripts/vendor/manage.py check flashinfer-prims-ts ``` diff --git a/AGENTS.md b/AGENTS.md index cbfe22c8ab96..7a25d28964d3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -193,17 +193,21 @@ The bot-to-bot semantic review workflow is disabled. See [.github/semantic-review.md](.github/semantic-review.md) for its current status and historical implementation details. -### PrimTS vendor promotion - -Use `scripts/maintain_prims_ts.py promote` for the lock-only maintainer follow-up -after a PrimTS source-update PR merges. It defaults to dry-run; publication and -auto-merge require explicit flags. For maintainer promotions, publish with -`--publish --auto-merge`. Specify the maintainer fork explicitly with -`--canonical-repo OWNER/REPO`. Publication posts -`/bot skip --comment "skip CI since no code change"` after verifying the lock-only -PR, before enabling optional auto-merge. See `3rdparty/vendor-sources.md`. -Preserve its structured promotion metadata and DCO sign-off in the final squash -message. +### Vendor source tools and promotion + +Use `scripts/vendor/manage.py` for lock/vendor-state changes and +`scripts/vendor/promote.py promote` for lock-only promotion after a source-update +PR merges. Select the vendor, upstream, and canonical fork/branch explicitly. +Manual publication uses `--publish --auto-merge`; dry-run is the default. +`scripts/vendor/bot.py run` provides an optional local monitor with `--workdir`, +`--daemon`, and opt-in `--publish`. Do not launch a publishing daemon unless the +user requested ongoing automation. Never commit operator credentials or runtime +state. The bot asks the PR author to correct metadata or ambiguous attribution; +it never approves code merely because metadata passes. The old script paths +remain compatibility entry points. See `scripts/vendor/README.md` and +`3rdparty/vendor-sources.md`. Preserve structured promotion provenance and DCO +sign-off in the final squash message. CI skip applies only to verified lock-only +promotions, not changes to these tools. ### Triggering CI diff --git a/scripts/maintain_prims_ts.py b/scripts/maintain_prims_ts.py index 16d3fbb4d5e1..15ee285fbf38 100644 --- a/scripts/maintain_prims_ts.py +++ b/scripts/maintain_prims_ts.py @@ -1,965 +1,25 @@ #!/usr/bin/env python3 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Promote reviewed PrimTS source revisions; dry-run unless --publish is supplied. +"""Compatibility entry point for the original PrimTS promotion command.""" -Only Git, gh, and PyYAML are required. GitHub reads and temporary source fetches -are allowed in dry-run; remote writes and consumer worktrees require --publish. -Provenance lives in signed-off commits, not in a separate tracking file. -""" - -from __future__ import annotations - -import argparse -import base64 -import contextlib -import json -import os -import re -import subprocess import sys -import tempfile -import time -from collections.abc import Iterator -from dataclasses import dataclass -from pathlib import Path -from urllib.parse import urlencode - -import yaml if __package__: - from . import vendor_sources as vendor + from .vendor.promote import main else: - import vendor_sources as vendor - -_TRTLLM = "NVIDIA/TensorRT-LLM" -_FLASHINFER = "flashinfer-ai/flashinfer" -_VENDOR = "flashinfer-prims-ts" -_LOCK = "3rdparty/vendor_sources.lock.yaml" -_BEGIN = "----- BEGIN PRIMTS PROMOTION V1 -----" -_END = "----- END PRIMTS PROMOTION V1 -----" -_SHA = re.compile(r"[0-9a-f]{40}\Z") -_CANONICAL = re.compile(r"trtllm-prims-ts-dev(?:-[0-9]{8})?\Z") -_CI_SKIP_COMMAND = '/bot skip --comment "skip CI since no code change"' - - -def _run( - command: list[str], - *, - cwd: Path | None = None, - text: str | None = None, - timeout: int = 300, - stream: bool = False, -) -> str: - # Do not use vendor_sources._run_git here: its synthetic identity is for - # patch generation. Real promotion commits must use the maintainer's identity. - env = os.environ.copy() - for key in vendor._GIT_LOCAL_ENVIRONMENT_VARIABLES: - env.pop(key, None) - env.setdefault("GH_CONFIG_DIR", str(Path.home() / ".config/gh")) - env.update(GIT_TERMINAL_PROMPT="0", GH_PROMPT_DISABLED="1", GIT_NO_REPLACE_OBJECTS="1") - if command[0] == "git": - # Lock-only worktrees need source text, not unrelated LFS assets. - env["GIT_LFS_SKIP_SMUDGE"] = "1" - result = subprocess.run( - command, - cwd=cwd, - env=env, - input=text, - text=True, - capture_output=not stream, - timeout=timeout, - ) - if result.returncode: - detail = (result.stderr or result.stdout or "See streamed output above.").strip() - raise RuntimeError(f"{command[0]} failed: {detail}") - return (result.stdout or "").strip() - - -def _git(repo: Path, *args: str, text: str | None = None) -> str: - return _run( - ["git", "-C", str(repo), *args], - text=text, - timeout=1800 if args[:2] == ("worktree", "add") else 300, - stream=args[:2] == ("worktree", "add") or args[0] in ("fetch", "commit", "push"), - ) - - -class GitHub: - """Small gh-backed client; respects GH_CONFIG_DIR without handling tokens.""" - - def api( - self, endpoint: str, *, method: str = "GET", payload: dict | None = None - ) -> dict | list: - command = ["gh", "api", "--hostname", "github.com", "--method", method, endpoint] - if payload is not None: - command += ["--input", "-"] - result = json.loads(_run(command, text=None if payload is None else json.dumps(payload))) - if not isinstance(result, (dict, list)): - raise ValueError(f"Unexpected GitHub response for {endpoint}") - return result - - def get(self, endpoint: str) -> dict: - result = self.api(endpoint) - if not isinstance(result, dict): - raise ValueError(f"Expected an object from {endpoint}") - return result - - def optional(self, endpoint: str) -> dict | None: - try: - return self.get(endpoint) - except RuntimeError as error: - if "HTTP 404" not in str(error): - raise - return None - - def compare(self, repo: str, base: str, head: str, *, all_commits: bool = False) -> dict: - first = self.get(f"repos/{repo}/compare/{base}...{head}?per_page=100&page=1") - if all_commits: - count = int(first["total_commits"]) - for page in range(2, (count + 99) // 100 + 1): - more = self.get(f"repos/{repo}/compare/{base}...{head}?per_page=100&page={page}") - first["commits"].extend(more["commits"]) - if len(first["commits"]) != count: - raise ValueError("Incomplete GitHub comparison; refusing partial provenance.") - return first - - -def _sha(value: str) -> str: - if not isinstance(value, str) or not _SHA.fullmatch(value): - raise ValueError(f"Expected a full immutable commit SHA, got {value!r}.") - return value - - -def _repo_name(url: str) -> str: - match = re.fullmatch( - r"(?:https://github\.com/|git@github\.com:)([\w.-]+/[\w.-]+?)(?:\.git)?/?", url - ) - if not match: - raise ValueError(f"Expected a credential-free GitHub repository URL, got {url!r}.") - return match[1] - - -def _pr_number(value: str, repo: str) -> int: - for prefix in (f"https://github.com/{repo}/pull/", f"{repo}#"): - if value.startswith(prefix): - value = value[len(prefix) :] - break - if not re.fullmatch(r"[1-9][0-9]*", value): - raise ValueError(f"Expected a PR number or PR URL in {repo}: {value!r}.") - return int(value) - - -def _lock_document_at(gh: GitHub, repo: str, commit: str) -> dict: - response = gh.get(f"repos/{repo}/contents/{_LOCK}?ref={_sha(commit)}") - if response.get("encoding") != "base64": - raise ValueError("GitHub did not return the complete vendor lock.") - data = yaml.load(base64.b64decode(response["content"]), Loader=vendor._UniqueKeyLoader) - if not isinstance(data, dict) or data.get("schema_version") != 1: - raise ValueError("Unsupported vendor lock schema.") - return data - - -def _lock_at(gh: GitHub, repo: str, commit: str) -> vendor.Vendor: - data = _lock_document_at(gh, repo, commit) - return vendor._validate_vendor(_VENDOR, data["vendors"][_VENDOR]) - - -def _main_sha(gh: GitHub) -> str: - return _sha(gh.get(f"repos/{_TRTLLM}/git/ref/heads/main")["object"]["sha"]) - - -def _check_fork(gh: GitHub, repo: str, upstream: str) -> None: - info = gh.get(f"repos/{repo}") - if ( - repo.lower() == upstream.lower() - or info.get("source", {}).get("full_name", "").lower() != upstream.lower() - ): - raise ValueError(f"{repo} must be a personal fork of {upstream}, not upstream itself.") - - -def _record_from_message(message: str) -> dict: - if message.count(_BEGIN) != 1 or message.count(_END) != 1: - raise ValueError("Missing or ambiguous PrimTS promotion metadata in commit message.") - record = json.loads(message.split(_BEGIN, 1)[1].split(_END, 1)[0]) - if not isinstance(record, dict) or record.get("version") != 1: - raise ValueError("Unsupported PrimTS promotion metadata version.") - return record - - -def _record_text(record: dict) -> str: - return f"{_BEGIN}\n{json.dumps(record, indent=2, sort_keys=True)}\n{_END}" - - -def _map_commits(commits: list[dict], prs: dict[int, dict], mappings: list[str]) -> dict[str, int]: - assignments: dict[str, int] = {} - for mapping in mappings: - prefix, separator, reference = mapping.partition("=") - matches = [item["sha"] for item in commits if item["sha"].startswith(prefix)] - if not separator or not re.fullmatch(r"[0-9a-f]{7,40}", prefix) or len(matches) != 1: - raise ValueError(f"Ambiguous or invalid --map-upstream {mapping!r}.") - number = _pr_number(reference, _FLASHINFER) - if number not in prs or matches[0] in assignments: - raise ValueError(f"Duplicate mapping or PR missing from --upstream-pr: {mapping!r}.") - assignments[matches[0]] = number - for item in commits: - sha = item["sha"] - if sha in assignments: - continue - if len(prs) == 1: - assignments[sha] = next(iter(prs)) - elif sys.stdin.isatty(): - title = item["commit"]["message"].splitlines()[0] - answer = input(f"Upstream PR for {sha[:12]} ({title}), choose {sorted(prs)}: ") - number = _pr_number(answer, _FLASHINFER) - if number not in prs: - raise ValueError("Answer must identify a supplied upstream PR.") - assignments[sha] = number - else: - raise ValueError(f"Provide --map-upstream {sha}=PR for every unmapped commit.") - if set(assignments.values()) != set(prs): - raise ValueError("Every supplied upstream PR must map to at least one new source commit.") - return assignments - - -@dataclass -class Promotion: - """An immutable source revision and the expected consumer/canonical states.""" - - number: int - main_sha: str - previous: vendor.Vendor - reviewed: vendor.Vendor - canonical_repo: str - fork: str - record: dict - completed_pr: dict | None = None - - @property - def branch(self) -> str: - return f"chore/prims-ts-promote-{self.number}-{self.reviewed.commit[:12]}" - - @property - def title(self) -> str: - return f"[None][chore] promote PrimTS source from TRT-LLM #{self.number}" - - @property - def promoted(self) -> dict: - result = self.reviewed.to_mapping() - result.update(url=self.previous.url, branch=self.previous.branch) - return result - - -def _check_current(gh: GitHub, plan: Promotion) -> None: - current = _lock_at(gh, _TRTLLM, _main_sha(gh)).to_mapping() - if current not in (plan.reviewed.to_mapping(), plan.promoted): - raise ValueError("TRT-LLM main's vendor lock has been superseded; do not promote this PR.") - ref = gh.get(f"repos/{plan.canonical_repo}/git/ref/heads/{plan.previous.branch}") - if ref["object"]["sha"] not in (plan.previous.commit, plan.reviewed.commit): - raise ValueError( - "Canonical branch moved unexpectedly; coordinate with the other maintainer." - ) - - -def _make_plan(args: argparse.Namespace, gh: GitHub) -> Promotion: - if args.unpaired_reason is not None: - if not args.unpaired_reason.strip() or args.upstream_pr or args.map_upstream: - raise ValueError( - "--unpaired-reason requires a nonempty reason and cannot be combined " - "with --upstream-pr or --map-upstream." - ) - elif not args.upstream_pr: - raise ValueError("Provide --upstream-pr or an explicit --unpaired-reason.") - number = _pr_number(args.trtllm_pr, _TRTLLM) - pr = gh.get(f"repos/{_TRTLLM}/pulls/{number}") - if not pr.get("merged") or pr["base"]["ref"] != "main": - raise ValueError("--trtllm-pr must identify a PR already merged into TRT-LLM main.") - merge_sha = _sha(pr["merge_commit_sha"]) - merge = gh.get(f"repos/{_TRTLLM}/commits/{merge_sha}") - previous = _lock_at(gh, _TRTLLM, merge["parents"][0]["sha"]) - reviewed = _lock_at(gh, _TRTLLM, merge_sha) - canonical_repo = _repo_name(previous.url) - source_repo = _repo_name(reviewed.url) - expected_repo = _repo_name(f"https://github.com/{args.canonical_repo}") - if not _CANONICAL.fullmatch(args.canonical_branch): - raise ValueError("--canonical-branch must name a supported canonical branch.") - if canonical_repo.lower() != expected_repo.lower() or previous.branch != args.canonical_branch: - raise ValueError( - "Previous lock does not match the explicit canonical repository/branch; " - "finish the preceding promotion first." - ) - if ( - previous.commit == reviewed.commit - or not reviewed.branch - or (canonical_repo.lower() == source_repo.lower() and _CANONICAL.fullmatch(reviewed.branch)) - ): - raise ValueError("The PR must introduce a new source SHA on a temporary branch.") - fork = args.fork or _repo_name(_git(args.repo, "remote", "get-url", "fork")) - record = { - "version": 1, - "kind": "promotion", - "vendor": _VENDOR, - "trtllm_pr": f"https://github.com/{_TRTLLM}/pull/{number}", - "trtllm_merge": merge_sha, - "canonical_url": previous.url, - "canonical_branch": previous.branch, - "previous": previous.commit, - "promoted": reviewed.commit, - "source_url": reviewed.url, - "source_branch": reviewed.branch, - } - plan = Promotion(number, _main_sha(gh), previous, reviewed, canonical_repo, fork, record) - # A completed operation is historical: later promotions, deleted temporary - # forks, or changed upstream PR state must not make it run again or fail. - existing = _find_followup(gh, plan) - if existing: - existing = gh.get(f"repos/{_TRTLLM}/pulls/{existing['number']}") - if existing.get("merged"): - _verify_remote_commit(gh, plan, _TRTLLM, existing["merge_commit_sha"]) - plan.completed_pr = existing - return plan - _check_fork(gh, canonical_repo, _FLASHINFER) - _check_fork(gh, source_repo, _FLASHINFER) - _check_fork(gh, fork, _TRTLLM) - comparison = gh.compare(canonical_repo, previous.commit, reviewed.commit, all_commits=True) - if comparison["merge_base_commit"]["sha"] != previous.commit: - raise ValueError( - "Reviewed source is not a fast-forward of the previous canonical revision." - ) - commits = comparison["commits"] - if not commits or any(len(item["parents"]) != 1 for item in commits): - raise ValueError( - "Promote requires a nonempty, linear source commit range; rebase merges first." - ) - prs = {} - for reference in args.upstream_pr: - upstream_number = _pr_number(reference, _FLASHINFER) - if upstream_number in prs: - raise ValueError("Duplicate --upstream-pr.") - upstream = gh.get(f"repos/{_FLASHINFER}/pulls/{upstream_number}") - if upstream["base"]["ref"] != "main" or ( - upstream["state"] == "closed" and not upstream.get("merged") - ): - raise ValueError("Upstream PR must target main and be open (draft allowed) or merged.") - prs[upstream_number] = upstream - assignments = _map_commits(commits, prs, args.map_upstream) if prs else {} - upstream_main = _sha(gh.get(f"repos/{_FLASHINFER}/git/ref/heads/main")["object"]["sha"]) - upstream_base = gh.compare(canonical_repo, previous.commit, upstream_main)["merge_base_commit"][ - "sha" - ] - groups = [] - if args.unpaired_reason is not None: - groups.append( - { - "commits": [item["sha"] for item in commits], - "upstream_pr": None, - "upstream_base": None, - "upstream_head": None, - "upstream_merge": None, - "unpaired_reason": args.unpaired_reason.strip(), - "refresh_policy": "retain", - } - ) - for upstream_number, upstream in prs.items(): - groups.append( - { - "commits": [ - item["sha"] for item in commits if assignments[item["sha"]] == upstream_number - ], - "upstream_pr": f"https://github.com/{_FLASHINFER}/pull/{upstream_number}", - "upstream_base": _sha(upstream["base"]["sha"]), - "upstream_head": _sha(upstream["head"]["sha"]), - "upstream_merge": upstream.get("merge_commit_sha") - if upstream.get("merged") - else None, - } - ) - record.update( - { - "upstream_base": _sha(upstream_base), - "upstream_observed_main": upstream_main, - "changes": groups, - } - ) - _check_current(gh, plan) - return plan - - -def _find_followup(gh: GitHub, plan: Promotion) -> dict | None: - query = urlencode( - {"state": "all", "base": "main", "head": f"{plan.fork.split('/')[0]}:{plan.branch}"} - ) - prs = gh.api(f"repos/{_TRTLLM}/pulls?{query}") - if not isinstance(prs, list) or len(prs) > 1: - raise ValueError("Ambiguous existing promotion PRs.") - if prs and prs[0]["state"] == "closed" and not prs[0].get("merged_at"): - raise ValueError("The promotion PR was closed without merging; inspect it before retrying.") - return prs[0] if prs else None - - -def _adopt_record(plan: Promotion, message: str) -> None: - """Keep the original upstream snapshot when resuming an already-published run.""" - existing = _record_from_message(message) - stable_keys = set(plan.record) - {"upstream_base", "upstream_observed_main", "changes"} - if any(existing.get(key) != plan.record[key] for key in stable_keys): - raise ValueError("Existing promotion metadata does not describe this source update.") - _validate_snapshot(existing) - if "changes" not in plan.record: - # Only the read-only, already-merged path restores an entire snapshot. - plan.record = existing - return - expected = { - group["upstream_pr"]: ( - group["commits"], - group.get("unpaired_reason"), - group.get("refresh_policy"), - ) - for group in plan.record["changes"] - } - actual = { - group["upstream_pr"]: ( - group["commits"], - group.get("unpaired_reason"), - group.get("refresh_policy"), - ) - for group in existing["changes"] - } - if actual != expected: - raise ValueError("Existing promotion has different upstream PR/commit mappings.") - plan.record = existing - - -def _validate_snapshot(record: dict) -> None: - _sha(record["upstream_base"]) - _sha(record["upstream_observed_main"]) - commits = [] - if not isinstance(record["changes"], list) or not record["changes"]: - raise ValueError("Promotion metadata must contain source changes.") - for group in record["changes"]: - if not isinstance(group["commits"], list) or not group["commits"]: - raise ValueError("Promotion change group has no commits.") - commits.extend(_sha(sha) for sha in group["commits"]) - if group["upstream_pr"] is None: - if ( - group.get("refresh_policy") != "retain" - or not group.get("unpaired_reason", "").strip() - ): - raise ValueError( - "Unpaired promotion changes must retain an explicit reason/policy." - ) - else: - _pr_number(group["upstream_pr"], _FLASHINFER) - _sha(group["upstream_base"]) - _sha(group["upstream_head"]) - if group["upstream_merge"] is not None: - _sha(group["upstream_merge"]) - if len(set(commits)) != len(commits) or record["promoted"] not in commits: - raise ValueError("Promotion metadata has duplicate or missing source commits.") - - -def _verify_remote_commit(gh: GitHub, plan: Promotion, repo: str, sha: str) -> str: - commit = gh.get(f"repos/{repo}/commits/{_sha(sha)}") - if len(commit["parents"]) != 1 or [item["filename"] for item in commit["files"]] != [_LOCK]: - raise ValueError("Promotion commit must change only the vendor lock and have one parent.") - parent = commit["parents"][0]["sha"] - comparison = gh.compare(_TRTLLM, parent, _main_sha(gh)) - if comparison["merge_base_commit"]["sha"] != parent: - raise ValueError( - "Promotion parent is not on TRT-LLM main; unrelated commits may be present." - ) - before = _lock_document_at(gh, repo, parent) - after = _lock_document_at(gh, repo, sha) - _check_document_change(before, after, plan) - message = commit["commit"]["message"] - _adopt_record(plan, message) - if not re.search(r"^Signed-off-by: .+ <.+>$", message, re.MULTILINE): - raise ValueError("Promotion commit has no DCO sign-off.") - return message - - -def _check_document_change(before: dict, after: dict, plan: Promotion) -> None: - if before["vendors"][_VENDOR] != plan.reviewed.to_mapping(): - raise ValueError("Promotion parent does not contain the reviewed temporary pin.") - expected = {**before, "vendors": {**before["vendors"], _VENDOR: plan.promoted}} - if expected != after: - raise ValueError("Promotion commit is not the expected branch/URL-only lock change.") - - -@contextlib.contextmanager -def _source_repo(args: argparse.Namespace, plan: Promotion) -> Iterator[Path]: - with tempfile.TemporaryDirectory(prefix="prims-ts-promote-source-") as temporary: - repo = args.flashinfer_repo or Path(temporary) - if args.flashinfer_repo is None: - print(f"Fetching reviewed FlashInfer source {plan.reviewed.commit}...", flush=True) - _git(repo, "init", "--quiet") - _git(repo, "fetch", "--quiet", "--no-tags", plan.reviewed.url, plan.reviewed.commit) - _git(repo, "cat-file", "-e", f"{plan.reviewed.commit}^{{commit}}") - _git(repo, "merge-base", "--is-ancestor", plan.previous.commit, plan.reviewed.commit) - yield repo - - -def _local_lock_document(worktree: Path, revision: str | None = None) -> dict: - content = ( - (worktree / _LOCK).read_text() - if revision is None - else _git(worktree, "show", f"{revision}:{_LOCK}") - ) - return yaml.load(content, Loader=vendor._UniqueKeyLoader) - - -def _pending_paths(worktree: Path) -> set[str]: - # A file restored in the working tree can still have an unrelated staged edit. - return set(_git(worktree, "diff", "--name-only", "HEAD").splitlines()) | set( - _git(worktree, "diff", "--cached", "--name-only", "HEAD").splitlines() - ) - - -def _check_lock_only(worktree: Path, before: dict, plan: Promotion) -> None: - _check_document_change(before, _local_lock_document(worktree), plan) - if _pending_paths(worktree) != {_LOCK}: - raise ValueError("Promotion must change only the vendor lock.") - if _git(worktree, "ls-files", "--others", "--exclude-standard"): - raise ValueError("Unexpected untracked files in the promotion worktree.") - - -def _prepare_commit(args: argparse.Namespace, plan: Promotion, source: Path) -> tuple[Path, str]: - print(f"Fetching TRT-LLM base {plan.main_sha}...", flush=True) - _git( - args.repo, - "fetch", - "--quiet", - "--no-tags", - f"https://github.com/{_TRTLLM}.git", - plan.main_sha, - ) - worktree = ( - args.worktree - or args.repo.parent / f"prims-ts-promote-{plan.number}-{plan.reviewed.commit[:12]}" - ) - worktree = worktree.resolve() - if worktree.exists(): - if _git(worktree, "branch", "--show-current") != plan.branch or _git( - worktree, "rev-parse", "--path-format=absolute", "--git-common-dir" - ) != _git(args.repo, "rev-parse", "--path-format=absolute", "--git-common-dir"): - raise ValueError( - f"Existing worktree is not this repository's promotion branch: {worktree}" - ) - head = _git(worktree, "rev-parse", "HEAD") - if _git(worktree, "merge-base", head, plan.main_sha) != head: - if _git(worktree, "status", "--porcelain"): - raise ValueError( - f"Completed promotion worktree contains unexpected edits: {worktree}" - ) - _adopt_record(plan, _git(worktree, "log", "-1", "--format=%B")) - parent = _git(worktree, "rev-parse", "HEAD^") - _git(worktree, "merge-base", "--is-ancestor", parent, plan.main_sha) - before = _local_lock_document(worktree, "HEAD^") - _check_committed_lock(worktree, before, plan) - lock = vendor._load_lock(worktree / _LOCK) - vendor._verify_source(lock, lock.vendors[_VENDOR], source) - print(f"Reusing completed promotion commit {head} in {worktree}.", flush=True) - return worktree, head - print(f"Resuming incomplete promotion in {worktree}...", flush=True) - else: - print(f"Creating promotion worktree {worktree} (LFS downloads disabled)...", flush=True) - _git(args.repo, "worktree", "add", "-b", plan.branch, str(worktree), plan.main_sha) - before = _local_lock_document(worktree, "HEAD") - if before["vendors"][_VENDOR] != plan.reviewed.to_mapping(): - raise ValueError("Worktree base no longer contains the reviewed temporary pin.") - # Recover only the two states this tool writes. Preserve arbitrary user or - # hook edits, including staged-only changes, instead of overwriting them. - for revision in (None, ""): - pending = _local_lock_document(worktree, revision) - if pending != before: - _check_document_change(before, pending, plan) - if _pending_paths(worktree) - {_LOCK}: - raise ValueError( - "Incomplete promotion has changes outside the vendor lock; inspect before retrying." - ) - if _git(worktree, "ls-files", "--others", "--exclude-standard"): - raise ValueError("Incomplete promotion has untracked files; inspect before retrying.") - lock = vendor._load_lock(worktree / _LOCK) - print("Verifying source materialization and the lock-only change...", flush=True) - vendor._verify_source(lock, lock.vendors[_VENDOR], source) - result = vendor.main( - [ - "--lock", - str(worktree / _LOCK), - "pin", - _VENDOR, - "--url", - plan.previous.url, - "--branch", - plan.previous.branch, - "--commit", - plan.reviewed.commit, - "--repo", - str(source), - ] - ) - if result: - raise ValueError("Vendor pin failed; worktree preserved for inspection.") - _check_lock_only(worktree, before, plan) - _git(worktree, "add", "--", _LOCK) - print("Creating signed-off promotion commit...", flush=True) - _git(worktree, "commit", "-s", "-F", "-", text=f"{plan.title}\n\n{_record_text(plan.record)}\n") - if _git(worktree, "status", "--porcelain"): - raise ValueError(f"Commit hooks left changes in {worktree}; inspect and retry.") - _check_committed_lock(worktree, before, plan) - return worktree, _git(worktree, "rev-parse", "HEAD") - - -def _check_committed_lock(worktree: Path, before: dict, plan: Promotion) -> None: - _check_document_change(before, _local_lock_document(worktree), plan) - if len(_git(worktree, "rev-list", "--parents", "-n", "1", "HEAD").split()) != 2: - raise ValueError("Promotion commit must have exactly one parent.") - if _git(worktree, "diff", "--name-only", "HEAD^", "HEAD") != _LOCK: - raise ValueError("Commit hooks added changes outside the vendor lock.") - message = _git(worktree, "log", "-1", "--format=%B") - if _record_from_message(message) != plan.record: - raise ValueError("Commit hooks changed the promotion provenance.") - if not re.search(r"^Signed-off-by: .+ <.+>$", message, re.MULTILINE): - raise ValueError("Promotion commit has no DCO sign-off.") - - -def _check_auto_merge(gh: GitHub) -> None: - settings = gh.get(f"repos/{_TRTLLM}") - if not settings.get("allow_auto_merge") or not settings.get("allow_squash_merge"): - raise ValueError("Repository must allow auto-merge and squash merging.") - rules = gh.api(f"repos/{_TRTLLM}/rules/branches/main") - if not isinstance(rules, list) or any(rule["type"] == "merge_queue" for rule in rules): - raise ValueError( - "Merge queues are unsupported: custom squash-message retention is required." - ) - - -def _disable_auto_merge(gh: GitHub, pr: dict) -> None: - gh.api( - "graphql", - method="POST", - payload={ - "query": ( - "mutation($id:ID!){disablePullRequestAutoMerge(" - "input:{pullRequestId:$id}){clientMutationId}}" - ), - "variables": {"id": pr["node_id"]}, - }, - ) - - -def _report_merged(pr: dict) -> None: - print( - f"Promotion merged: {pr['html_url']}\n" - f"Squash commit: {pr['merge_commit_sha']}\n" - "Final lock-only change, provenance, and DCO sign-off verified.", - flush=True, - ) - - -def _arm_auto_merge(gh: GitHub, plan: Promotion, pr: dict, message: str) -> None: - _check_auto_merge(gh) - _check_current(gh, plan) - number, head = pr["number"], pr["head"]["sha"] - state = gh.api( - "graphql", - method="POST", - payload={ - "query": "query($id:ID!){node(id:$id){... on PullRequest{headRefOid mergeStateStatus}}}", - "variables": {"id": pr["node_id"]}, - }, - ) - node = state["data"]["node"] - if node["headRefOid"] != head: - raise ValueError("Follow-up PR head changed; revalidate before enabling auto-merge.") - title, _, body = message.partition("\n") - saved = pr.get("auto_merge") - if saved: - if ( - saved.get("merge_method") == "squash" - and saved.get("commit_title") == title - and saved.get("commit_message", "").strip() == body.strip() - ): - print(f"Existing squash auto-merge message verified: {pr['html_url']}") - return - _disable_auto_merge(gh, pr) - raise ValueError("Existing auto-merge message differs; disabled. Inspect before retrying.") - # Query only necessary fields: gh pr merge's broader PR query can require - # read:org even when the token already has permission to merge this PR. - operation = ( - "mergePullRequest" if node["mergeStateStatus"] == "CLEAN" else "enablePullRequestAutoMerge" - ) - input_type = ( - "MergePullRequestInput" - if operation == "mergePullRequest" - else "EnablePullRequestAutoMergeInput" - ) - gh.api( - "graphql", - method="POST", - payload={ - "query": f"mutation($input:{input_type}!){{{operation}(input:$input){{clientMutationId}}}}", - "variables": { - "input": { - "pullRequestId": pr["node_id"], - "expectedHeadOid": head, - "mergeMethod": "SQUASH", - "commitHeadline": title, - "commitBody": body.strip(), - } - }, - }, - ) - updated = gh.get(f"repos/{_TRTLLM}/pulls/{number}") - if updated.get("merged"): - _verify_remote_commit(gh, plan, _TRTLLM, updated["merge_commit_sha"]) - _report_merged(updated) - else: - saved = updated.get("auto_merge") or {} - if ( - updated["head"]["sha"] != head - or saved.get("merge_method") != "squash" - or saved.get("commit_title") != title - or saved.get("commit_message", "").strip() != body.strip() - ): - _disable_auto_merge(gh, pr) - raise ValueError("Auto-merge message/head verification failed; auto-merge disabled.") - print(f"Squash auto-merge enabled and message verified: {updated['html_url']}") - - -def _pr_body(plan: Promotion, message: str) -> str: - unpaired = "" - if any(group["upstream_pr"] is None for group in plan.record["changes"]): - unpaired = ( - "No paired FlashInfer PR exists for this source update. Its provenance " - "explicitly records it as unpaired, with `refresh_policy: retain`. A future " - "refresh must carry it forward until an upstream pairing or equivalence " - "is explicitly established.\n\n" - ) - return ( - "@coderabbitai summary\n\n## Description\n\n" - f"Promote the exact source revision reviewed in {plan.record['trtllm_pr']}.\n" - "Only the PrimTS lock URL/branch changes. The source SHA, selected vendor files, " - "compatibility patch, and digests are unchanged.\n\n" - f"{unpaired}" - "## Test Coverage\n\nSource materialization and lock-only diff verified by the promotion tool. " - "No new kernel behavior. The tool posts " - f"`{_CI_SKIP_COMMAND}` to request the bot's no-code-change CI path. " - "Automatic GitHub checks and required reviews still apply.\n\n" - "## PR Checklist\n\n" - "- [x] No runtime/API/dependency changes.\n" - "- [x] Immutable source pin and vendor content preserved.\n" - "- [x] Please check this after reviewing the above items as appropriate for this PR.\n\n" - "## Required squash commit message\n\n" - "Preserve the following message, including the provenance block and DCO sign-off.\n\n" - f"```text\n{message.strip()}\n```\n" - ) - - -def _ensure_ci_skip(gh: GitHub, plan: Promotion, pr: dict) -> dict: - """Request the no-code-change bot path once across sequential retries.""" - current = gh.get(f"repos/{_TRTLLM}/pulls/{pr['number']}") - if current.get("merged"): - _verify_remote_commit(gh, plan, _TRTLLM, current["merge_commit_sha"]) - _report_merged(current) - return current - if current["state"] != "open" or current["head"]["sha"] != pr["head"]["sha"]: - raise ValueError("Follow-up PR closed or changed before the CI request; inspect it.") - _check_current(gh, plan) - _verify_remote_commit(gh, plan, _TRTLLM, current["head"]["sha"]) - actor = gh.get("user")["login"] - endpoint = f"repos/{_TRTLLM}/issues/{pr['number']}/comments" - page = 1 - while True: - comments = gh.api(f"{endpoint}?per_page=100&page={page}") - if not isinstance(comments, list): - raise ValueError("Unexpected issue-comment response; refusing a duplicate CI request.") - if any( - comment.get("body", "").strip() == _CI_SKIP_COMMAND - and comment.get("user", {}).get("login") == actor - for comment in comments - ): - print(f"Existing CI skip request found: {current['html_url']}", flush=True) - return current - if len(comments) < 100: - break - page += 1 - comment = gh.api(endpoint, method="POST", payload={"body": _CI_SKIP_COMMAND}) - print(f"Posted CI skip request: {comment['html_url']}", flush=True) - return current - - -def _publish(args: argparse.Namespace, gh: GitHub, plan: Promotion) -> dict: - if plan.completed_pr is not None: - _report_merged(plan.completed_pr) - return plan.completed_pr - existing = _find_followup(gh, plan) - remote = gh.optional(f"repos/{plan.fork}/git/ref/heads/{plan.branch}") - if existing: - existing = gh.get(f"repos/{_TRTLLM}/pulls/{existing['number']}") - sha = existing["merge_commit_sha"] if existing.get("merged") else existing["head"]["sha"] - message = _verify_remote_commit(gh, plan, _TRTLLM, sha) - if existing.get("merged"): - _report_merged(existing) - return existing - if remote is None or remote["object"]["sha"] != sha: - raise ValueError("Follow-up branch is missing or changed; inspect before resuming.") - elif remote: - sha = remote["object"]["sha"] - message = _verify_remote_commit(gh, plan, plan.fork, sha) - else: - message = "" - if args.auto_merge: - _check_auto_merge(gh) - _check_current(gh, plan) - with _source_repo(args, plan) as source: - if remote is None: - worktree, sha = _prepare_commit(args, plan, source) - message = _git(worktree, "log", "-1", "--format=%B") - _check_current(gh, plan) - # No force push: Git also enforces the fast-forward constraint remotely. - print(f"Ensuring canonical branch points to {plan.reviewed.commit}...", flush=True) - _git( - source, - "push", - plan.previous.url, - f"{plan.reviewed.commit}:refs/heads/{plan.previous.branch}", - ) - print(f"Canonical branch now contains reviewed revision {plan.reviewed.commit}.") - if remote is None: - print(f"Publishing follow-up branch {plan.fork}:{plan.branch}...", flush=True) - _git( - worktree, - "push", - f"https://github.com/{plan.fork}.git", - f"{sha}:refs/heads/{plan.branch}", - ) - _check_current(gh, plan) - if existing is None: - print("Creating the lock-only follow-up PR...", flush=True) - existing = gh.api( - f"repos/{_TRTLLM}/pulls", - method="POST", - payload={ - "title": plan.title, - "body": _pr_body(plan, message), - "base": "main", - "head": f"{plan.fork.split('/')[0]}:{plan.branch}", - "head_repo": plan.fork.split("/")[1], - }, - ) - print(f"Lock-only promotion PR: {existing['html_url']}") - existing = _ensure_ci_skip(gh, plan, existing) - if existing.get("merged"): - return existing - if args.auto_merge: - _arm_auto_merge(gh, plan, existing, message) - if args.wait: - deadline = time.monotonic() + args.timeout - while True: - current = gh.get(f"repos/{_TRTLLM}/pulls/{existing['number']}") - if current.get("merged"): - _verify_remote_commit(gh, plan, _TRTLLM, current["merge_commit_sha"]) - _report_merged(current) - break - if current["state"] == "closed" or current["head"]["sha"] != existing["head"]["sha"]: - if args.auto_merge and current.get("auto_merge"): - _disable_auto_merge(gh, current) - raise ValueError( - "Follow-up PR closed or changed while waiting; inspect before proceeding." - ) - if time.monotonic() >= deadline: - raise TimeoutError( - "Promotion PR is still pending; rerun promote to verify after merge." - ) - time.sleep(min(15, max(0, deadline - time.monotonic()))) - return existing - - -def main(argv: list[str] | None = None) -> int: - """Run the maintainer CLI; dry-run unless publication is explicitly requested.""" - parser = argparse.ArgumentParser(description=__doc__) - commands = parser.add_subparsers(dest="command", required=True) - promote = commands.add_parser("promote", help="Promote a merged TRT-LLM source update.") - promote.add_argument("--trtllm-pr", required=True) - promote.add_argument( - "--canonical-repo", - required=True, - help="Explicit canonical FlashInfer fork OWNER/REPO (not the developer's temporary fork).", - ) - promote.add_argument( - "--canonical-branch", - default="trtllm-prims-ts-dev", - help="Expected canonical branch; specify a dated branch when applicable.", - ) - provenance = promote.add_mutually_exclusive_group(required=True) - provenance.add_argument("--upstream-pr", action="append", default=[]) - provenance.add_argument( - "--unpaired-reason", - help="Explicit exception for changes with no upstream PR; retain them during refresh.", - ) - promote.add_argument("--map-upstream", action="append", default=[], metavar="COMMIT=PR") - promote.add_argument("--repo", type=Path, default=Path(__file__).resolve().parents[1]) - promote.add_argument( - "--fork", help="TRT-LLM personal fork OWNER/REPO; defaults to remote 'fork'." - ) - promote.add_argument( - "--flashinfer-repo", - type=Path, - help="Local source repository containing both immutable SHAs.", - ) - promote.add_argument("--worktree", type=Path, help="New/preserved follow-up worktree path.") - promote.add_argument( - "--publish", - action="store_true", - help="Publish the lock-only PR and post the no-code-change /bot skip request.", - ) - promote.add_argument("--auto-merge", action="store_true") - promote.add_argument( - "--wait", action="store_true", help="Wait for merge and verify final commit metadata." - ) - promote.add_argument( - "--timeout", type=int, default=3600, help="Maximum --wait duration in seconds." - ) - args = parser.parse_args(argv) - if (args.auto_merge or args.wait) and not args.publish: - parser.error("--auto-merge and --wait require --publish") - if args.timeout <= 0: - parser.error("--timeout must be positive") - args.repo = args.repo.resolve() - try: - gh = GitHub() - plan = _make_plan(args, gh) - mode = ( - "ALREADY COMPLETED" if plan.completed_pr else "PUBLISH" if args.publish else "DRY RUN" - ) - print(f"{mode}: {plan.canonical_repo}:{plan.previous.branch}", flush=True) - print(f"Source: {plan.previous.commit} -> {plan.reviewed.commit}") - print(f"Follow-up: {plan.fork}:{plan.branch}") - print(_record_text(plan.record)) - if args.publish: - _publish(args, gh, plan) - elif plan.completed_pr is not None: - _report_merged(plan.completed_pr) - else: - print(f"On publication: post {_CI_SKIP_COMMAND}") - print(f"Squash auto-merge: {'enabled' if args.auto_merge else 'off'}") - print("No remote writes or consumer worktree changes. Use --publish to proceed.") - return 0 - except ( - RuntimeError, - ValueError, - OSError, - KeyError, - subprocess.TimeoutExpired, - vendor.VendorError, - ) as error: - print(f"Promotion stopped: {error}", file=sys.stderr) - print( - "Any completed pushes/PRs are preserved. Inspect the error before retrying.", - file=sys.stderr, - ) - return 1 - + from vendor.promote import main if __name__ == "__main__": - raise SystemExit(main()) + arguments = sys.argv[1:] + if arguments and arguments[0] == "promote": + arguments[1:1] = [ + "--vendor", + "flashinfer-prims-ts", + "--upstream-repo", + "flashinfer-ai/flashinfer", + "--canonical-branch", + "trtllm-prims-ts-dev", + "--legacy-prims-ts", + ] + raise SystemExit(main(arguments)) diff --git a/scripts/vendor/README.md b/scripts/vendor/README.md new file mode 100644 index 000000000000..16911a6422c0 --- /dev/null +++ b/scripts/vendor/README.md @@ -0,0 +1,227 @@ + + +# Vendor source toolkit + +These tools work with any vendor key in `3rdparty/vendor_sources.lock.yaml`. +They contain no operator credentials, personal repository defaults, or private +configuration. Use Python 3.10+, Git, PyYAML, and, for GitHub promotion/monitoring, +an authenticated `gh` CLI. Promotion and monitoring currently support GitHub.com +repositories and canonical **branches**, not tag promotion or periodic rebases. + +| Entry point | Responsibility | +| --- | --- | +| `python scripts/vendor/manage.py` | Create, inspect, sync, export, patch, pin, and verify vendored trees | +| `python scripts/vendor/promote.py promote` | Promote an immutable, merged source update and prepare a lock-only PR | +| `python scripts/vendor/bot.py run` | Monitor source-update PRs, validate metadata, resolve attribution, and promote | + +`scripts/vendor_sources.py` remains a compatibility entry point. The old +`scripts/maintain_prims_ts.py` command supplies the original PrimTS defaults and +branch naming for historical operation retries. New invocations should use the +generic entry points. The separate legacy Triton packager does not use this lock +and is not migrated by this change. + +## Workflow + +```mermaid +flowchart TD + source[Source-update PR changes a vendor pin] --> metadata[Author lists upstream PRs in description] + metadata --> match[Bot validates metadata and matches immutable commits] + match -->|missing or ambiguous| feedback[Request changes with author-facing instructions] + feedback -->|author corrects block, pin, or targeted resolution| metadata + match -->|valid and source PR merged| verify[Verify ancestry, current lock, and materialization] + verify --> ff[Fast-forward configured canonical source branch] + ff --> promotion[Create or reuse URL/branch-only promotion PR] + promotion --> ci[Request CI skip and enable squash auto-merge] + ci -->|required checks and owner reviews| merged[Verify final lock and squash provenance] + merged --> rebase[Other source-update authors rebase onto the canonical pin] +``` + +Promotion preserves the source SHA, selected files, compatibility patch, and +digests. It does not re-vendor code. A periodic upstream refresh is a separate +code-changing operation and must not use the promotion CI-skip path. + +## Operator setup + +Use your own approved GitHub credentials (`gh auth login`, `GH_CONFIG_DIR`, or +the usual GitHub CLI token environment). Configure Git authentication for pushes +and a real Git name/email for DCO-signed commits. Tokens are never accepted in +repository URLs, stored in SQLite, or copied into repository configuration by +these tools. Do not put credentials in command-line arguments. + +The operator needs permission to push the configured canonical source branch +and consumer publishing fork, create consumer PRs/comments/reviews, request the +CI-skip path, and enable auto-merge. Review dismissal can require additional +repository authorization. No GitHub App, repository-admin access, webhook +server, or systemd unit is required merely to run the monitor. Repository rules, +SSO/token policies, CI-bot authorization, and required owner reviews still apply. + +Run only one **publishing** monitor per consumer/vendor/canonical branch. The +workdir lock prevents duplicate processes using the same directory; it is not a +distributed lock between operators or machines. Finish one promotion before +merging the next source update for that vendor. Unexpected concurrent changes +stop publication rather than force-pushing or selecting an arbitrary newer pin. + +## Contributor metadata + +Authors add exactly one block to the source-update PR description. Multiple +vendors can share it; each monitor reads its configured vendor entry: + +````markdown + +```yaml +schema_version: 1 +vendors: + example-library: + upstream_prs: + - https://github.com/UPSTREAM/library/pull/123 + - https://github.com/UPSTREAM/library/pull/456 +``` + +```` + +No complete commit map is required. The bot compares the exact source delta +against immutable upstream PR snapshots. Matching uses commit identity, +whitespace- and context-preserving edits, and whole-series aggregate changes for squash/split +cases. Commit titles, cherry-pick trailers, and fuzzy similarity are not proof. +Merged upstream PRs can still be paired using their original head history. + +If attribution is unresolved, feedback lists the full source SHAs and candidate +PRs. **The PR author**, not the operator, resolves it by adding missing upstream +PRs, removing unrelated PRs, correcting/rebasing the pinned source, or adding a +targeted explanation under that vendor: + +```yaml +resolutions: + FULL_40_CHARACTER_SOURCE_SHA: + upstream_pr: https://github.com/UPSTREAM/library/pull/123 + reason: Explain the downstream adaptation or why this PR owns the change. +``` + +This is an explicit author assertion, not an automatic equivalence claim. The +final provenance records its reason and `retain-until-verified` refresh policy. +When there is no upstream PR for the update, replace `upstream_prs` with a +nonempty `unpaired_reason`. Do not combine those alternatives. Unpaired changes +are retained during future refresh until equivalence is established separately. + +The source head and normalized metadata are rechecked before feedback and +publication. Metadata corrections rerun validation. Historical promotion +records remain authoritative; editing a merged description does not rewrite an +already completed promotion's provenance. + +## Launch the local monitor + +All operator-specific settings are CLI inputs. Start with one dry-run pass: + +```bash +python scripts/vendor/bot.py run \ + --vendor example-library \ + --upstream-repo UPSTREAM/library \ + --canonical-repo MAINTAINER/library \ + --canonical-branch consumer-dev \ + --fork YOUR_ACCOUNT/TensorRT-LLM \ + --workdir /path/to/private/vendor-bot-state \ + --once +``` + +For PrimTS, use vendor `flashinfer-prims-ts`, upstream +`flashinfer-ai/flashinfer`, and the operator's current canonical fork/branch. +The canonical repository is always explicit: the bot never adopts a contributor +fork as the publishing destination merely because it appears in a lock or PR. + +Omit `--once` to run continuously in the foreground. Add **both** `--daemon` and +`--publish` to detach and enable remote writes. `--daemon` alone remains dry-run. +The bot always enables squash auto-merge when publishing. It retains the +complete signed-off provenance message and verifies the final merge later. + +```bash +python scripts/vendor/bot.py status --workdir /path/to/private/vendor-bot-state +python scripts/vendor/bot.py stop --workdir /path/to/private/vendor-bot-state +``` + +`stop` requests a graceful exit after the current operation; it does not kill an +unrelated process using a stale PID. Logs go to `WORKDIR/bot.log` in daemon mode. +The daemon survives terminal closure but does not automatically restart after a +crash or reboot; a supervisor is optional. Stop before changing launch settings. + +The workdir must be empty on first use, owned by the current user, and outside a +Git worktree. It holds private SQLite state, a bare source cache, process-lock +and stop files, and isolated promotion worktrees. Runtime state is not committed. +An existing workdir is bound to its operator and vendor configuration; use a +different directory for another operator/vendor. Authentication is inherited +from the launching environment and is not persisted in that state. + +The default interval is 120 seconds (`--interval`, minimum 30). First launch +discovers existing open PRs and merges occurring from startup onward. To include +earlier merges, set `--since 2026-01-01T00:00:00Z` on first launch. The saved +cutoff and tracked PRs survive restarts; retries discover interrupted pushes, +PR creation, CI requests, and auto-merge without duplicating them. Old historical +operations should be resumed with their original CLI/settings, not backfilled +indiscriminately into a newly configured monitor. + +One process monitors one vendor; separate workdirs can monitor different vendors. +`--consumer-repo`, `--base-branch`, and `--upstream-branch` also support consumers +and upstreams whose default branch is not `main`. `--repo` selects a **trusted** +local consumer checkout; no code or hooks from source forks are executed. + +## Feedback and review safety + +With `--publish`, invalid metadata produces an updatable comment containing the +specific error and a copyable marked block. Eligible open PRs also receive one +metadata-only `REQUEST_CHANGES` review. The bot deduplicates its feedback and +never submits `APPROVE` just because metadata is valid. + +After correction it dismisses only its own marked metadata reviews, when +authorized. If dismissal is forbidden, an authorized human must dismiss the +review; the bot reports the failure rather than approving the code. If the +authenticated account already has a manual approval/objection, the bot uses +comments so it does not supersede that review. It likewise uses comments on the +operator's own PRs. Polling and review permissions do not create a mandatory +pre-merge gate: strict blocking requires repository-admin rules. Promotion still +refuses invalid metadata even if the source PR merges between polls. + +## Manual promotion and recovery + +The same generic engine can run without a daemon: + +```bash +python scripts/vendor/promote.py promote \ + --vendor example-library \ + --source-pr 123 \ + --upstream-repo UPSTREAM/library \ + --canonical-repo MAINTAINER/library \ + --canonical-branch consumer-dev \ + --fork YOUR_ACCOUNT/TensorRT-LLM \ + --from-pr-description \ + --publish --auto-merge +``` + +Omit the final flags for a dry run. Alternatively supply repeated `--upstream-pr` +with `--auto-match`, or an explicit `--unpaired-reason`. The existing +`--map-upstream SHA=PR` interface remains available for manual provenance +assertions but cannot accompany automatic matching. `--source-repo` selects a +local immutable source checkout for materialization; `--worktree` overrides the +follow-up checkout location. `--wait` optionally waits for and verifies merge. + +Publication checks the merged source PR, expected previous canonical lock, +linear fast-forward ancestry, current consumer pin, complete materialization, +and full lock-only diff. It uses non-force pushes and DCO-signed commits. The +`VENDOR PROMOTION V1` JSON block in the commit/squash message records vendor, +source/consumer identities, source SHAs, upstream snapshots, and attribution. +Historical `PRIMTS PROMOTION V1` messages remain readable. No provenance file is +added to the consumer repository. + +## Tests + +Tests use temporary Git repositories and fake GitHub responses, including a real +detached-process smoke test with a fake `gh`; no credentials, network, or GPU: + +```bash +PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 pytest --noconftest -c /dev/null \ + -p no:cacheprovider -o 'markers=cpu_only: CPU-only test' \ + tests/unittest/others/test_vendor_sources.py \ + tests/unittest/others/test_maintain_prims_ts.py \ + tests/unittest/others/test_vendor_bot.py +``` diff --git a/scripts/vendor/__init__.py b/scripts/vendor/__init__.py new file mode 100644 index 000000000000..424dcded61bf --- /dev/null +++ b/scripts/vendor/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Source vendoring, reviewed-revision promotion, and local automation.""" diff --git a/scripts/vendor/bot.py b/scripts/vendor/bot.py new file mode 100644 index 000000000000..f69dbebd2712 --- /dev/null +++ b/scripts/vendor/bot.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Local, restartable vendor-promotion monitor. Remote writes require --publish.""" + +from __future__ import annotations + +import argparse +import contextlib +import datetime +import fcntl +import json +import logging +import os +import signal +import sqlite3 +import subprocess +import sys +import threading +import time +from collections.abc import Iterator +from pathlib import Path + +if __package__: + from . import manage, metadata, promote +else: + import manage + import metadata + import promote + +_LOG = logging.getLogger("vendor-bot") +_STATE_FILE = "state.sqlite" +_STOP = threading.Event() + + +class Store: + """Transactional runtime state; no authentication material is persisted.""" + + def __init__(self, path: Path) -> None: + self.connection = sqlite3.connect(path) + self.connection.execute( + "CREATE TABLE IF NOT EXISTS state (key TEXT PRIMARY KEY, value TEXT NOT NULL)" + ) + + def get(self, key: str, default: object = None) -> object: + row = self.connection.execute("SELECT value FROM state WHERE key = ?", (key,)).fetchone() + return json.loads(row[0]) if row else default + + def put(self, key: str, value: object) -> None: + with self.connection: + self.connection.execute( + "INSERT INTO state VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", + (key, json.dumps(value, sort_keys=True)), + ) + + def close(self) -> None: + """Close SQLite before releasing the process lock.""" + self.connection.close() + + +def _pages(gh: promote.GitHub, endpoint: str) -> Iterator[dict]: + separator = "&" if "?" in endpoint else "?" + page = 1 + while True: + values = gh.api(f"{endpoint}{separator}per_page=100&page={page}") + if not isinstance(values, list): + raise ValueError(f"Expected a list from {endpoint}.") + yield from values + if len(values) < 100: + return + page += 1 + + +class Monitor: + """One operator-configured vendor; run other vendors in separate workdirs.""" + + def __init__(self, args: argparse.Namespace, gh: promote.GitHub, store: Store) -> None: + self.args, self.gh, self.store = args, gh, store + self.actor = gh.get("user")["login"] + self.prefix = f"repos/{gh.consumer_repo}" + self.marker = f"" + settings = { + "consumer_repo": gh.consumer_repo, + "vendor": gh.vendor_name, + "upstream_repo": gh.upstream_repo, + "base_branch": gh.base_branch, + "upstream_branch": gh.upstream_branch, + "canonical_repo": args.canonical_repo, + "canonical_branch": args.canonical_branch, + "fork": args.fork, + "actor": self.actor, + "repo": str(args.repo), + } + saved = store.get("settings") + if saved is not None and saved != settings: + raise ValueError( + "Workdir belongs to a different operator/vendor configuration; choose another." + ) + store.put("settings", settings) + if store.get("since") is None: + store.put( + "since", + args.since + or datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + ) + + def _discover(self) -> list[int]: + # Advance only after a complete scan, with overlap for edits arriving + # during pagination. Tracked operations are reconciled independently. + cutoff = ( + datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(minutes=5) + ).strftime("%Y-%m-%dT%H:%M:%SZ") + tracked = set(self.store.get("tracked", [])) + endpoints = [f"{self.prefix}/pulls?state=all&sort=updated&direction=desc"] + if not self.store.get("bootstrapped", False): + endpoints.insert(0, f"{self.prefix}/pulls?state=open") + since = self.store.get("cursor", self.store.get("since")) + for endpoint in endpoints: + for pr in _pages(self.gh, endpoint): + if "state=all" in endpoint and pr["updated_at"] < since: + break + if pr["base"]["ref"] != self.gh.base_branch: + continue + number = pr["number"] + if pr["state"] == "closed" and not pr.get("merged_at"): + continue + head = pr["head"]["sha"] + key = f"candidate:{number}" + cached = self.store.get(key, {}) + if cached.get("head") != head: + relevant = any( + item["filename"] == promote._LOCK + for item in _pages(self.gh, f"{self.prefix}/pulls/{number}/files") + ) + cached = {"head": head, "relevant": relevant} + self.store.put(key, cached) + if cached["relevant"]: + tracked.add(number) + self.store.put("tracked", sorted(tracked)) + self.store.put("bootstrapped", True) + self.store.put("cursor", cutoff) + return sorted(tracked) + + def _feedback(self, pr: dict, problem: str | None) -> None: + number = pr["number"] + message = ( + f"@{pr['user']['login']} — vendor promotion needs author action:\n\n{problem}\n\n" + "Add or correct this block in the **PR description** (replace the example PR URL):\n\n" + f"````markdown\n{metadata.template(self.gh.vendor_name, self.gh.upstream_repo)}\n````\n\n" + "List every upstream PR; a full commit_map is not required. For an unresolved commit, " + "add a missing PR, update the source pin, or add under this vendor:\n\n" + "```yaml\nresolutions:\n FULL_SOURCE_SHA:\n upstream_pr: https://github.com/OWNER/REPO/pull/123\n" + " reason: Explain the adaptation or why this PR owns the change.\n```\n\n" + "If there is no paired upstream PR, use only `unpaired_reason: ...` for this vendor. " + "Author assertions are retained during future refresh until separately verified." + if problem + else "Vendor promotion metadata and source attribution are valid. " + "This is **not** a code-review approval. Promotion still requires the source PR to merge." + ) + body = f"{self.marker}\n{message}" + dismissal_notice = ( + "\n\nAn authorized reviewer must dismiss the automated metadata " + "review; this account lacks dismissal permission." + ) + if not self.args.publish: + _LOG.info("PR #%s: %s", number, problem or "metadata valid (dry run)") + return + # Check immutable head and description immediately before feedback. A + # concurrent author edit will be retried on the next poll. + current = self.gh.get(f"{self.prefix}/pulls/{number}") + if current["head"]["sha"] != pr["head"]["sha"] or current.get("body") != pr.get("body"): + return + comments = list(_pages(self.gh, f"{self.prefix}/issues/{number}/comments")) + existing = [ + item + for item in comments + if item["user"]["login"] == self.actor and item.get("body", "").startswith(self.marker) + ] + if len(existing) > 1: + raise ValueError("Multiple bot feedback comments; inspect before continuing.") + if existing: + if existing[0]["body"] != body and not ( + not problem and existing[0]["body"] == body + dismissal_notice + ): + self.gh.api( + f"{self.prefix}/issues/comments/{existing[0]['id']}", + method="PATCH", + payload={"body": body}, + ) + else: + self.gh.api( + f"{self.prefix}/issues/{number}/comments", method="POST", payload={"body": body} + ) + if current["state"] != "open" or current["user"]["login"] == self.actor: + return + reviews = [ + item + for item in _pages(self.gh, f"{self.prefix}/pulls/{number}/reviews") + if item["user"]["login"] == self.actor + ] + automated = [ + item + for item in reviews + if item.get("body", "").startswith(self.marker) and item["state"] == "CHANGES_REQUESTED" + ] + manual = [ + item + for item in reviews + if not item.get("body", "").startswith(self.marker) + and item["state"] in ("APPROVED", "CHANGES_REQUESTED") + ] + if problem and not automated and not manual: + self.gh.api( + f"{self.prefix}/pulls/{number}/reviews", + method="POST", + payload={ + "event": "REQUEST_CHANGES", + "commit_id": pr["head"]["sha"], + "body": ( + f"{self.marker}\n{problem}\n\nSee the vendor-bot comment for the required " + "description format and author resolution instructions." + ), + }, + ) + elif not problem: + for review in automated: + # Never replace a metadata objection with APPROVE or dismiss a + # human review. GitHub may require an authorized human dismissal. + try: + self.gh.api( + f"{self.prefix}/pulls/{number}/reviews/{review['id']}/dismissals", + method="PUT", + payload={ + "message": "Automated vendor metadata issue resolved; code review remains required.", + "event": "DISMISS", + }, + ) + except RuntimeError as error: + if "HTTP 403" not in str(error): + raise + _LOG.warning( + "PR #%s: metadata is valid, but an authorized human must dismiss automated review %s.", + number, + review["id"], + ) + if existing and existing[0]["body"] != body + dismissal_notice: + self.gh.api( + f"{self.prefix}/issues/comments/{existing[0]['id']}", + method="PATCH", + payload={"body": body + dismissal_notice}, + ) + return + if existing and existing[0]["body"] == body + dismissal_notice: + self.gh.api( + f"{self.prefix}/issues/comments/{existing[0]['id']}", + method="PATCH", + payload={"body": body}, + ) + + def _inputs(self, pr: dict) -> tuple[manage.Vendor, manage.Vendor] | None: + if pr.get("merged"): + revision = promote._sha(pr["merge_commit_sha"]) + commit = self.gh.get(f"{self.prefix}/commits/{revision}") + base = commit["parents"][0]["sha"] + else: + base, revision = promote._main_sha(self.gh), pr["head"]["sha"] + before = promote._lock_document_at(self.gh, self.gh.consumer_repo, base)["vendors"] + after = promote._lock_document_at(self.gh, self.gh.consumer_repo, revision)["vendors"] + name = self.gh.vendor_name + if name not in before or name not in after: + return None # Vendor creation/removal is not promotion. + previous = manage._validate_vendor(name, before[name]) + reviewed = manage._validate_vendor(name, after[name]) + if previous.commit == reviewed.commit or not reviewed.branch: + return None # Includes lock-only promotions and tag-only updates. + canonical = ( + promote._repo_name(previous.url).lower() == self.args.canonical_repo.lower() + and previous.branch == self.args.canonical_branch + ) + target_is_canonical = ( + promote._repo_name(reviewed.url).lower() == self.args.canonical_repo.lower() + and reviewed.branch == self.args.canonical_branch + ) + if target_is_canonical: + return None # Periodic refresh/direct canonical update: separate workflow. + if not canonical: + raise ValueError( + "Finish the preceding vendor promotion, then rebase this source-update PR onto the canonical pin." + ) + return previous, reviewed + + def inspect(self, number: int) -> None: + """Reconcile one source-update PR and its deterministic promotion operation.""" + if self.store.get(f"complete:{number}"): + return + pr = self.gh.get(f"{self.prefix}/pulls/{number}") + if pr["state"] == "closed" and not pr.get("merged"): + return + try: + inputs = self._inputs(pr) + if inputs is None: + return + previous, reviewed = inputs + entry = metadata.parse(pr.get("body") or "", self.gh.vendor_name, self.gh.upstream_repo) + signature = metadata.fingerprint( + { + "previous": previous.to_mapping(), + "reviewed": reviewed.to_mapping(), + "metadata": entry, + } + ) + key = f"matching:{number}:{signature}" + saved = self.store.get(key) + cache = self.args.workdir / "sources.git" + if saved is None: + assignments, evidence = metadata.resolve(self.gh, previous, reviewed, entry, cache) + saved = {"assignments": assignments, "evidence": evidence} + self.store.put(key, saved) + except (ValueError, manage.VendorError) as error: + self._feedback(pr, str(error)) + self.store.put(f"problem:{number}", str(error)) + return + self._feedback(pr, None) + if not pr.get("merged"): + return + # Re-read mutable description/head before any promotion. Immutable lock + # checks inside promote also run before each remote write. + current = self.gh.get(f"{self.prefix}/pulls/{number}") + if current.get("body") != pr.get("body") or current["head"]["sha"] != pr["head"]["sha"]: + return + arguments = argparse.Namespace( + source_pr=str(number), + canonical_repo=self.args.canonical_repo, + canonical_branch=self.args.canonical_branch, + upstream_pr=[str(item) for item in entry["upstream_prs"]], + unpaired_reason=entry["unpaired_reason"], + map_upstream=[f"{sha}={target}" for sha, target in saved["assignments"].items()], + repo=self.args.repo, + fork=self.args.fork, + source_repo=cache if not entry["unpaired_reason"] and cache.exists() else None, + worktree=self.args.workdir / f"promotion-{number}-{reviewed.commit[:12]}", + publish=self.args.publish, + auto_merge=True, + wait=False, + timeout=3600, + legacy_prims_ts=False, + ) + plan = promote._make_plan(arguments, self.gh) + # Attribution evidence is versioned with the durable promotion record. + # Reuse the original published record on retries rather than replacing it. + if plan.completed_pr is None: + plan.record["attribution"] = saved["evidence"] + plan.record["contributor_metadata"] = { + "source_head": pr["head"]["sha"], + "metadata_digest": metadata.fingerprint(entry), + "pr_author": pr["user"]["login"], + } + if not self.args.publish: + _LOG.info("PR #%s: promotion ready (dry run): %s", number, plan.branch) + return + followup = promote._publish(arguments, self.gh, plan) + self.store.put(f"promotion:{number}", followup["number"]) + latest = self.gh.get(f"{self.prefix}/pulls/{followup['number']}") + if latest.get("merged"): + promote._verify_remote_commit( + self.gh, plan, self.gh.consumer_repo, latest["merge_commit_sha"] + ) + self.store.put(f"complete:{number}", latest["html_url"]) + _LOG.info("PR #%s: promotion %s", number, latest["html_url"]) + + def poll(self) -> None: + """Process candidates serially; preserve failures for safe later retries.""" + for number in self._discover(): + if _STOP.is_set() or (self.args.workdir / "stop").exists(): + _STOP.set() + break + try: + self.inspect(number) + except ( + RuntimeError, + ValueError, + OSError, + KeyError, + subprocess.TimeoutExpired, + ) as error: + self.store.put(f"problem:{number}", str(error)) + _LOG.error("PR #%s paused: %s", number, error) + + +def _initialize_workdir(path: Path) -> None: + if path.exists() and any(path.iterdir()) and not (path / _STATE_FILE).exists(): + raise ValueError("Choose an empty workdir or an existing vendor-bot workdir.") + path.mkdir(parents=True, exist_ok=True, mode=0o700) + if path.stat().st_uid != os.getuid(): + raise ValueError("Workdir must be owned by the current user.") + result = subprocess.run( + ["git", "-C", str(path), "rev-parse", "--is-inside-work-tree"], + capture_output=True, + text=True, + ) + if result.returncode == 0: + raise ValueError( + "Bot workdir must be outside a Git worktree to keep runtime state private." + ) + path.chmod(0o700) + + +@contextlib.contextmanager +def _locked(workdir: Path) -> Iterator[None]: + with (workdir / "bot.lock").open("a+") as handle: + try: + fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as error: + raise ValueError("A bot already owns this workdir.") from error + yield + + +def _running(workdir: Path) -> bool: + if not (workdir / "bot.lock").exists(): + return False + try: + with _locked(workdir): + return False + except ValueError: + return True + + +def _serve(args: argparse.Namespace) -> None: + with _locked(args.workdir): + (args.workdir / "stop").unlink(missing_ok=True) + with contextlib.closing(Store(args.workdir / _STATE_FILE)) as store: + store.put("process", {"pid": None, "publish": args.publish}) + gh = promote.GitHub( + args.consumer_repo, + args.vendor, + args.upstream_repo, + args.base_branch, + args.upstream_branch, + ) + monitor = Monitor(args, gh, store) + store.put("process", {"pid": os.getpid(), "publish": args.publish}) + _LOG.info( + "Started vendor %s as %s; remote writes %s", + args.vendor, + monitor.actor, + args.publish, + ) + failures = 0 + while not _STOP.is_set(): + try: + monitor.poll() + failures = 0 + store.put("last_poll", datetime.datetime.now(datetime.timezone.utc).isoformat()) + except (RuntimeError, OSError, ValueError, subprocess.TimeoutExpired) as error: + failures += 1 + _LOG.error("Poll paused: %s", error) + if args.once: + raise + if args.once: + break + deadline = time.monotonic() + min(args.interval * 2 ** min(failures, 5), 3600) + while time.monotonic() < deadline and not _STOP.wait(1): + if (args.workdir / "stop").exists(): + _STOP.set() + break + store.put("process", {"pid": None, "publish": args.publish}) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + run = commands.add_parser("run", help="Monitor in foreground, or detach with --daemon.") + run.add_argument("--workdir", type=Path, required=True) + run.add_argument("--vendor", required=True) + run.add_argument("--upstream-repo", required=True) + run.add_argument("--canonical-repo", required=True) + run.add_argument("--canonical-branch", required=True) + run.add_argument("--fork", required=True, help="Consumer publishing fork OWNER/REPO.") + run.add_argument("--consumer-repo", default="NVIDIA/TensorRT-LLM") + run.add_argument("--base-branch", default="main") + run.add_argument("--upstream-branch", default="main") + run.add_argument( + "--repo", + type=Path, + default=Path(__file__).resolve().parents[2], + help="Trusted consumer checkout used for isolated promotion worktrees.", + ) + run.add_argument( + "--interval", type=int, default=120, help="Polling interval, seconds (minimum 30)." + ) + run.add_argument( + "--since", + help="First-run merged-PR cutoff, UTC YYYY-MM-DDTHH:MM:SSZ; default startup time.", + ) + run.add_argument( + "--publish", + action="store_true", + help="Allow feedback/reviews, pushes, PR creation, CI skip, and squash auto-merge.", + ) + mode = run.add_mutually_exclusive_group() + mode.add_argument("--daemon", action="store_true") + mode.add_argument( + "--once", + action="store_true", + help="One reconciliation pass, useful for testing or a scheduler.", + ) + for name in ("status", "stop"): + command = commands.add_parser(name) + command.add_argument("--workdir", required=True, type=Path) + return parser + + +def main(argv: list[str] | None = None) -> int: + """Launch/control a POSIX daemon without storing credentials or requiring systemd.""" + args = _parser().parse_args(argv) + args.workdir = args.workdir.expanduser().resolve() + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + try: + if args.command != "run": + active = _running(args.workdir) + if args.command == "stop" and active: + (args.workdir / "stop").touch(mode=0o600) + print( + json.dumps({"running": active, "stop_requested": args.command == "stop" and active}) + ) + return 0 + if args.interval < 30: + raise ValueError("--interval must be at least 30 seconds.") + if not manage._NAME_PATTERN.fullmatch(args.vendor): + raise ValueError("Invalid vendor key.") + for name in (args.consumer_repo, args.upstream_repo, args.canonical_repo, args.fork): + promote._repo_name(f"https://github.com/{name}") + for branch in (args.canonical_branch, args.base_branch, args.upstream_branch): + promote._run(["git", "check-ref-format", f"refs/heads/{branch}"]) + if args.since: + datetime.datetime.strptime(args.since, "%Y-%m-%dT%H:%M:%SZ") + args.repo = args.repo.expanduser().resolve() + _initialize_workdir(args.workdir) + if args.daemon: + # Initialize the state marker before launching so two competing + # starts can both reach the authoritative child-held flock. + with contextlib.closing(Store(args.workdir / _STATE_FILE)): + pass + if _running(args.workdir): + raise ValueError("A bot already owns this workdir.") + command = [ + sys.executable, + str(Path(__file__).resolve()), + *list(argv if argv is not None else sys.argv[1:]), + ] + command.remove("--daemon") + with (args.workdir / "bot.log").open("ab", buffering=0) as log: + child = subprocess.Popen( + command, + stdin=subprocess.DEVNULL, + stdout=log, + stderr=log, + start_new_session=True, + close_fds=True, + umask=0o077, + ) + for _ in range(100): + if child.poll() is not None: + raise ValueError( + f"Daemon exited during startup; inspect {args.workdir / 'bot.log'}." + ) + with contextlib.closing(Store(args.workdir / _STATE_FILE)) as state: + process = state.get("process", {}) + if _running(args.workdir) and process.get("pid") == child.pid: + print(f"Started daemon PID {child.pid}; log: {args.workdir / 'bot.log'}") + return 0 + time.sleep(0.1) + raise ValueError( + "Daemon startup not confirmed; inspect log and status before retrying." + ) + signal.signal(signal.SIGTERM, lambda *_: _STOP.set()) + signal.signal(signal.SIGINT, lambda *_: _STOP.set()) + _STOP.clear() + _serve(args) + return 0 + except (ValueError, RuntimeError, OSError, sqlite3.Error, subprocess.TimeoutExpired) as error: + _LOG.error("Bot stopped: %s", error) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/vendor/manage.py b/scripts/vendor/manage.py new file mode 100644 index 000000000000..722b2b1a7552 --- /dev/null +++ b/scripts/vendor/manage.py @@ -0,0 +1,1644 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Manage source trees vendored from immutable Git commits.""" + +from __future__ import annotations + +import argparse +import contextlib +import dataclasses +import fnmatch +import hashlib +import os +import re +import shutil +import stat +import subprocess +import sys +import tempfile +import urllib.parse +from collections.abc import Iterator, Mapping, Sequence +from pathlib import Path, PurePosixPath + +import yaml + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_DEFAULT_LOCK = _REPO_ROOT / "3rdparty" / "vendor_sources.lock.yaml" +_SCHEMA_VERSION = 1 +_COMMIT_PATTERN = re.compile(r"[0-9a-f]{40}\Z") +_NAME_PATTERN = re.compile(r"[a-z0-9][a-z0-9._-]*\Z") +_DIGEST_PREFIX = "sha256-tree-v1:" +_PATCH_DIGEST_PREFIX = "sha256:" +_GIT_TIMEOUT_SECONDS = 60 +_GIT_LOCAL_ENVIRONMENT_VARIABLES = { + "GIT_ALTERNATE_OBJECT_DIRECTORIES", + "GIT_ATTR_GLOBAL", + "GIT_ATTR_SOURCE", + "GIT_ATTR_SYSTEM", + "GIT_CEILING_DIRECTORIES", + "GIT_COMMON_DIR", + "GIT_CONFIG", + "GIT_CONFIG_COUNT", + "GIT_CONFIG_PARAMETERS", + "GIT_DIR", + "GIT_GRAFT_FILE", + "GIT_IMPLICIT_WORK_TREE", + "GIT_INDEX_FILE", + "GIT_NO_REPLACE_OBJECTS", + "GIT_OBJECT_DIRECTORY", + "GIT_PREFIX", + "GIT_REPLACE_REF_BASE", + "GIT_SHALLOW_FILE", + "GIT_WORK_TREE", +} +_VENDOR_FIELDS = { + "url", + "branch", + "tag", + "commit", + "source", + "destination", + "include", + "patch", + "patch_digest", + "digest", +} + + +class VendorError(RuntimeError): + """A user-actionable vendoring failure.""" + + +class UpstreamUnavailable(VendorError): + """The recorded upstream could not be accessed.""" + + +class _UniqueKeyLoader(yaml.SafeLoader): + """Safe YAML loader that rejects duplicate mapping keys.""" + + +def _construct_unique_mapping( + loader: _UniqueKeyLoader, + node: yaml.MappingNode, + deep: bool = False, +) -> dict[object, object]: + if any( + isinstance(key_node, yaml.ScalarNode) and key_node.value == "<<" + for key_node, _ in node.value + ): + raise yaml.constructor.ConstructorError( + "while constructing a mapping", + node.start_mark, + "YAML merge keys are not allowed in the vendor lock", + node.start_mark, + ) + loader.flatten_mapping(node) + result: dict[object, object] = {} + for key_node, value_node in node.value: + key = loader.construct_object(key_node, deep=deep) + try: + duplicate = key in result + except TypeError as error: + raise yaml.constructor.ConstructorError( + "while constructing a mapping", + node.start_mark, + "found an unhashable mapping key", + key_node.start_mark, + ) from error + if duplicate: + raise yaml.constructor.ConstructorError( + "while constructing a mapping", + node.start_mark, + f"found duplicate key {key!r}", + key_node.start_mark, + ) + result[key] = loader.construct_object(value_node, deep=deep) + return result + + +_UniqueKeyLoader.add_constructor( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, + _construct_unique_mapping, +) + + +@dataclasses.dataclass(frozen=True) +class FileEntry: + """One regular file in a normalized selected tree.""" + + data: bytes + executable: bool = False + + +Tree = dict[str, FileEntry] + + +@dataclasses.dataclass +class Vendor: + """Validated lock entry for one managed source tree.""" + + name: str + url: str + commit: str + source: str + destination: str + include: tuple[str, ...] + digest: str + branch: str | None = None + tag: str | None = None + patch: str | None = None + patch_digest: str | None = None + + def to_mapping(self) -> dict[str, object]: + """Serialize this entry in the canonical field order.""" + result: dict[str, object] = {"url": self.url} + if self.branch is not None: + result["branch"] = self.branch + if self.tag is not None: + result["tag"] = self.tag + result.update( + { + "commit": self.commit, + "source": self.source, + "destination": self.destination, + "include": list(self.include), + } + ) + if self.patch is not None: + result["patch"] = self.patch + result["patch_digest"] = self.patch_digest + result["digest"] = self.digest + return result + + +@dataclasses.dataclass +class LockFile: + """Validated collection of vendor entries and its consumer root.""" + + path: Path + root: Path + vendors: dict[str, Vendor] + + +def _require_mapping(value: object, description: str) -> Mapping[object, object]: + if not isinstance(value, Mapping): + raise VendorError(f"{description} must be a mapping.") + return value + + +def _validate_mapping_keys( + mapping: Mapping[object, object], allowed: set[str], description: str +) -> set[str]: + keys: set[str] = set() + for key in mapping: + if not isinstance(key, str): + raise VendorError(f"{description} keys must be strings; got {key!r}.") + keys.add(key) + unknown = keys - allowed + if unknown: + raise VendorError(f"{description} has unexpected fields: {sorted(unknown)}.") + return keys + + +def _require_string(value: object, description: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise VendorError(f"{description} must be a non-empty string.") + return value + + +def _validate_relative_path(value: object, description: str) -> str: + path_text = _require_string(value, description) + if "\\" in path_text: + raise VendorError(f"{description} must use POSIX separators: {path_text!r}.") + path = PurePosixPath(path_text) + if ( + path.is_absolute() + or path_text == "." + or path_text.startswith(":") + or any(part in ("", ".", "..", ".git") for part in path.parts) + ): + raise VendorError(f"{description} must be a safe relative path: {path_text!r}.") + return path.as_posix() + + +def _validate_url(value: object, vendor_name: str) -> str: + url = _require_string(value, f"Vendor {vendor_name!r} url") + parsed = urllib.parse.urlsplit(url) + if parsed.password is not None or ( + parsed.scheme in ("http", "https") and parsed.username is not None + ): + raise VendorError( + f"Vendor {vendor_name!r} URL must not contain credentials; use Git credential helpers." + ) + return url + + +def _validate_include(value: object, vendor_name: str) -> tuple[str, ...]: + if not isinstance(value, list) or not value: + raise VendorError(f"Vendor {vendor_name!r} include must be a non-empty list.") + patterns: list[str] = [] + for index, item in enumerate(value): + pattern = _require_string(item, f"Vendor {vendor_name!r} include[{index}]") + if "\\" in pattern or pattern.startswith("/"): + raise VendorError(f"Vendor {vendor_name!r} has an unsafe include pattern: {pattern!r}.") + if any(part == ".." for part in PurePosixPath(pattern).parts): + raise VendorError(f"Vendor {vendor_name!r} has an unsafe include pattern: {pattern!r}.") + patterns.append(pattern) + if len(patterns) != len(set(patterns)): + raise VendorError(f"Vendor {vendor_name!r} include patterns must be unique.") + return tuple(patterns) + + +def _validate_digest(value: object, description: str) -> str: + digest = _require_string(value, description) + suffix = digest.removeprefix(_DIGEST_PREFIX) + if not digest.startswith(_DIGEST_PREFIX) or not re.fullmatch(r"[0-9a-f]{64}", suffix): + raise VendorError(f"{description} must use {_DIGEST_PREFIX}<64 lowercase hex digits>.") + return digest + + +def _validate_patch_digest(value: object, description: str) -> str: + digest = _require_string(value, description) + suffix = digest.removeprefix(_PATCH_DIGEST_PREFIX) + if not digest.startswith(_PATCH_DIGEST_PREFIX) or not re.fullmatch(r"[0-9a-f]{64}", suffix): + raise VendorError( + f"{description} must use {_PATCH_DIGEST_PREFIX}<64 lowercase hex digits>." + ) + return digest + + +def _patch_digest(content: bytes) -> str: + return f"{_PATCH_DIGEST_PREFIX}{hashlib.sha256(content).hexdigest()}" + + +def _validate_vendor(name: object, value: object) -> Vendor: + vendor_name = _require_string(name, "Vendor name") + if _NAME_PATTERN.fullmatch(vendor_name) is None: + raise VendorError( + f"Vendor name {vendor_name!r} must contain only lowercase letters, digits, '.', '_', or '-'." + ) + mapping = _require_mapping(value, f"Vendor {vendor_name!r}") + fields = _validate_mapping_keys(mapping, _VENDOR_FIELDS, f"Vendor {vendor_name!r}") + required = {"url", "commit", "source", "destination", "include", "digest"} + missing = required - fields + if missing: + raise VendorError(f"Vendor {vendor_name!r} is missing fields: {sorted(missing)}.") + branch = mapping.get("branch") + tag = mapping.get("tag") + if branch is not None and tag is not None: + raise VendorError(f"Vendor {vendor_name!r} cannot record both branch and tag.") + branch_text = ( + None if branch is None else _require_string(branch, f"Vendor {vendor_name!r} branch") + ) + tag_text = None if tag is None else _require_string(tag, f"Vendor {vendor_name!r} tag") + for kind, ref in (("branch", branch_text), ("tag", tag_text)): + if ref is not None: + invalid_ref = ( + ref.startswith(("refs/", ".", "/")) + or ref.endswith((".", "/", ".lock")) + or ".." in ref + or "@{" in ref + or "//" in ref + or any(char.isspace() or char in "~^:?*[\\" for char in ref) + ) + if invalid_ref: + raise VendorError(f"Vendor {vendor_name!r} {kind} must be a short Git name.") + commit = _require_string(mapping["commit"], f"Vendor {vendor_name!r} commit") + if _COMMIT_PATTERN.fullmatch(commit) is None: + raise VendorError(f"Vendor {vendor_name!r} commit must be a full lowercase 40-hex SHA.") + patch_value = mapping.get("patch") + patch = ( + None + if patch_value is None + else _validate_relative_path(patch_value, f"Vendor {vendor_name!r} patch") + ) + expected_patch = f"3rdparty/vendor_patches/{vendor_name}.patch" + if patch is not None and patch != expected_patch: + raise VendorError(f"Vendor {vendor_name!r} patch must be {expected_patch!r}.") + patch_digest_value = mapping.get("patch_digest") + if (patch is None) != (patch_digest_value is None): + raise VendorError( + f"Vendor {vendor_name!r} patch and patch_digest must either both be present or both be absent." + ) + patch_digest = ( + None + if patch_digest_value is None + else _validate_patch_digest(patch_digest_value, f"Vendor {vendor_name!r} patch_digest") + ) + include = _validate_include(mapping["include"], vendor_name) + return Vendor( + name=vendor_name, + url=_validate_url(mapping["url"], vendor_name), + branch=branch_text, + tag=tag_text, + commit=commit, + source=_validate_relative_path(mapping["source"], f"Vendor {vendor_name!r} source"), + destination=_validate_relative_path( + mapping["destination"], f"Vendor {vendor_name!r} destination" + ), + include=include, + patch=patch, + patch_digest=patch_digest, + digest=_validate_digest(mapping["digest"], f"Vendor {vendor_name!r} digest"), + ) + + +def _consumer_root(lock_path: Path) -> Path: + if lock_path.parent.name == "3rdparty": + return lock_path.parent.parent.resolve() + return lock_path.parent.resolve() + + +def _validate_destination_layout(lock: LockFile) -> None: + destinations: list[tuple[str, PurePosixPath]] = [] + try: + lock_relative = PurePosixPath(lock.path.relative_to(lock.root).as_posix()) + except ValueError as error: + raise VendorError(f"Vendor lock must be inside the consumer root: {lock.path}.") from error + patch_directory = PurePosixPath("3rdparty/vendor_patches") + for name, vendor in lock.vendors.items(): + path = PurePosixPath(vendor.destination) + for other_name, other_path in destinations: + if ( + path == other_path + or path.is_relative_to(other_path) + or other_path.is_relative_to(path) + ): + raise VendorError( + f"Vendor destinations overlap: {name!r} ({path}) and {other_name!r} ({other_path})." + ) + destinations.append((name, path)) + if ( + path == patch_directory + or path.is_relative_to(patch_directory) + or patch_directory.is_relative_to(path) + ): + raise VendorError( + f"Vendor {name!r} destination overlaps vendoring control files: {path}." + ) + if lock_relative == path or lock_relative.is_relative_to(path): + raise VendorError(f"Vendor {name!r} destination contains the vendor lock: {path}.") + if vendor.patch is not None: + patch_path = PurePosixPath(vendor.patch) + if ( + patch_path == path + or patch_path.is_relative_to(path) + or path.is_relative_to(patch_path) + ): + raise VendorError(f"Vendor {name!r} destination overlaps its patch path: {path}.") + + +def _load_lock( + path: Path, + *, + allow_missing: bool = False, + allow_patch_recovery: bool = False, +) -> LockFile: + resolved = path.resolve() + if not resolved.exists(): + if allow_missing: + return LockFile(path=resolved, root=_consumer_root(resolved), vendors={}) + raise VendorError(f"Vendor lock does not exist: {resolved}.") + try: + content = resolved.read_text(encoding="utf-8") + if any( + isinstance(token, (yaml.tokens.AliasToken, yaml.tokens.AnchorToken)) + for token in yaml.scan(content) + ): + raise VendorError("Vendor lock YAML aliases and anchors are not allowed.") + loaded = yaml.load(content, Loader=_UniqueKeyLoader) + except VendorError: + raise + except (OSError, UnicodeError, yaml.YAMLError) as error: + raise VendorError(f"Failed to parse vendor lock {resolved}: {error}") from error + mapping = _require_mapping(loaded, "Vendor lock") + _validate_mapping_keys(mapping, {"schema_version", "vendors"}, "Vendor lock") + if mapping.get("schema_version") != _SCHEMA_VERSION: + raise VendorError(f"Vendor lock schema_version must be {_SCHEMA_VERSION}.") + raw_vendors = _require_mapping(mapping.get("vendors"), "Vendor lock vendors") + vendors = { + _require_string(name, "Vendor name"): _validate_vendor(name, value) + for name, value in raw_vendors.items() + } + lock = LockFile(path=resolved, root=_consumer_root(resolved), vendors=vendors) + _validate_destination_layout(lock) + for vendor in vendors.values(): + _checked_root_path(lock.root, vendor.destination, f"Vendor {vendor.name!r} destination") + if vendor.patch is not None: + patch_path = _checked_root_path( + lock.root, vendor.patch, f"Vendor {vendor.name!r} patch" + ) + if patch_path.is_symlink() or (patch_path.exists() and not patch_path.is_file()): + raise VendorError( + f"Vendor {vendor.name!r} patch does not name a regular file: {vendor.patch}." + ) + if not patch_path.exists() and allow_patch_recovery: + continue + if not patch_path.is_file(): + raise VendorError( + f"Vendor {vendor.name!r} patch does not name a regular file: {vendor.patch}." + ) + actual_patch_digest = _patch_digest(patch_path.read_bytes()) + if actual_patch_digest != vendor.patch_digest and not allow_patch_recovery: + raise VendorError( + f"Vendor {vendor.name!r} patch digest mismatch: expected " + f"{vendor.patch_digest}, got {actual_patch_digest}." + ) + return lock + + +def _save_lock(lock: LockFile) -> None: + payload = { + "schema_version": _SCHEMA_VERSION, + "vendors": {name: vendor.to_mapping() for name, vendor in sorted(lock.vendors.items())}, + } + content = yaml.safe_dump( + payload, + sort_keys=False, + default_flow_style=False, + allow_unicode=True, + ).encode("utf-8") + _atomic_write(lock.path, content) + + +def _save_lock_checked(lock: LockFile, description: str) -> None: + try: + _save_lock(lock) + except (OSError, yaml.YAMLError) as error: + raise VendorError(f"Failed to save {description}: {error}") from error + + +def _checked_root_path(root: Path, relative_path: str, description: str) -> Path: + target = root.joinpath(*PurePosixPath(relative_path).parts) + try: + resolved = target.resolve(strict=False) + except OSError as error: + raise VendorError(f"Could not resolve {description}: {error}") from error + if not resolved.is_relative_to(root.resolve()): + raise VendorError(f"{description} escapes the consumer root: {relative_path!r}.") + current = root.resolve() + for part in PurePosixPath(relative_path).parts: + current = current / part + if current.is_symlink(): + raise VendorError(f"{description} traverses a symlink: {relative_path!r}.") + if not current.exists(): + break + return target + + +def _atomic_write(path: Path, content: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + file_descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + temporary = Path(temporary_name) + try: + with os.fdopen(file_descriptor, "wb") as output: + output.write(content) + output.flush() + os.fsync(output.fileno()) + temporary.chmod(0o644) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def _sync_directory(path: Path) -> None: + directory_descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(directory_descriptor) + finally: + os.close(directory_descriptor) + + +def _matches(path: str, patterns: Sequence[str]) -> bool: + """Match a full relative POSIX path, with ** spanning zero or more components.""" + path_parts = PurePosixPath(path).parts + for pattern in patterns: + # Each entry records whether that path prefix matches the pattern so far. + matched = [True] + [False] * len(path_parts) + for part in PurePosixPath(pattern).parts: + if part == "**": + for index in range(1, len(matched)): + matched[index] = matched[index] or matched[index - 1] + else: + matched = [False] + [ + matched[index] and fnmatch.fnmatchcase(path_part, part) + for index, path_part in enumerate(path_parts) + ] + if matched[-1]: + return True + return False + + +def _validate_tree_relative_path(path: str) -> str: + pure_path = PurePosixPath(path) + if ( + pure_path.is_absolute() + or any(part in ("", ".", "..", ".git") for part in pure_path.parts) + or pure_path.as_posix() != path + ): + raise VendorError(f"Managed tree contains an unsafe path: {path!r}.") + return path + + +def _tree_digest(tree: Mapping[str, FileEntry]) -> str: + digest = hashlib.sha256() + digest.update(b"sha256-tree-v1\0") + for path, entry in sorted(tree.items()): + encoded_path = path.encode("utf-8") + digest.update(len(encoded_path).to_bytes(8, "big")) + digest.update(encoded_path) + digest.update(b"x" if entry.executable else b"-") + digest.update(len(entry.data).to_bytes(8, "big")) + digest.update(entry.data) + return f"{_DIGEST_PREFIX}{digest.hexdigest()}" + + +def _read_directory_tree(directory: Path, patterns: Sequence[str]) -> Tree: + if not directory.exists(): + return {} + if directory.is_symlink() or not directory.is_dir(): + raise VendorError(f"Managed destination must be a regular directory: {directory}.") + tree: Tree = {} + for path in sorted(directory.rglob("*")): + relative = _validate_tree_relative_path(path.relative_to(directory).as_posix()) + if path.is_symlink(): + raise VendorError(f"Managed trees cannot contain symlinks: {relative}.") + if path.is_dir(): + continue + if not path.is_file(): + raise VendorError(f"Managed trees cannot contain special files: {relative}.") + if not _matches(relative, patterns): + continue + mode = path.stat().st_mode + tree[relative] = FileEntry(path.read_bytes(), bool(mode & stat.S_IXUSR)) + return tree + + +def _read_all_directory_files(directory: Path, patterns: Sequence[str]) -> Tree: + tree = _read_directory_tree(directory, patterns) + for path in sorted(directory.rglob("*")): + if path.is_dir(): + continue + relative = path.relative_to(directory).as_posix() + if not _matches(relative, patterns): + raise VendorError(f"Patch produced a file outside the vendor include set: {relative}.") + return tree + + +def _write_tree(directory: Path, tree: Mapping[str, FileEntry]) -> None: + directory.mkdir(parents=True, exist_ok=True) + for relative, entry in tree.items(): + _validate_tree_relative_path(relative) + target = directory.joinpath(*PurePosixPath(relative).parts) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(entry.data) + target.chmod(0o755 if entry.executable else 0o644) + + +def _remove_path(path: Path) -> None: + if path.is_symlink() or path.is_file(): + path.unlink() + elif path.is_dir(): + shutil.rmtree(path) + + +@contextlib.contextmanager +def _replace_selected_destination_transaction( + lock: LockFile, vendor: Vendor, tree: Mapping[str, FileEntry] +) -> Iterator[None]: + destination = _checked_root_path( + lock.root, vendor.destination, f"Vendor {vendor.name!r} destination" + ) + backup = destination.parent / f".{destination.name}.vendor-backup" + staging: Path | None = None + destination_existed = False + replacement_installed = False + preserve_staging = False + try: + destination.parent.mkdir(parents=True, exist_ok=True) + if destination.exists(): + _read_directory_tree(destination, vendor.include) + if backup.exists() or backup.is_symlink(): + raise VendorError(f"Stale vendor backup blocks synchronization: {backup}.") + staging = Path( + tempfile.mkdtemp(prefix=f".{destination.name}.vendor-", dir=destination.parent) + ) + destination_existed = destination.exists() + if destination_existed: + shutil.copytree(destination, staging, dirs_exist_ok=True) + for path in sorted(staging.rglob("*"), reverse=True): + if path.is_dir(): + with contextlib.suppress(OSError): + path.rmdir() + continue + relative = path.relative_to(staging).as_posix() + if _matches(relative, vendor.include): + path.unlink() + _write_tree(staging, tree) + if destination_existed: + os.replace(destination, backup) + try: + os.replace(staging, destination) + except OSError: + if backup.exists() and not destination.exists(): + os.replace(backup, destination) + raise + replacement_installed = True + try: + yield + except BaseException as triggering_error: + try: + if replacement_installed and (destination.exists() or destination.is_symlink()): + os.replace(destination, staging) + if destination_existed and (backup.exists() or backup.is_symlink()): + os.replace(backup, destination) + except OSError as rollback_error: + preserve_staging = True + raise VendorError( + f"Failed to roll back destination for vendor {vendor.name!r} after " + f"{triggering_error}; recovery data remains at {backup} and {staging}: " + f"{rollback_error}" + ) from rollback_error + raise + else: + if backup.exists() or backup.is_symlink(): + _remove_path(backup) + except OSError as error: + raise VendorError( + f"Failed to replace destination for vendor {vendor.name!r}: {error}" + ) from error + finally: + if ( + staging is not None + and not preserve_staging + and (staging.exists() or staging.is_symlink()) + ): + try: + _remove_path(staging) + except OSError as error: + raise VendorError( + f"Failed to clean staging directory for vendor {vendor.name!r}: " + f"{staging}: {error}" + ) from error + + +def _replace_selected_destination( + lock: LockFile, vendor: Vendor, tree: Mapping[str, FileEntry] +) -> None: + with _replace_selected_destination_transaction(lock, vendor, tree): + pass + + +@contextlib.contextmanager +def _atomic_file_replacement(path: Path, content: bytes) -> Iterator[None]: + try: + if path.is_symlink() or (path.exists() and not path.is_file()): + raise VendorError(f"Generated patch path must be a regular file: {path}.") + existed = path.exists() + old_content = path.read_bytes() if existed else None + old_mode = stat.S_IMODE(path.stat().st_mode) if existed else None + _atomic_write(path, content) + except VendorError: + raise + except OSError as error: + raise VendorError(f"Failed to write generated patch {path}: {error}") from error + try: + yield + except BaseException: + try: + if old_content is None: + path.unlink(missing_ok=True) + else: + _atomic_write(path, old_content) + assert old_mode is not None + path.chmod(old_mode) + except OSError as error: + raise VendorError(f"Failed to roll back generated patch {path}: {error}") from error + raise + + +def _git_environment(*, isolated: bool, repository_ceiling: Path | None = None) -> dict[str, str]: + environment = os.environ.copy() + for variable in _GIT_LOCAL_ENVIRONMENT_VARIABLES: + environment.pop(variable, None) + environment.update( + { + "GIT_TERMINAL_PROMPT": "0", + "GIT_ATTR_NOSYSTEM": "1", + "GIT_LITERAL_PATHSPECS": "1", + "GIT_NO_REPLACE_OBJECTS": "1", + "GIT_HTTP_LOW_SPEED_LIMIT": "1", + "GIT_HTTP_LOW_SPEED_TIME": "30", + "GIT_AUTHOR_NAME": "TensorRT-LLM Vendor Tool", + "GIT_AUTHOR_EMAIL": "vendor-tool@nvidia.com", + "GIT_COMMITTER_NAME": "TensorRT-LLM Vendor Tool", + "GIT_COMMITTER_EMAIL": "vendor-tool@nvidia.com", + } + ) + if isolated: + environment["GIT_CONFIG_NOSYSTEM"] = "1" + environment["GIT_CONFIG_GLOBAL"] = os.devnull + if repository_ceiling is not None: + environment["GIT_CEILING_DIRECTORIES"] = str(repository_ceiling.resolve()) + environment.setdefault("GIT_SSH_COMMAND", "ssh -o BatchMode=yes -o ConnectTimeout=20") + return environment + + +def _run_git( + arguments: Sequence[str | Path], + *, + cwd: Path, + capture_bytes: bool = False, + isolated: bool = False, + input_bytes: bytes | None = None, + repository_ceiling: Path | None = None, +) -> bytes | str: + if input_bytes is not None and not capture_bytes: + raise ValueError("Git commands with byte input must capture byte output.") + command = ["git", *[str(argument) for argument in arguments]] + try: + result = subprocess.run( + command, + cwd=cwd, + env=_git_environment( + isolated=isolated, + repository_ceiling=repository_ceiling, + ), + check=True, + capture_output=True, + input=input_bytes, + text=not capture_bytes, + timeout=_GIT_TIMEOUT_SECONDS, + ) + except FileNotFoundError as error: + raise VendorError("Git is required for this command but was not found.") from error + except subprocess.TimeoutExpired as error: + raise VendorError( + f"Git command timed out after {_GIT_TIMEOUT_SECONDS} seconds: {' '.join(command)}" + ) from error + except subprocess.CalledProcessError as error: + stderr_value = error.stderr + if isinstance(stderr_value, bytes): + stderr_text = stderr_value.decode("utf-8", errors="replace") + else: + stderr_text = stderr_value or "" + raise VendorError( + f"Git command failed: {' '.join(command)}\n{stderr_text.strip()}" + ) from error + return result.stdout + + +def _read_git_blobs(repo: Path, object_ids: Sequence[bytes], source: str) -> list[bytes]: + batch_input = b"".join(object_id + b"\n" for object_id in object_ids) + output = _run_git( + ["-C", repo, "cat-file", "--batch"], + cwd=repo, + capture_bytes=True, + input_bytes=batch_input, + ) + assert isinstance(output, bytes) + blobs: list[bytes] = [] + position = 0 + for expected_object_id in object_ids: + header_end = output.find(b"\n", position) + if header_end < 0: + raise VendorError(f"Git produced truncated blob metadata for source {source!r}.") + header = output[position:header_end] + fields = header.split(b" ") + if len(fields) != 3: + raise VendorError(f"Git produced invalid blob metadata for source {source!r}.") + object_id, object_type, raw_size = fields + try: + size = int(raw_size) + except ValueError as error: + raise VendorError( + f"Git produced an invalid blob size for source {source!r}." + ) from error + if object_id != expected_object_id or object_type != b"blob" or size < 0: + raise VendorError(f"Git returned an unexpected object for source {source!r}.") + data_start = header_end + 1 + data_end = data_start + size + if data_end >= len(output) or output[data_end : data_end + 1] != b"\n": + raise VendorError(f"Git produced truncated blob data for source {source!r}.") + blobs.append(output[data_start:data_end]) + position = data_end + 1 + if position != len(output): + raise VendorError(f"Git produced trailing blob data for source {source!r}.") + return blobs + + +def _archive_tree(repo: Path, commit: str, source: str, patterns: Sequence[str]) -> Tree: + try: + listing = _run_git( + ["-C", repo, "ls-tree", "-r", "-z", "--full-tree", commit, "--", source], + cwd=repo, + capture_bytes=True, + ) + except VendorError as error: + raise VendorError( + f"Could not read source {source!r} at commit {commit}: {error}" + ) from error + assert isinstance(listing, bytes) + if listing and not listing.endswith(b"\0"): + raise VendorError(f"Git produced an invalid tree listing for source {source!r}.") + + source_path = PurePosixPath(source) + selected: list[tuple[str, bytes, bool]] = [] + seen_paths: set[str] = set() + for record in listing.split(b"\0")[:-1]: + metadata, separator, raw_path = record.partition(b"\t") + fields = metadata.split(b" ") + if not separator or len(fields) != 3: + raise VendorError(f"Git produced an invalid tree entry for source {source!r}.") + raw_mode, object_type, object_id = fields + try: + path = raw_path.decode("utf-8") + mode = raw_mode.decode("ascii") + except UnicodeError as error: + raise VendorError( + f"Git tree paths and modes must be valid UTF-8 for {source!r}." + ) from error + _validate_tree_relative_path(path) + try: + relative = PurePosixPath(path).relative_to(source_path).as_posix() + except ValueError as error: + raise VendorError(f"Git tree entry escaped source {source!r}: {path!r}.") from error + if relative in ("", "."): + continue + _validate_tree_relative_path(relative) + if object_type != b"blob" or mode not in ("100644", "100755"): + raise VendorError( + f"Selected upstream trees cannot contain links or special files: {relative}." + ) + if not _matches(relative, patterns): + continue + if relative in seen_paths: + raise VendorError( + f"Git produced a duplicate tree entry for source {source!r}: {relative}." + ) + if re.fullmatch(rb"[0-9a-f]{40}|[0-9a-f]{64}", object_id) is None: + raise VendorError(f"Git produced an invalid object ID for source {source!r}.") + seen_paths.add(relative) + selected.append((relative, object_id, mode == "100755")) + + blobs = _read_git_blobs(repo, [object_id for _, object_id, _ in selected], source) + tree = { + relative: FileEntry(data=data, executable=executable) + for (relative, _, executable), data in zip(selected, blobs, strict=True) + } + if not tree: + raise VendorError(f"Source {source!r} at {commit} selected no files.") + return tree + + +def _local_source_tree(vendor: Vendor, repo: Path) -> Tree: + repository = repo.resolve() + if not repository.is_dir(): + raise VendorError(f"Local upstream repository does not exist: {repository}.") + try: + _run_git( + ["-C", repository, "cat-file", "-e", f"{vendor.commit}^{{commit}}"], cwd=repository + ) + except VendorError as error: + raise VendorError( + f"Local repository {repository} does not contain locked commit {vendor.commit}." + ) from error + return _archive_tree(repository, vendor.commit, vendor.source, vendor.include) + + +@contextlib.contextmanager +def _remote_repository(vendor: Vendor) -> Iterator[Path]: + with tempfile.TemporaryDirectory(prefix="trtllm-vendor-fetch-") as temporary_directory: + repository = Path(temporary_directory) + try: + _run_git(["init", "--quiet"], cwd=repository) + _run_git(["remote", "add", "origin", vendor.url], cwd=repository) + _run_git(["ls-remote", "--quiet", "origin"], cwd=repository) + except VendorError as error: + raise UpstreamUnavailable( + f"Upstream unavailable for {vendor.name!r} at {vendor.url}: {error}" + ) from error + try: + try: + _run_git( + ["fetch", "--quiet", "--depth", "1", "origin", vendor.commit], cwd=repository + ) + except VendorError: + ref = None + if vendor.branch is not None: + ref = f"refs/heads/{vendor.branch}" + elif vendor.tag is not None: + ref = f"refs/tags/{vendor.tag}" + if ref is None: + raise + _run_git(["fetch", "--quiet", "origin", ref], cwd=repository) + _run_git(["cat-file", "-e", f"{vendor.commit}^{{commit}}"], cwd=repository) + except VendorError as error: + raise VendorError( + f"Upstream {vendor.url} is reachable but locked commit {vendor.commit} " + f"could not be fetched: {error}" + ) from error + yield repository + + +def _source_tree(vendor: Vendor, repo: Path | None) -> Tree: + if repo is not None: + return _local_source_tree(vendor, repo) + with _remote_repository(vendor) as remote: + return _archive_tree(remote, vendor.commit, vendor.source, vendor.include) + + +def _store_git_tree(repository: Path, tree: Mapping[str, FileEntry]) -> str: + _run_git(["read-tree", "--empty"], cwd=repository, isolated=True) + index_entries = bytearray() + for relative, entry in sorted(tree.items()): + _validate_tree_relative_path(relative) + object_id = _run_git( + ["hash-object", "-w", "--stdin", "--no-filters"], + cwd=repository, + capture_bytes=True, + isolated=True, + input_bytes=entry.data, + ).strip() + assert isinstance(object_id, bytes) + if re.fullmatch(rb"[0-9a-f]{40}", object_id) is None: + raise VendorError(f"Git produced an invalid blob object ID for {relative!r}.") + mode = b"100755" if entry.executable else b"100644" + index_entries.extend(mode + b" " + object_id + b"\t" + relative.encode("utf-8") + b"\0") + if index_entries: + _run_git( + ["update-index", "-z", "--index-info"], + cwd=repository, + capture_bytes=True, + isolated=True, + input_bytes=bytes(index_entries), + ) + object_id = _run_git(["write-tree"], cwd=repository, isolated=True) + assert isinstance(object_id, str) + object_id = object_id.strip() + if re.fullmatch(r"[0-9a-f]{40}", object_id) is None: + raise VendorError("Git produced an invalid tree object ID.") + return object_id + + +def _generate_patch( + baseline: Mapping[str, FileEntry], result_tree: Mapping[str, FileEntry] +) -> bytes: + with tempfile.TemporaryDirectory(prefix="trtllm-vendor-patch-") as temporary_directory: + repository = Path(temporary_directory) + _run_git(["init", "--quiet", "--object-format=sha1"], cwd=repository, isolated=True) + baseline_object = _store_git_tree(repository, baseline) + result_object = _store_git_tree(repository, result_tree) + # Keep selected attributes out of Git's index fallback during the tree diff. + _run_git(["read-tree", "--empty"], cwd=repository, isolated=True) + patch = _run_git( + [ + "-c", + f"core.attributesFile={os.devnull}", + "diff-tree", + "-p", + "-r", + "--binary", + "--full-index", + "--no-renames", + "--no-ext-diff", + "--no-textconv", + "--unified=3", + "--diff-algorithm=myers", + "--no-indent-heuristic", + baseline_object, + result_object, + ], + cwd=repository, + capture_bytes=True, + isolated=True, + ) + assert isinstance(patch, bytes) + if patch: + reproduced = _apply_patch_bytes(baseline, patch, ("**/*",)) + else: + reproduced = dict(baseline) + expected = dict(result_tree) + if reproduced != expected: + differing = sorted( + path + for path in set(reproduced) | set(expected) + if reproduced.get(path) != expected.get(path) + ) + raise VendorError( + f"Generated vendor patch does not exactly reproduce its result tree: {differing}." + ) + return patch + + +def _apply_patch( + baseline: Mapping[str, FileEntry], + patch_path: Path, + patterns: Sequence[str], +) -> Tree: + with tempfile.TemporaryDirectory(prefix="trtllm-vendor-apply-") as temporary_directory: + directory = Path(temporary_directory) + _write_tree(directory, baseline) + try: + # Stop before Git can inspect a parent that belongs to an enclosing worktree. + _run_git( + ["apply", "--check", "--binary", patch_path], + cwd=directory, + isolated=True, + repository_ceiling=directory.parent, + ) + _run_git( + ["apply", "--binary", patch_path], + cwd=directory, + isolated=True, + repository_ceiling=directory.parent, + ) + except VendorError as error: + raise VendorError(f"Failed to apply vendor patch {patch_path}: {error}") from error + return _read_all_directory_files(directory, patterns) + + +def _normal_tree(lock: LockFile, vendor: Vendor, repo: Path | None) -> Tree: + tree = _source_tree(vendor, repo) + if vendor.patch is None: + return tree + patch_path = _checked_root_path(lock.root, vendor.patch, f"Vendor {vendor.name!r} patch") + return _apply_patch(tree, patch_path, vendor.include) + + +def _current_tree(lock: LockFile, vendor: Vendor) -> Tree: + destination = _checked_root_path( + lock.root, vendor.destination, f"Vendor {vendor.name!r} destination" + ) + return _read_directory_tree(destination, vendor.include) + + +def _changed_paths( + baseline: Mapping[str, FileEntry], result_tree: Mapping[str, FileEntry] +) -> list[str]: + return sorted( + path + for path in set(baseline) | set(result_tree) + if baseline.get(path) != result_tree.get(path) + ) + + +def _select_vendors(lock: LockFile, name: str | None) -> list[Vendor]: + if name is None: + return [lock.vendors[key] for key in sorted(lock.vendors)] + try: + return [lock.vendors[name]] + except KeyError as error: + raise VendorError(f"Unknown vendor: {name!r}.") from error + + +def _require_vendor(lock: LockFile, name: str) -> Vendor: + return _select_vendors(lock, name)[0] + + +def _verify_offline(lock: LockFile, vendor: Vendor) -> None: + actual = _tree_digest(_current_tree(lock, vendor)) + if actual != vendor.digest: + raise VendorError( + f"Vendor {vendor.name!r} destination digest mismatch: " + f"expected {vendor.digest}, got {actual}." + ) + + +def _verify_source(lock: LockFile, vendor: Vendor, repo: Path | None) -> None: + normal = _normal_tree(lock, vendor, repo) + normal_digest = _tree_digest(normal) + if normal_digest != vendor.digest: + raise VendorError( + f"Vendor {vendor.name!r} locked materialization digest mismatch: " + f"expected {vendor.digest}, got {normal_digest}." + ) + current = _current_tree(lock, vendor) + if current != normal: + raise VendorError( + f"Vendor {vendor.name!r} destination differs from its upstream materialization." + ) + + +def _command_list(args: argparse.Namespace) -> None: + lock = _load_lock(args.lock) + for vendor in _select_vendors(lock, None): + state = "patched" if vendor.patch else "exact" + print(f"{vendor.name}\t{state}\t{vendor.commit}\t{vendor.destination}") + + +def _command_status(args: argparse.Namespace) -> None: + lock = _load_lock(args.lock) + failed = False + for vendor in _select_vendors(lock, args.name): + try: + _verify_offline(lock, vendor) + except VendorError as error: + failed = True + print(f"FAIL-LOCAL-INTEGRITY {vendor.name}: {error}") + else: + state = "patched" if vendor.patch is not None else "exact" + print(f"PASS-OFFLINE {vendor.name}: {state} at {vendor.commit}") + if failed: + raise VendorError("One or more vendor snapshots failed offline integrity checks.") + + +def _new_vendor_from_args(args: argparse.Namespace) -> Vendor: + include = args.include if args.include is not None else ["**/*"] + value: dict[str, object] = { + "url": args.url, + "commit": args.commit, + "source": args.source, + "destination": args.destination, + "include": include, + "digest": f"{_DIGEST_PREFIX}{'0' * 64}", + } + if args.branch is not None: + value["branch"] = args.branch + if args.tag is not None: + value["tag"] = args.tag + return _validate_vendor(args.name, value) + + +def _command_create(args: argparse.Namespace) -> None: + lock = _load_lock(args.lock, allow_missing=True) + if args.name in lock.vendors: + raise VendorError(f"Vendor {args.name!r} already exists.") + vendor = _new_vendor_from_args(args) + lock.vendors[vendor.name] = vendor + _validate_destination_layout(lock) + destination = _checked_root_path( + lock.root, vendor.destination, f"Vendor {vendor.name!r} destination" + ) + if args.adopt is None and destination.exists(): + raise VendorError( + f"Vendor {vendor.name!r} destination already exists; use --adopt exact or patched." + ) + source_tree = _source_tree(vendor, args.repo) + + destination_transaction: contextlib.AbstractContextManager[None] = contextlib.nullcontext() + patch_transaction: contextlib.AbstractContextManager[None] = contextlib.nullcontext() + if args.adopt is None: + result_tree = source_tree + destination_transaction = _replace_selected_destination_transaction( + lock, vendor, result_tree + ) + else: + if not destination.exists(): + raise VendorError(f"Cannot adopt missing destination for vendor {vendor.name!r}.") + result_tree = _current_tree(lock, vendor) + if args.adopt == "exact": + if result_tree != source_tree: + changed = _changed_paths(source_tree, result_tree) + raise VendorError( + f"Vendor {vendor.name!r} is not exact; differing files: {changed}. " + "Use --adopt patched to record the difference." + ) + else: + patch = _generate_patch(source_tree, result_tree) + if patch: + patch_relative = f"3rdparty/vendor_patches/{vendor.name}.patch" + patch_path = _checked_root_path( + lock.root, patch_relative, f"Vendor {vendor.name!r} patch" + ) + vendor.patch = patch_relative + vendor.patch_digest = _patch_digest(patch) + patch_transaction = _atomic_file_replacement(patch_path, patch) + vendor.digest = _tree_digest(result_tree) + with contextlib.ExitStack() as transaction: + transaction.enter_context(destination_transaction) + transaction.enter_context(patch_transaction) + _save_lock_checked(lock, f"lock state for new vendor {vendor.name!r}") + print(f"Created {vendor.name} at {vendor.commit} ({'patched' if vendor.patch else 'exact'}).") + + +def _command_sync(args: argparse.Namespace) -> None: + lock = _load_lock(args.lock) + vendor = _require_vendor(lock, args.name) + normal = _normal_tree(lock, vendor, args.repo) + digest = _tree_digest(normal) + if digest != vendor.digest: + raise VendorError( + f"Vendor {vendor.name!r} materializes to {digest}, but the lock records {vendor.digest}. " + "Refresh the patch or pin instead of silently changing the accepted digest." + ) + _replace_selected_destination(lock, vendor, normal) + print(f"Synchronized {vendor.name} to {vendor.commit}.") + + +def _command_check(args: argparse.Namespace) -> None: + if args.require_access and not args.upstream: + raise VendorError("--require-access requires --upstream.") + if args.repo is not None and args.name is None: + raise VendorError("--repo requires one vendor name.") + if args.repo is not None and args.upstream: + raise VendorError("Use either --repo or --upstream, not both.") + if args.repo is not None and args.offline: + raise VendorError("Use either --repo or --offline, not both.") + lock = _load_lock(args.lock) + vendors = _select_vendors(lock, args.name) + for vendor in vendors: + _verify_offline(lock, vendor) + print(f"PASS-OFFLINE {vendor.name}: local snapshot verified; upstream not verified.") + + if args.repo is not None: + vendor = vendors[0] + _verify_source(lock, vendor, args.repo) + print(f"PASS-LOCAL {vendor.name}: verified from {args.repo}.") + return + if not args.upstream: + return + for vendor in vendors: + try: + _verify_source(lock, vendor, None) + except UpstreamUnavailable as error: + if args.require_access: + raise VendorError(f"REMOTE-UNAVAILABLE {vendor.name}: {error}") from error + print(f"SKIP-REMOTE-UNAVAILABLE {vendor.name}: {error}", file=sys.stderr) + continue + print(f"PASS-REMOTE {vendor.name}: verified from {vendor.url}.") + + +def _update_patch(lock: LockFile, vendor: Vendor, repo: Path | None, *, create: bool) -> None: + if create and vendor.patch is not None: + raise VendorError(f"Vendor {vendor.name!r} already has a patch; use refresh.") + if not create and vendor.patch is None: + raise VendorError(f"Vendor {vendor.name!r} has no patch; use create.") + raw = _source_tree(vendor, repo) + current = _current_tree(lock, vendor) + patch = _generate_patch(raw, current) + old_patch = vendor.patch + old_patch_path = ( + None + if old_patch is None + else _checked_root_path(lock.root, old_patch, f"Vendor {vendor.name!r} patch") + ) + old_patch_content = ( + old_patch_path.read_bytes() + if old_patch_path is not None and old_patch_path.is_file() + else None + ) + written_patch_path: Path | None = None + if patch: + patch_relative = old_patch or f"3rdparty/vendor_patches/{vendor.name}.patch" + patch_path = _checked_root_path(lock.root, patch_relative, f"Vendor {vendor.name!r} patch") + _atomic_write(patch_path, patch) + written_patch_path = patch_path + vendor.patch = patch_relative + vendor.patch_digest = _patch_digest(patch) + else: + vendor.patch = None + vendor.patch_digest = None + vendor.digest = _tree_digest(current) + try: + _save_lock(lock) + except (OSError, yaml.YAMLError) as error: + if old_patch_path is not None and old_patch_content is not None: + _atomic_write(old_patch_path, old_patch_content) + elif written_patch_path is not None: + written_patch_path.unlink(missing_ok=True) + raise VendorError(f"Failed to save refreshed patch state: {error}") from error + if not patch and old_patch_path is not None: + old_patch_path.unlink(missing_ok=True) + print(f"{'Created' if create else 'Refreshed'} patch for {vendor.name}.") + + +def _command_patch(args: argparse.Namespace) -> None: + lock = _load_lock(args.lock, allow_patch_recovery=True) + vendor = _require_vendor(lock, args.name) + if args.action == "create": + _update_patch(lock, vendor, args.repo, create=True) + return + if args.action == "refresh": + _update_patch(lock, vendor, args.repo, create=False) + return + if vendor.patch is None: + raise VendorError(f"Vendor {vendor.name!r} has no patch to drop.") + raw = _source_tree(vendor, args.repo) + current = _current_tree(lock, vendor) + if current != raw: + raise VendorError( + f"Vendor {vendor.name!r} destination is not exact upstream; refusing to drop its patch." + ) + patch_path = _checked_root_path(lock.root, vendor.patch, f"Vendor {vendor.name!r} patch") + vendor.patch = None + vendor.patch_digest = None + vendor.digest = _tree_digest(raw) + _save_lock(lock) + patch_path.unlink(missing_ok=True) + print(f"Dropped patch for {vendor.name}.") + + +def _worktree_head(repo: Path) -> str: + result = _run_git(["-C", repo, "rev-parse", "HEAD"], cwd=repo) + assert isinstance(result, str) + return result.strip() + + +def _require_clean_source_worktree(repo: Path, source: str, patterns: Sequence[str]) -> None: + result = _run_git( + ["-C", repo, "status", "--porcelain", "--untracked-files=all", "--", source], + cwd=repo, + ) + assert isinstance(result, str) + if result.strip(): + raise VendorError(f"Local upstream source {source!r} has uncommitted changes.") + ignored = _run_git( + [ + "-C", + repo, + "ls-files", + "-z", + "--others", + "--ignored", + "--exclude-standard", + "--", + source, + ], + cwd=repo, + capture_bytes=True, + ) + assert isinstance(ignored, bytes) + if ignored and not ignored.endswith(b"\0"): + raise VendorError(f"Git produced an invalid ignored-file listing for source {source!r}.") + source_path = PurePosixPath(source) + selected: list[str] = [] + for raw_path in ignored.split(b"\0")[:-1]: + try: + path = raw_path.decode("utf-8") + except UnicodeError as error: + raise VendorError( + f"Ignored paths under local upstream source {source!r} must be valid UTF-8." + ) from error + _validate_tree_relative_path(path) + try: + relative = PurePosixPath(path).relative_to(source_path).as_posix() + except ValueError as error: + raise VendorError( + f"Git ignored-file entry escaped source {source!r}: {path!r}." + ) from error + if relative not in ("", ".") and _matches(relative, patterns): + selected.append(relative) + if selected: + raise VendorError( + f"Local upstream source {source!r} has ignored untracked files selected " + f"for export: {selected}." + ) + + +def _apply_patch_bytes( + baseline: Mapping[str, FileEntry], + patch: bytes, + patterns: Sequence[str], +) -> Tree: + with tempfile.TemporaryDirectory(prefix="trtllm-vendor-delta-") as temporary_directory: + patch_path = Path(temporary_directory) / "delta.patch" + patch_path.write_bytes(patch) + return _apply_patch(baseline, patch_path, patterns) + + +def _command_export(args: argparse.Namespace) -> None: + lock = _load_lock(args.lock) + vendor = _require_vendor(lock, args.name) + repository = args.repo.resolve() + if _worktree_head(repository) != vendor.commit: + raise VendorError( + f"Local upstream HEAD must equal locked commit {vendor.commit} before export." + ) + _require_clean_source_worktree(repository, vendor.source, vendor.include) + raw = _local_source_tree(vendor, repository) + normal = raw + if vendor.patch is not None: + patch_path = _checked_root_path(lock.root, vendor.patch, f"Vendor {vendor.name!r} patch") + normal = _apply_patch(raw, patch_path, vendor.include) + if _tree_digest(normal) != vendor.digest: + raise VendorError( + f"Vendor {vendor.name!r} normal materialization does not match its digest." + ) + current = _current_tree(lock, vendor) + delta = _generate_patch(normal, current) + if not delta: + raise VendorError(f"Vendor {vendor.name!r} has no downstream change to export.") + try: + exported = _apply_patch_bytes(raw, delta, vendor.include) + except VendorError as error: + raise VendorError( + "The local change overlaps the vendor compatibility patch and cannot be exported " + f"automatically: {error}" + ) from error + export_vendor = dataclasses.replace(vendor, destination=vendor.source) + export_lock = LockFile(path=lock.path, root=repository, vendors={vendor.name: export_vendor}) + _replace_selected_destination(export_lock, export_vendor, exported) + print( + f"Exported {vendor.name} changes to {repository / vendor.source}; review and commit them upstream." + ) + + +def _command_pin(args: argparse.Namespace) -> None: + lock = _load_lock(args.lock) + vendor = _require_vendor(lock, args.name) + repository = args.repo.resolve() + commit = args.commit or _worktree_head(repository) + value = vendor.to_mapping() + value["url"] = args.url or vendor.url + value["commit"] = commit + if args.branch is not None: + value["branch"] = args.branch + value.pop("tag", None) + elif args.tag is not None: + value["tag"] = args.tag + value.pop("branch", None) + value["digest"] = vendor.digest + proposed = _validate_vendor(vendor.name, value) + raw = _local_source_tree(proposed, repository) + current = _current_tree(lock, vendor) + drop_patch = False + if proposed.patch is None: + materialized = raw + else: + patch_path = _checked_root_path(lock.root, proposed.patch, f"Vendor {vendor.name!r} patch") + try: + materialized = _apply_patch(raw, patch_path, proposed.include) + except VendorError: + if raw != current: + raise + materialized = raw + drop_patch = True + if materialized != current: + if proposed.patch is not None and raw == current: + materialized = raw + drop_patch = True + else: + changed = _changed_paths(materialized, current) + raise VendorError( + f"New pin does not reproduce the destination for {vendor.name!r}; " + f"differing files: {changed}." + ) + old_patch = proposed.patch + if drop_patch: + proposed.patch = None + proposed.patch_digest = None + proposed.digest = _tree_digest(materialized) + lock.vendors[vendor.name] = proposed + patch_path: Path | None = None + if drop_patch and old_patch is not None: + patch_path = _checked_root_path(lock.root, old_patch, f"Vendor {vendor.name!r} patch") + if patch_path.is_symlink() or not patch_path.is_file(): + raise VendorError(f"Generated patch path must be a regular file: {patch_path}.") + _save_lock_checked(lock, f"new pin for vendor {vendor.name!r}") + if patch_path is not None: + try: + _sync_directory(lock.path.parent) + except OSError as error: + raise VendorError( + f"Failed to make the new pin durable for vendor {vendor.name!r}: {error}" + ) from error + try: + patch_path.unlink() + except OSError as error: + print( + f"warning: Pinned vendor {vendor.name!r}, but could not remove its absorbed " + f"patch {patch_path}: {error}. The lock is committed and valid; this patch is " + "now an unreferenced orphan. Delete that file manually.", + file=sys.stderr, + ) + print(f"Pinned {vendor.name} to {commit}; offline enforcement restored.") + + +def _command_remove(args: argparse.Namespace) -> None: + lock = _load_lock(args.lock) + vendor = _require_vendor(lock, args.name) + del lock.vendors[vendor.name] + _save_lock(lock) + if vendor.patch is not None and all( + other.patch != vendor.patch for other in lock.vendors.values() + ): + _checked_root_path(lock.root, vendor.patch, f"Vendor {vendor.name!r} patch").unlink( + missing_ok=True + ) + print(f"Removed {vendor.name} from the lock; destination bytes were preserved.") + + +def _add_repo_argument(parser: argparse.ArgumentParser, *, required: bool = False) -> None: + parser.add_argument( + "--repo", + type=Path, + required=required, + help="Local Git checkout containing the locked commit; never contacts the lock URL.", + ) + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--lock", + type=Path, + default=_DEFAULT_LOCK, + help=f"Vendor lock path (default: {_DEFAULT_LOCK.relative_to(_REPO_ROOT)}).", + ) + commands = parser.add_subparsers(dest="command", required=True) + + list_parser = commands.add_parser("list", help="List configured vendors.") + list_parser.set_defaults(handler=_command_list) + + status = commands.add_parser("status", help="Show offline vendor status.") + status.add_argument("name", nargs="?") + status.set_defaults(handler=_command_status) + + create = commands.add_parser("create", help="Create or adopt a vendor entry.") + create.add_argument("name") + create.add_argument("--url", required=True) + reference = create.add_mutually_exclusive_group() + reference.add_argument("--branch") + reference.add_argument("--tag") + create.add_argument("--commit", required=True) + create.add_argument("--source", required=True) + create.add_argument("--destination", required=True) + create.add_argument("--include", action="append") + create.add_argument("--adopt", choices=("exact", "patched")) + _add_repo_argument(create) + create.set_defaults(handler=_command_create) + + sync = commands.add_parser("sync", help="Restore a locked vendor materialization.") + sync.add_argument("name") + _add_repo_argument(sync) + sync.set_defaults(handler=_command_sync) + + check = commands.add_parser("check", help="Verify vendor integrity.") + check.add_argument("name", nargs="?") + check_mode = check.add_mutually_exclusive_group() + check_mode.add_argument( + "--offline", + action="store_true", + help="Explicitly select the default no-network integrity check.", + ) + check_mode.add_argument( + "--upstream", action="store_true", help="Attempt upstream verification." + ) + check.add_argument( + "--require-access", + action="store_true", + help="Fail when an upstream is inaccessible; for trusted maintainer environments.", + ) + _add_repo_argument(check) + check.set_defaults(handler=_command_check) + + patch = commands.add_parser("patch", help="Create, refresh, or drop a downstream patch.") + patch.add_argument("name") + patch.add_argument("action", choices=("create", "refresh", "drop")) + _add_repo_argument(patch) + patch.set_defaults(handler=_command_patch) + + export = commands.add_parser("export", help="Export pending destination changes upstream.") + export.add_argument("name") + _add_repo_argument(export, required=True) + export.set_defaults(handler=_command_export) + + pin = commands.add_parser("pin", help="Pin a vendor to a new committed source revision.") + pin.add_argument("name") + pin.add_argument("--url") + pin_reference = pin.add_mutually_exclusive_group() + pin_reference.add_argument("--branch") + pin_reference.add_argument("--tag") + pin.add_argument("--commit") + _add_repo_argument(pin, required=True) + pin.set_defaults(handler=_command_pin) + + remove = commands.add_parser( + "remove", help="Remove a vendor lock entry, preserving its destination." + ) + remove.add_argument("name") + remove.set_defaults(handler=_command_remove) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + """Run the vendoring command-line interface.""" + parser = _build_parser() + args = parser.parse_args(argv) + if args.lock != _DEFAULT_LOCK: + args.lock = args.lock.resolve() + try: + args.handler(args) + except VendorError as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/vendor/metadata.py b/scripts/vendor/metadata.py new file mode 100644 index 000000000000..ddcaac7bd2df --- /dev/null +++ b/scripts/vendor/metadata.py @@ -0,0 +1,302 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Strict contributor metadata and conservative upstream commit attribution. + +Matching establishes provenance, not permission to drop changes during refresh. +Source code and commit messages are data and are never executed. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path + +import yaml + +if __package__: + from . import manage, promote +else: + import manage + import promote + +BEGIN = "" +END = "" + + +def template(vendor_name: str, upstream_repo: str) -> str: + """Return a copyable PR-description block; repository settings stay operator-owned.""" + return ( + f"{BEGIN}\n```yaml\nschema_version: 1\nvendors:\n {vendor_name}:\n" + f" upstream_prs:\n - https://github.com/{upstream_repo}/pull/123\n" + f"```\n{END}" + ) + + +def parse(body: str, vendor_name: str, upstream_repo: str) -> dict: + """Read exactly one marked block and validate the selected vendor's inputs.""" + if len(body) > 65536: + raise ValueError("PR description exceeds the supported metadata size.") + if body.count(BEGIN) != 1 or body.count(END) != 1: + raise ValueError("PR description must contain exactly one vendor-promotion marked block.") + start, end = body.index(BEGIN) + len(BEGIN), body.index(END) + block = body[start:end].strip() + if start >= end or not block.startswith("```yaml\n") or not block.endswith("\n```"): + raise ValueError("The marked block must contain one fenced yaml mapping.") + try: + content = block[len("```yaml\n") : -len("\n```")] + if any( + isinstance(token, (yaml.tokens.AliasToken, yaml.tokens.AnchorToken)) + for token in yaml.scan(content) + ): + raise ValueError("Promotion YAML aliases and anchors are not allowed.") + data = yaml.load(content, Loader=manage._UniqueKeyLoader) + except yaml.YAMLError as error: + raise ValueError(f"Invalid promotion YAML: {error}") from error + if not isinstance(data, dict) or set(data) != {"schema_version", "vendors"}: + raise ValueError("Promotion YAML requires only schema_version and vendors.") + if type(data["schema_version"]) is not int or data["schema_version"] != 1: + raise ValueError("Unsupported promotion schema_version; expected 1.") + entries = data["vendors"] + if not isinstance(entries, dict) or vendor_name not in entries: + raise ValueError(f"Promotion YAML must include vendors.{vendor_name}.") + if any( + not isinstance(name, str) or not manage._NAME_PATTERN.fullmatch(name) for name in entries + ): + raise ValueError("Invalid vendor key in promotion YAML.") + entry = entries[vendor_name] + if not isinstance(entry, dict) or set(entry) - { + "upstream_prs", + "unpaired_reason", + "resolutions", + }: + raise ValueError( + "Vendor metadata allows upstream_prs, unpaired_reason, and resolutions only." + ) + if "unpaired_reason" in entry: + if ( + set(entry) != {"unpaired_reason"} + or not isinstance(entry["unpaired_reason"], str) + or not entry["unpaired_reason"].strip() + ): + raise ValueError("unpaired_reason must be nonempty and cannot accompany upstream PRs.") + return { + "upstream_prs": [], + "resolutions": {}, + "unpaired_reason": entry["unpaired_reason"].strip(), + } + references = entry.get("upstream_prs") + if not isinstance(references, list) or not references: + raise ValueError("upstream_prs must be a nonempty list of upstream PR URLs.") + numbers = [] + for reference in references: + if not isinstance(reference, str) or not reference.startswith( + f"https://github.com/{upstream_repo}/pull/" + ): + raise ValueError(f"Each upstream PR must be a URL in {upstream_repo}.") + numbers.append(promote._pr_number(reference, upstream_repo)) + if len(set(numbers)) != len(numbers): + raise ValueError("upstream_prs contains duplicate PRs.") + resolutions = entry.get("resolutions", {}) + if not isinstance(resolutions, dict): + raise ValueError("resolutions must be a mapping keyed by full source commit SHA.") + normalized = {} + for sha, resolution in resolutions.items(): + promote._sha(sha) + if not isinstance(resolution, dict) or set(resolution) != {"upstream_pr", "reason"}: + raise ValueError("Each resolution requires upstream_pr and reason.") + reference, reason = resolution["upstream_pr"], resolution["reason"] + if not isinstance(reference, str): + raise ValueError("Resolution upstream_pr must be a PR URL.") + number = promote._pr_number(reference, upstream_repo) + if number not in numbers or not isinstance(reason, str) or not reason.strip(): + raise ValueError("Resolution must name a listed PR and explain the unmatched patch.") + normalized[sha] = {"upstream_pr": number, "reason": reason.strip()} + return {"upstream_prs": sorted(numbers), "resolutions": normalized, "unpaired_reason": None} + + +def fingerprint(value: dict) -> str: + """Hash normalized metadata without recording credentials or unrelated PR prose.""" + return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest() + + +def _patch(repo: Path, base: str, head: str) -> str: + """Normalize hunk coordinates, retaining whitespace, paths, modes, and binary data.""" + diff = promote._run( + [ + "git", + "-C", + str(repo), + "diff", + "--no-ext-diff", + "--no-textconv", + "--no-renames", + "--binary", + "--full-index", + "--no-color", + "--unified=3", + base, + head, + "--", + ], + strip=False, + binary_output=True, + ) + # Blob IDs and hunk positions change on rebases. Keep surrounding lines and + # function context so identical edits at different code sites do not match. + return "\n".join( + re.sub(r"^@@ -[0-9]+(?:,[0-9]+)? \+[0-9]+(?:,[0-9]+)? @@", "@@", line) + for line in diff.split("\n") + if not line.startswith("index ") + ) + + +def _commits(repo: Path, base: str, head: str) -> list[str]: + return promote._git(repo, "rev-list", "--reverse", f"{base}..{head}").splitlines() + + +def match( + repo: Path, + previous: str, + reviewed: str, + upstream_prs: dict[int, dict], + resolutions: dict[str, dict] | None = None, +) -> tuple[dict[str, int], dict[str, dict]]: + """Match a linear source range against fetched immutable upstream PR snapshots. + + Exact edits are compared across all files, not only selected vendored files. + Ambiguities remain author-actionable; titles and cherry-pick trailers are not proof. + """ + commits = _commits(repo, previous, reviewed) + if not commits: + raise ValueError("Source update has no new commits to attribute.") + for sha in commits: + if len(promote._git(repo, "show", "-s", "--format=%P", sha).split()) != 1: + raise ValueError("Source history must be linear; rebase merge commits first.") + resolutions = resolutions or {} + if set(resolutions) - set(commits): + raise ValueError("Resolutions include commits outside the reviewed source delta.") + candidates: dict[str, set[int]] = {sha: set() for sha in commits} + patches = {sha: _patch(repo, f"{sha}^", sha) for sha in commits} + methods: dict[tuple[str, int], str] = {} + for number, pr in upstream_prs.items(): + base, head = promote._sha(pr["base"]["sha"]), promote._sha(pr["head"]["sha"]) + merge_base = promote._git(repo, "merge-base", base, head) + upstream_commits = _commits(repo, merge_base, head) + upstream_patches = set() + for sha in upstream_commits: + parents = promote._git(repo, "show", "-s", "--format=%P", sha).split() + if len(parents) == 1: + upstream_patches.add(_patch(repo, parents[0], sha)) + aggregate = _patch(repo, merge_base, head) + for sha in commits: + if sha in upstream_commits: + candidates[sha].add(number) + methods[sha, number] = "same-commit" + elif patches[sha] and patches[sha] in upstream_patches: + candidates[sha].add(number) + methods[sha, number] = "same-edits" + elif patches[sha] and patches[sha] == aggregate: + candidates[sha].add(number) + methods[sha, number] = "upstream-aggregate" + # A whole reviewed series may be squashed into one upstream PR. Do not + # invent individual correspondences: identify the shared group explicitly. + if aggregate and _patch(repo, previous, reviewed) == aggregate: + for sha in commits: + candidates[sha].add(number) + methods.setdefault((sha, number), "source-series-aggregate") + assignments, evidence, unresolved = {}, {}, [] + for sha in commits: + if sha in resolutions: + resolution = resolutions[sha] + number = resolution["upstream_pr"] + if number not in upstream_prs: + raise ValueError("Resolution names an unlisted upstream PR.") + assignments[sha] = number + evidence[sha] = { + "method": "author-assertion", + "reason": resolution["reason"], + "refresh_policy": "retain-until-verified", + } + elif len(candidates[sha]) == 1: + number = next(iter(candidates[sha])) + assignments[sha] = number + evidence[sha] = { + "method": methods[sha, number], + "upstream_head": upstream_prs[number]["head"]["sha"], + "upstream_base": upstream_prs[number]["base"]["sha"], + } + else: + options = ( + ", ".join(f"#{number}" for number in sorted(candidates[sha])) or "no exact match" + ) + unresolved.append(f"- `{sha}`: {options}") + if unresolved: + raise ValueError( + "Author action required: upstream attribution is unresolved. Add missing upstream PRs, " + "update/rebase the source pin, or add a targeted resolution with an upstream_pr and " + "reason for each commit below. No complete commit_map is required.\n" + + "\n".join(unresolved) + ) + if set(assignments.values()) != set(upstream_prs): + raise ValueError( + "Some listed upstream PRs cover no new source commits; remove unrelated PRs." + ) + return assignments, evidence + + +def resolve( + gh: promote.GitHub, + previous: manage.Vendor, + reviewed: manage.Vendor, + entry: dict, + cache: Path, +) -> tuple[dict[str, int], dict[str, dict]]: + """Fetch only Git data into a private bare cache and resolve declared PRs.""" + cache.mkdir(parents=True, exist_ok=True) + if not (cache / "HEAD").exists(): + promote._git(cache, "init", "--bare", "--quiet") + if promote._git(cache, "rev-parse", "--is-bare-repository") != "true": + raise ValueError("Matching cache must be a dedicated bare repository.") + promote._check_source_repo(gh, promote._repo_name(reviewed.url)) + for sha in (previous.commit, reviewed.commit): + promote._git(cache, "fetch", "--quiet", "--no-tags", reviewed.url, sha) + try: + common = promote._git(cache, "merge-base", previous.commit, reviewed.commit) + except RuntimeError as error: + raise ValueError( + "Source revisions have no merge base; rebase the temporary source branch onto the canonical pin." + ) from error + if common != previous.commit: + raise ValueError( + "Source update is not a fast-forward; rebase the temporary source branch onto the canonical pin." + ) + if entry["unpaired_reason"]: + commits = _commits(cache, previous.commit, reviewed.commit) + if not commits or any( + len(promote._git(cache, "show", "-s", "--format=%P", sha).split()) != 1 + for sha in commits + ): + raise ValueError("Unpaired source updates still require a nonempty linear history.") + return {}, {} + prs = {} + for number in entry["upstream_prs"]: + pr = gh.get(f"repos/{gh.upstream_repo}/pulls/{number}") + if pr["base"]["ref"] != gh.upstream_branch or ( + pr["state"] == "closed" and not pr.get("merged") + ): + raise ValueError( + f"Upstream PR #{number} must target {gh.upstream_branch} and be open or merged." + ) + for sha in (pr["base"]["sha"], pr["head"]["sha"]): + promote._git( + cache, + "fetch", + "--quiet", + "--no-tags", + f"https://github.com/{gh.upstream_repo}.git", + promote._sha(sha), + ) + prs[number] = pr + return match(cache, previous.commit, reviewed.commit, prs, entry["resolutions"]) diff --git a/scripts/vendor/promote.py b/scripts/vendor/promote.py new file mode 100644 index 000000000000..cac87271bb6d --- /dev/null +++ b/scripts/vendor/promote.py @@ -0,0 +1,1105 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Promote reviewed vendor source revisions; dry-run unless --publish is supplied. + +Only Git, gh, and PyYAML are required. GitHub reads and temporary source fetches +are allowed in dry-run; remote writes and consumer worktrees require --publish. +Provenance lives in signed-off commits, not in a separate tracking file. +""" + +from __future__ import annotations + +import argparse +import base64 +import contextlib +import json +import os +import re +import subprocess +import sys +import tempfile +import time +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import urlencode + +import yaml + +if __package__: + from . import manage as vendor +else: + import manage as vendor + +_LOCK = "3rdparty/vendor_sources.lock.yaml" +_BEGIN = "----- BEGIN VENDOR PROMOTION V1 -----" +_END = "----- END VENDOR PROMOTION V1 -----" +_SHA = re.compile(r"[0-9a-f]{40}\Z") +_CI_SKIP_COMMAND = '/bot skip --comment "skip CI since no code change"' + + +def _run( + command: list[str], + *, + cwd: Path | None = None, + text: str | None = None, + timeout: int = 300, + stream: bool = False, + strip: bool = True, + binary_output: bool = False, +) -> str: + # Do not use vendor_sources._run_git here: its synthetic identity is for + # patch generation. Real promotion commits must use the maintainer's identity. + env = os.environ.copy() + for key in vendor._GIT_LOCAL_ENVIRONMENT_VARIABLES: + env.pop(key, None) + env.setdefault("GH_CONFIG_DIR", str(Path.home() / ".config/gh")) + env.update(GIT_TERMINAL_PROMPT="0", GH_PROMPT_DISABLED="1", GIT_NO_REPLACE_OBJECTS="1") + if command[0] == "git": + # Lock-only worktrees need source text, not unrelated LFS assets. + env["GIT_LFS_SKIP_SMUDGE"] = "1" + result = subprocess.run( + command, + cwd=cwd, + env=env, + input=text.encode() if binary_output and text is not None else text, + text=not binary_output, + capture_output=not stream, + timeout=timeout, + ) + if result.returncode: + detail = (result.stderr or result.stdout or "See streamed output above.").strip() + raise RuntimeError(f"{command[0]} failed: {detail}") + output = result.stdout or "" + if isinstance(output, bytes): + output = output.decode("utf-8", errors="surrogateescape") + return output.strip() if strip else output + + +def _git(repo: Path, *args: str, text: str | None = None) -> str: + return _run( + ["git", "-C", str(repo), *args], + text=text, + timeout=1800 if args[:2] == ("worktree", "add") else 300, + stream=args[:2] == ("worktree", "add") or args[0] in ("fetch", "commit", "push"), + ) + + +class GitHub: + """Small gh-backed client; respects GH_CONFIG_DIR without handling tokens.""" + + def __init__( + self, + consumer_repo: str = "NVIDIA/TensorRT-LLM", + vendor_name: str = "", + upstream_repo: str = "", + base_branch: str = "main", + upstream_branch: str = "main", + ) -> None: + self.consumer_repo = _repo_name(f"https://github.com/{consumer_repo}") + self.vendor_name = vendor_name + self.upstream_repo = upstream_repo + self.base_branch = base_branch + self.upstream_branch = upstream_branch + + def api( + self, endpoint: str, *, method: str = "GET", payload: dict | None = None + ) -> dict | list: + command = ["gh", "api", "--hostname", "github.com", "--method", method, endpoint] + if payload is not None: + command += ["--input", "-"] + result = json.loads(_run(command, text=None if payload is None else json.dumps(payload))) + if not isinstance(result, (dict, list)): + raise ValueError(f"Unexpected GitHub response for {endpoint}") + return result + + def get(self, endpoint: str) -> dict: + result = self.api(endpoint) + if not isinstance(result, dict): + raise ValueError(f"Expected an object from {endpoint}") + return result + + def optional(self, endpoint: str) -> dict | None: + try: + return self.get(endpoint) + except RuntimeError as error: + if "HTTP 404" not in str(error): + raise + return None + + def compare(self, repo: str, base: str, head: str, *, all_commits: bool = False) -> dict: + first = self.get(f"repos/{repo}/compare/{base}...{head}?per_page=100&page=1") + if all_commits: + count = int(first["total_commits"]) + for page in range(2, (count + 99) // 100 + 1): + more = self.get(f"repos/{repo}/compare/{base}...{head}?per_page=100&page={page}") + first["commits"].extend(more["commits"]) + if len(first["commits"]) != count: + raise ValueError("Incomplete GitHub comparison; refusing partial provenance.") + return first + + +def _sha(value: str) -> str: + if not isinstance(value, str) or not _SHA.fullmatch(value): + raise ValueError(f"Expected a full immutable commit SHA, got {value!r}.") + return value + + +def _repo_name(url: str) -> str: + match = re.fullmatch( + r"(?:https://github\.com/|git@github\.com:)([\w.-]+/[\w.-]+?)(?:\.git)?/?", url + ) + if not match: + raise ValueError(f"Expected a credential-free GitHub repository URL, got {url!r}.") + return match[1] + + +def _pr_number(value: str, repo: str) -> int: + for prefix in (f"https://github.com/{repo}/pull/", f"{repo}#"): + if value.startswith(prefix): + value = value[len(prefix) :] + break + if not re.fullmatch(r"[1-9][0-9]*", value): + raise ValueError(f"Expected a PR number or PR URL in {repo}: {value!r}.") + return int(value) + + +def _lock_document_at(gh: GitHub, repo: str, commit: str) -> dict: + response = gh.get(f"repos/{repo}/contents/{_LOCK}?ref={_sha(commit)}") + if response.get("encoding") != "base64": + raise ValueError("GitHub did not return the complete vendor lock.") + try: + content = base64.b64decode("".join(response["content"].split()), validate=True) + if any( + isinstance(token, (yaml.tokens.AliasToken, yaml.tokens.AnchorToken)) + for token in yaml.scan(content) + ): + raise ValueError("Vendor lock YAML aliases and anchors are not allowed.") + data = yaml.load(content, Loader=vendor._UniqueKeyLoader) + except yaml.YAMLError as error: + raise ValueError(f"Invalid vendor lock YAML: {error}") from error + if ( + not isinstance(data, dict) + or set(data) != {"schema_version", "vendors"} + or type(data["schema_version"]) is not int + or data["schema_version"] != 1 + or not isinstance(data["vendors"], dict) + ): + raise ValueError("Unsupported vendor lock schema.") + return data + + +def _lock_at(gh: GitHub, repo: str, commit: str) -> vendor.Vendor: + data = _lock_document_at(gh, repo, commit) + return vendor._validate_vendor(gh.vendor_name, data["vendors"][gh.vendor_name]) + + +def _main_sha(gh: GitHub) -> str: + return _sha(gh.get(f"repos/{gh.consumer_repo}/git/ref/heads/{gh.base_branch}")["object"]["sha"]) + + +def _check_fork(gh: GitHub, repo: str, upstream: str) -> None: + info = gh.get(f"repos/{repo}") + if ( + repo.lower() == upstream.lower() + or info.get("source", {}).get("full_name", "").lower() != upstream.lower() + ): + raise ValueError(f"{repo} must be a personal fork of {upstream}, not upstream itself.") + + +def _check_source_repo(gh: GitHub, repo: str) -> None: + if repo.lower() != gh.upstream_repo.lower(): + _check_fork(gh, repo, gh.upstream_repo) + + +def _record_from_message(message: str) -> dict: + legacy = "----- BEGIN PRIMTS PROMOTION V1 -----" + if legacy in message: + message = message.replace(legacy, _BEGIN).replace( + "----- END PRIMTS PROMOTION V1 -----", _END + ) + if message.count(_BEGIN) != 1 or message.count(_END) != 1: + raise ValueError("Missing or ambiguous Vendor promotion metadata in commit message.") + record = json.loads(message.split(_BEGIN, 1)[1].split(_END, 1)[0]) + if not isinstance(record, dict) or record.get("version") != 1: + raise ValueError("Unsupported Vendor promotion metadata version.") + if "trtllm_pr" in record: + record["consumer_pr"] = record.pop("trtllm_pr") + record["consumer_merge"] = record.pop("trtllm_merge") + record["upstream_repository"] = "flashinfer-ai/flashinfer" + record["consumer_base"] = "main" + record["upstream_branch"] = "main" + return record + + +def _record_text(record: dict) -> str: + return f"{_BEGIN}\n{json.dumps(record, indent=2, sort_keys=True)}\n{_END}" + + +def _map_commits( + commits: list[dict], prs: dict[int, dict], mappings: list[str], upstream_repo: str +) -> dict[str, int]: + assignments: dict[str, int] = {} + for mapping in mappings: + prefix, separator, reference = mapping.partition("=") + matches = [item["sha"] for item in commits if item["sha"].startswith(prefix)] + if not separator or not re.fullmatch(r"[0-9a-f]{7,40}", prefix) or len(matches) != 1: + raise ValueError(f"Ambiguous or invalid --map-upstream {mapping!r}.") + number = _pr_number(reference, upstream_repo) + if number not in prs or matches[0] in assignments: + raise ValueError(f"Duplicate mapping or PR missing from --upstream-pr: {mapping!r}.") + assignments[matches[0]] = number + for item in commits: + sha = item["sha"] + if sha in assignments: + continue + if len(prs) == 1: + assignments[sha] = next(iter(prs)) + elif sys.stdin.isatty(): + title = item["commit"]["message"].splitlines()[0] + answer = input(f"Upstream PR for {sha[:12]} ({title}), choose {sorted(prs)}: ") + number = _pr_number(answer, upstream_repo) + if number not in prs: + raise ValueError("Answer must identify a supplied upstream PR.") + assignments[sha] = number + else: + raise ValueError(f"Provide --map-upstream {sha}=PR for every unmapped commit.") + if set(assignments.values()) != set(prs): + raise ValueError("Every supplied upstream PR must map to at least one new source commit.") + return assignments + + +@dataclass +class Promotion: + """An immutable source revision and the expected consumer/canonical states.""" + + number: int + main_sha: str + previous: vendor.Vendor + reviewed: vendor.Vendor + canonical_repo: str + fork: str + record: dict + consumer_repo: str = "NVIDIA/TensorRT-LLM" + base_branch: str = "main" + legacy_prims_ts: bool = False + completed_pr: dict | None = None + + @property + def branch(self) -> str: + name = "prims-ts" if self.legacy_prims_ts else self.previous.name + return f"chore/{name}-promote-{self.number}-{self.reviewed.commit[:12]}" + + @property + def title(self) -> str: + return f"[None][chore] promote {self.previous.name} source from #{self.number}" + + @property + def promoted(self) -> dict: + result = self.reviewed.to_mapping() + result.update(url=self.previous.url, branch=self.previous.branch) + return result + + +def _check_current(gh: GitHub, plan: Promotion) -> None: + current = _lock_at(gh, gh.consumer_repo, _main_sha(gh)).to_mapping() + if current not in (plan.reviewed.to_mapping(), plan.promoted): + raise ValueError( + "Consumer branch's vendor lock has been superseded; do not promote this PR." + ) + ref = gh.get(f"repos/{plan.canonical_repo}/git/ref/heads/{plan.previous.branch}") + if ref["object"]["sha"] not in (plan.previous.commit, plan.reviewed.commit): + raise ValueError( + "Canonical branch moved unexpectedly; coordinate with the other maintainer." + ) + + +def _make_plan( + args: argparse.Namespace, + gh: GitHub, + *, + auto_match: bool = False, + resolutions: dict[str, dict] | None = None, +) -> Promotion: + if args.unpaired_reason is not None: + if not args.unpaired_reason.strip() or args.upstream_pr or args.map_upstream: + raise ValueError( + "--unpaired-reason requires a nonempty reason and cannot be combined " + "with --upstream-pr or --map-upstream." + ) + elif not args.upstream_pr: + raise ValueError("Provide --upstream-pr or an explicit --unpaired-reason.") + number = _pr_number(args.source_pr, gh.consumer_repo) + pr = gh.get(f"repos/{gh.consumer_repo}/pulls/{number}") + if not pr.get("merged") or pr["base"]["ref"] != gh.base_branch: + raise ValueError( + "--source-pr must identify a PR already merged into the consumer base branch." + ) + merge_sha = _sha(pr["merge_commit_sha"]) + merge = gh.get(f"repos/{gh.consumer_repo}/commits/{merge_sha}") + previous = _lock_at(gh, gh.consumer_repo, merge["parents"][0]["sha"]) + reviewed = _lock_at(gh, gh.consumer_repo, merge_sha) + canonical_repo = _repo_name(previous.url) + source_repo = _repo_name(reviewed.url) + expected_repo = _repo_name(f"https://github.com/{args.canonical_repo}") + _run(["git", "check-ref-format", f"refs/heads/{args.canonical_branch}"]) + if canonical_repo.lower() != expected_repo.lower() or previous.branch != args.canonical_branch: + raise ValueError( + "Previous lock does not match the explicit canonical repository/branch; " + "finish the preceding promotion first." + ) + if ( + previous.commit == reviewed.commit + or not reviewed.branch + or (canonical_repo.lower() == source_repo.lower() and reviewed.branch == previous.branch) + ): + raise ValueError("The PR must introduce a new source SHA on a temporary branch.") + fork = args.fork or _repo_name(_git(args.repo, "remote", "get-url", "fork")) + record = { + "version": 1, + "kind": "promotion", + "vendor": gh.vendor_name, + "consumer_pr": f"https://github.com/{gh.consumer_repo}/pull/{number}", + "consumer_merge": merge_sha, + "consumer_base": gh.base_branch, + "upstream_repository": gh.upstream_repo, + "upstream_branch": gh.upstream_branch, + "canonical_url": previous.url, + "canonical_branch": previous.branch, + "previous": previous.commit, + "promoted": reviewed.commit, + "source_url": reviewed.url, + "source_branch": reviewed.branch, + } + plan = Promotion( + number, + _main_sha(gh), + previous, + reviewed, + canonical_repo, + fork, + record, + consumer_repo=gh.consumer_repo, + base_branch=gh.base_branch, + legacy_prims_ts=args.legacy_prims_ts, + ) + # A completed operation is historical: later promotions, deleted temporary + # forks, or changed upstream PR state must not make it run again or fail. + existing = _find_followup(gh, plan) + if existing: + existing = gh.get(f"repos/{gh.consumer_repo}/pulls/{existing['number']}") + if existing.get("merged"): + _verify_remote_commit(gh, plan, gh.consumer_repo, existing["merge_commit_sha"]) + plan.completed_pr = existing + return plan + _check_source_repo(gh, canonical_repo) + _check_source_repo(gh, source_repo) + _check_fork(gh, fork, gh.consumer_repo) + comparison = gh.compare(canonical_repo, previous.commit, reviewed.commit, all_commits=True) + if comparison["merge_base_commit"]["sha"] != previous.commit: + raise ValueError( + "Reviewed source is not a fast-forward of the previous canonical revision." + ) + commits = comparison["commits"] + if not commits or any(len(item["parents"]) != 1 for item in commits): + raise ValueError( + "Promote requires a nonempty, linear source commit range; rebase merges first." + ) + prs = {} + for reference in args.upstream_pr: + upstream_number = _pr_number(reference, gh.upstream_repo) + if upstream_number in prs: + raise ValueError("Duplicate --upstream-pr.") + upstream = gh.get(f"repos/{gh.upstream_repo}/pulls/{upstream_number}") + if upstream["base"]["ref"] != gh.upstream_branch or ( + upstream["state"] == "closed" and not upstream.get("merged") + ): + raise ValueError( + f"Upstream PR must target {gh.upstream_branch} and be open (draft allowed) or merged." + ) + prs[upstream_number] = upstream + if auto_match and prs: + if __package__: + from . import metadata + else: + import metadata + with tempfile.TemporaryDirectory(prefix="vendor-attribution-") as directory: + entry = { + "upstream_prs": list(prs), + "unpaired_reason": None, + "resolutions": resolutions or {}, + } + assignments, evidence = metadata.resolve(gh, previous, reviewed, entry, Path(directory)) + record["attribution"] = evidence + else: + assignments = _map_commits(commits, prs, args.map_upstream, gh.upstream_repo) if prs else {} + upstream_main = _sha( + gh.get(f"repos/{gh.upstream_repo}/git/ref/heads/{gh.upstream_branch}")["object"]["sha"] + ) + upstream_base = gh.compare(canonical_repo, previous.commit, upstream_main)["merge_base_commit"][ + "sha" + ] + groups = [] + if args.unpaired_reason is not None: + groups.append( + { + "commits": [item["sha"] for item in commits], + "upstream_pr": None, + "upstream_base": None, + "upstream_head": None, + "upstream_merge": None, + "unpaired_reason": args.unpaired_reason.strip(), + "refresh_policy": "retain", + } + ) + for upstream_number, upstream in prs.items(): + groups.append( + { + "commits": [ + item["sha"] for item in commits if assignments[item["sha"]] == upstream_number + ], + "upstream_pr": f"https://github.com/{gh.upstream_repo}/pull/{upstream_number}", + "upstream_base": _sha(upstream["base"]["sha"]), + "upstream_head": _sha(upstream["head"]["sha"]), + "upstream_merge": upstream.get("merge_commit_sha") + if upstream.get("merged") + else None, + } + ) + record.update( + { + "upstream_base": _sha(upstream_base), + "upstream_observed_main": upstream_main, + "changes": groups, + } + ) + _check_current(gh, plan) + return plan + + +def _find_followup(gh: GitHub, plan: Promotion) -> dict | None: + query = urlencode( + { + "state": "all", + "base": plan.base_branch, + "head": f"{plan.fork.split('/')[0]}:{plan.branch}", + } + ) + prs = gh.api(f"repos/{gh.consumer_repo}/pulls?{query}") + if not isinstance(prs, list) or len(prs) > 1: + raise ValueError("Ambiguous existing promotion PRs.") + if prs and prs[0]["state"] == "closed" and not prs[0].get("merged_at"): + raise ValueError("The promotion PR was closed without merging; inspect it before retrying.") + return prs[0] if prs else None + + +def _adopt_record(plan: Promotion, message: str) -> None: + """Keep the original upstream snapshot when resuming an already-published run.""" + existing = _record_from_message(message) + stable_keys = set(plan.record) - {"upstream_base", "upstream_observed_main", "changes"} + if any(existing.get(key) != plan.record[key] for key in stable_keys): + raise ValueError("Existing promotion metadata does not describe this source update.") + _validate_snapshot(existing, plan.record["upstream_repository"]) + if "changes" not in plan.record: + # Only the read-only, already-merged path restores an entire snapshot. + plan.record = existing + return + expected = { + group["upstream_pr"]: ( + group["commits"], + group.get("unpaired_reason"), + group.get("refresh_policy"), + ) + for group in plan.record["changes"] + } + actual = { + group["upstream_pr"]: ( + group["commits"], + group.get("unpaired_reason"), + group.get("refresh_policy"), + ) + for group in existing["changes"] + } + if actual != expected: + raise ValueError("Existing promotion has different upstream PR/commit mappings.") + plan.record = existing + + +def _validate_snapshot(record: dict, upstream_repo: str) -> None: + _sha(record["upstream_base"]) + _sha(record["upstream_observed_main"]) + commits = [] + if not isinstance(record["changes"], list) or not record["changes"]: + raise ValueError("Promotion metadata must contain source changes.") + for group in record["changes"]: + if not isinstance(group["commits"], list) or not group["commits"]: + raise ValueError("Promotion change group has no commits.") + commits.extend(_sha(sha) for sha in group["commits"]) + if group["upstream_pr"] is None: + if ( + group.get("refresh_policy") != "retain" + or not group.get("unpaired_reason", "").strip() + ): + raise ValueError( + "Unpaired promotion changes must retain an explicit reason/policy." + ) + else: + _pr_number(group["upstream_pr"], upstream_repo) + _sha(group["upstream_base"]) + _sha(group["upstream_head"]) + if group["upstream_merge"] is not None: + _sha(group["upstream_merge"]) + if len(set(commits)) != len(commits) or record["promoted"] not in commits: + raise ValueError("Promotion metadata has duplicate or missing source commits.") + + +def _verify_remote_commit(gh: GitHub, plan: Promotion, repo: str, sha: str) -> str: + commit = gh.get(f"repos/{repo}/commits/{_sha(sha)}") + if len(commit["parents"]) != 1 or [item["filename"] for item in commit["files"]] != [_LOCK]: + raise ValueError("Promotion commit must change only the vendor lock and have one parent.") + parent = commit["parents"][0]["sha"] + comparison = gh.compare(gh.consumer_repo, parent, _main_sha(gh)) + if comparison["merge_base_commit"]["sha"] != parent: + raise ValueError( + "Promotion parent is not on the consumer base branch; unrelated commits may be present." + ) + before = _lock_document_at(gh, repo, parent) + after = _lock_document_at(gh, repo, sha) + _check_document_change(before, after, plan) + message = commit["commit"]["message"] + _adopt_record(plan, message) + if not re.search(r"^Signed-off-by: .+ <.+>$", message, re.MULTILINE): + raise ValueError("Promotion commit has no DCO sign-off.") + return message + + +def _check_document_change(before: dict, after: dict, plan: Promotion) -> None: + if before["vendors"][plan.previous.name] != plan.reviewed.to_mapping(): + raise ValueError("Promotion parent does not contain the reviewed temporary pin.") + expected = {**before, "vendors": {**before["vendors"], plan.previous.name: plan.promoted}} + if expected != after: + raise ValueError("Promotion commit is not the expected branch/URL-only lock change.") + + +@contextlib.contextmanager +def _source_repo(args: argparse.Namespace, plan: Promotion) -> Iterator[Path]: + with tempfile.TemporaryDirectory(prefix="vendor-promote-source-") as temporary: + repo = args.source_repo or Path(temporary) + if args.source_repo is None: + print(f"Fetching reviewed vendor source {plan.reviewed.commit}...", flush=True) + _git(repo, "init", "--quiet") + _git(repo, "fetch", "--quiet", "--no-tags", plan.reviewed.url, plan.reviewed.commit) + _git(repo, "cat-file", "-e", f"{plan.reviewed.commit}^{{commit}}") + _git(repo, "merge-base", "--is-ancestor", plan.previous.commit, plan.reviewed.commit) + yield repo + + +def _local_lock_document(worktree: Path, revision: str | None = None) -> dict: + content = ( + (worktree / _LOCK).read_text() + if revision is None + else _git(worktree, "show", f"{revision}:{_LOCK}") + ) + return yaml.load(content, Loader=vendor._UniqueKeyLoader) + + +def _pending_paths(worktree: Path) -> set[str]: + # A file restored in the working tree can still have an unrelated staged edit. + return set(_git(worktree, "diff", "--name-only", "HEAD").splitlines()) | set( + _git(worktree, "diff", "--cached", "--name-only", "HEAD").splitlines() + ) + + +def _check_lock_only(worktree: Path, before: dict, plan: Promotion) -> None: + _check_document_change(before, _local_lock_document(worktree), plan) + if _pending_paths(worktree) != {_LOCK}: + raise ValueError("Promotion must change only the vendor lock.") + if _git(worktree, "ls-files", "--others", "--exclude-standard"): + raise ValueError("Unexpected untracked files in the promotion worktree.") + + +def _prepare_commit(args: argparse.Namespace, plan: Promotion, source: Path) -> tuple[Path, str]: + print(f"Fetching consumer base {plan.main_sha}...", flush=True) + _git( + args.repo, + "fetch", + "--quiet", + "--no-tags", + f"https://github.com/{plan.consumer_repo}.git", + plan.main_sha, + ) + worktree = args.worktree or args.repo.parent / plan.branch.removeprefix("chore/") + worktree = worktree.resolve() + if worktree.exists(): + if _git(worktree, "branch", "--show-current") != plan.branch or _git( + worktree, "rev-parse", "--path-format=absolute", "--git-common-dir" + ) != _git(args.repo, "rev-parse", "--path-format=absolute", "--git-common-dir"): + raise ValueError( + f"Existing worktree is not this repository's promotion branch: {worktree}" + ) + head = _git(worktree, "rev-parse", "HEAD") + if _git(worktree, "merge-base", head, plan.main_sha) != head: + if _git(worktree, "status", "--porcelain"): + raise ValueError( + f"Completed promotion worktree contains unexpected edits: {worktree}" + ) + _adopt_record(plan, _git(worktree, "log", "-1", "--format=%B")) + parent = _git(worktree, "rev-parse", "HEAD^") + _git(worktree, "merge-base", "--is-ancestor", parent, plan.main_sha) + before = _local_lock_document(worktree, "HEAD^") + _check_committed_lock(worktree, before, plan) + lock = vendor._load_lock(worktree / _LOCK) + vendor._verify_source(lock, lock.vendors[plan.previous.name], source) + print(f"Reusing completed promotion commit {head} in {worktree}.", flush=True) + return worktree, head + print(f"Resuming incomplete promotion in {worktree}...", flush=True) + else: + print(f"Creating promotion worktree {worktree} (LFS downloads disabled)...", flush=True) + _git(args.repo, "worktree", "add", "-b", plan.branch, str(worktree), plan.main_sha) + before = _local_lock_document(worktree, "HEAD") + if before["vendors"][plan.previous.name] != plan.reviewed.to_mapping(): + raise ValueError("Worktree base no longer contains the reviewed temporary pin.") + # Recover only the two states this tool writes. Preserve arbitrary user or + # hook edits, including staged-only changes, instead of overwriting them. + for revision in (None, ""): + pending = _local_lock_document(worktree, revision) + if pending != before: + _check_document_change(before, pending, plan) + if _pending_paths(worktree) - {_LOCK}: + raise ValueError( + "Incomplete promotion has changes outside the vendor lock; inspect before retrying." + ) + if _git(worktree, "ls-files", "--others", "--exclude-standard"): + raise ValueError("Incomplete promotion has untracked files; inspect before retrying.") + lock = vendor._load_lock(worktree / _LOCK) + print("Verifying source materialization and the lock-only change...", flush=True) + vendor._verify_source(lock, lock.vendors[plan.previous.name], source) + result = vendor.main( + [ + "--lock", + str(worktree / _LOCK), + "pin", + plan.previous.name, + "--url", + plan.previous.url, + "--branch", + plan.previous.branch, + "--commit", + plan.reviewed.commit, + "--repo", + str(source), + ] + ) + if result: + raise ValueError("Vendor pin failed; worktree preserved for inspection.") + _check_lock_only(worktree, before, plan) + _git(worktree, "add", "--", _LOCK) + print("Creating signed-off promotion commit...", flush=True) + _git(worktree, "commit", "-s", "-F", "-", text=f"{plan.title}\n\n{_record_text(plan.record)}\n") + if _git(worktree, "status", "--porcelain"): + raise ValueError(f"Commit hooks left changes in {worktree}; inspect and retry.") + _check_committed_lock(worktree, before, plan) + return worktree, _git(worktree, "rev-parse", "HEAD") + + +def _check_committed_lock(worktree: Path, before: dict, plan: Promotion) -> None: + _check_document_change(before, _local_lock_document(worktree), plan) + if len(_git(worktree, "rev-list", "--parents", "-n", "1", "HEAD").split()) != 2: + raise ValueError("Promotion commit must have exactly one parent.") + if _git(worktree, "diff", "--name-only", "HEAD^", "HEAD") != _LOCK: + raise ValueError("Commit hooks added changes outside the vendor lock.") + message = _git(worktree, "log", "-1", "--format=%B") + if _record_from_message(message) != plan.record: + raise ValueError("Commit hooks changed the promotion provenance.") + if not re.search(r"^Signed-off-by: .+ <.+>$", message, re.MULTILINE): + raise ValueError("Promotion commit has no DCO sign-off.") + + +def _check_auto_merge(gh: GitHub) -> None: + settings = gh.get(f"repos/{gh.consumer_repo}") + if not settings.get("allow_auto_merge") or not settings.get("allow_squash_merge"): + raise ValueError("Repository must allow auto-merge and squash merging.") + rules = gh.api(f"repos/{gh.consumer_repo}/rules/branches/{gh.base_branch}") + if not isinstance(rules, list) or any(rule["type"] == "merge_queue" for rule in rules): + raise ValueError( + "Merge queues are unsupported: custom squash-message retention is required." + ) + + +def _disable_auto_merge(gh: GitHub, pr: dict) -> None: + gh.api( + "graphql", + method="POST", + payload={ + "query": ( + "mutation($id:ID!){disablePullRequestAutoMerge(" + "input:{pullRequestId:$id}){clientMutationId}}" + ), + "variables": {"id": pr["node_id"]}, + }, + ) + + +def _report_merged(pr: dict) -> None: + print( + f"Promotion merged: {pr['html_url']}\n" + f"Squash commit: {pr['merge_commit_sha']}\n" + "Final lock-only change, provenance, and DCO sign-off verified.", + flush=True, + ) + + +def _arm_auto_merge(gh: GitHub, plan: Promotion, pr: dict, message: str) -> None: + _check_auto_merge(gh) + _check_current(gh, plan) + number, head = pr["number"], pr["head"]["sha"] + state = gh.api( + "graphql", + method="POST", + payload={ + "query": "query($id:ID!){node(id:$id){... on PullRequest{headRefOid mergeStateStatus}}}", + "variables": {"id": pr["node_id"]}, + }, + ) + node = state["data"]["node"] + if node["headRefOid"] != head: + raise ValueError("Follow-up PR head changed; revalidate before enabling auto-merge.") + title, _, body = message.partition("\n") + saved = pr.get("auto_merge") + if saved: + if ( + saved.get("merge_method") == "squash" + and saved.get("commit_title") == title + and saved.get("commit_message", "").strip() == body.strip() + ): + print(f"Existing squash auto-merge message verified: {pr['html_url']}") + return + _disable_auto_merge(gh, pr) + raise ValueError("Existing auto-merge message differs; disabled. Inspect before retrying.") + # Query only necessary fields: gh pr merge's broader PR query can require + # read:org even when the token already has permission to merge this PR. + operation = ( + "mergePullRequest" if node["mergeStateStatus"] == "CLEAN" else "enablePullRequestAutoMerge" + ) + input_type = ( + "MergePullRequestInput" + if operation == "mergePullRequest" + else "EnablePullRequestAutoMergeInput" + ) + gh.api( + "graphql", + method="POST", + payload={ + "query": f"mutation($input:{input_type}!){{{operation}(input:$input){{clientMutationId}}}}", + "variables": { + "input": { + "pullRequestId": pr["node_id"], + "expectedHeadOid": head, + "mergeMethod": "SQUASH", + "commitHeadline": title, + "commitBody": body.strip(), + } + }, + }, + ) + updated = gh.get(f"repos/{gh.consumer_repo}/pulls/{number}") + if updated.get("merged"): + _verify_remote_commit(gh, plan, gh.consumer_repo, updated["merge_commit_sha"]) + _report_merged(updated) + else: + saved = updated.get("auto_merge") or {} + if ( + updated["head"]["sha"] != head + or saved.get("merge_method") != "squash" + or saved.get("commit_title") != title + or saved.get("commit_message", "").strip() != body.strip() + ): + _disable_auto_merge(gh, pr) + raise ValueError("Auto-merge message/head verification failed; auto-merge disabled.") + print(f"Squash auto-merge enabled and message verified: {updated['html_url']}") + + +def _pr_body(plan: Promotion, message: str) -> str: + unpaired = "" + if any(group["upstream_pr"] is None for group in plan.record["changes"]): + unpaired = ( + "No paired upstream PR exists for this source update. Its provenance " + "explicitly records it as unpaired, with `refresh_policy: retain`. A future " + "refresh must carry it forward until an upstream pairing or equivalence " + "is explicitly established.\n\n" + ) + return ( + "@coderabbitai summary\n\n## Description\n\n" + f"Promote the exact source revision reviewed in {plan.record['consumer_pr']}.\n" + "Only the selected vendor lock URL/branch changes. The source SHA, selected vendor files, " + "compatibility patch, and digests are unchanged.\n\n" + f"{unpaired}" + "## Test Coverage\n\nSource materialization and lock-only diff verified by the promotion tool. " + "No new kernel behavior. The tool posts " + f"`{_CI_SKIP_COMMAND}` to request the bot's no-code-change CI path. " + "Automatic GitHub checks and required reviews still apply.\n\n" + "## PR Checklist\n\n" + "- [x] No runtime/API/dependency changes.\n" + "- [x] Immutable source pin and vendor content preserved.\n" + "- [x] Please check this after reviewing the above items as appropriate for this PR.\n\n" + "## Required squash commit message\n\n" + "Preserve the following message, including the provenance block and DCO sign-off.\n\n" + f"```text\n{message.strip()}\n```\n" + ) + + +def _ensure_ci_skip(gh: GitHub, plan: Promotion, pr: dict) -> dict: + """Request the no-code-change bot path once across sequential retries.""" + current = gh.get(f"repos/{gh.consumer_repo}/pulls/{pr['number']}") + if current.get("merged"): + _verify_remote_commit(gh, plan, gh.consumer_repo, current["merge_commit_sha"]) + _report_merged(current) + return current + if current["state"] != "open" or current["head"]["sha"] != pr["head"]["sha"]: + raise ValueError("Follow-up PR closed or changed before the CI request; inspect it.") + _check_current(gh, plan) + _verify_remote_commit(gh, plan, gh.consumer_repo, current["head"]["sha"]) + actor = gh.get("user")["login"] + endpoint = f"repos/{gh.consumer_repo}/issues/{pr['number']}/comments" + page = 1 + while True: + comments = gh.api(f"{endpoint}?per_page=100&page={page}") + if not isinstance(comments, list): + raise ValueError("Unexpected issue-comment response; refusing a duplicate CI request.") + if any( + comment.get("body", "").strip() == _CI_SKIP_COMMAND + and comment.get("user", {}).get("login") == actor + for comment in comments + ): + print(f"Existing CI skip request found: {current['html_url']}", flush=True) + return current + if len(comments) < 100: + break + page += 1 + comment = gh.api(endpoint, method="POST", payload={"body": _CI_SKIP_COMMAND}) + print(f"Posted CI skip request: {comment['html_url']}", flush=True) + return current + + +def _publish(args: argparse.Namespace, gh: GitHub, plan: Promotion) -> dict: + if plan.completed_pr is not None: + _report_merged(plan.completed_pr) + return plan.completed_pr + existing = _find_followup(gh, plan) + remote = gh.optional(f"repos/{plan.fork}/git/ref/heads/{plan.branch}") + if existing: + existing = gh.get(f"repos/{gh.consumer_repo}/pulls/{existing['number']}") + sha = existing["merge_commit_sha"] if existing.get("merged") else existing["head"]["sha"] + message = _verify_remote_commit(gh, plan, gh.consumer_repo, sha) + if existing.get("merged"): + _report_merged(existing) + return existing + if remote is None or remote["object"]["sha"] != sha: + raise ValueError("Follow-up branch is missing or changed; inspect before resuming.") + elif remote: + sha = remote["object"]["sha"] + message = _verify_remote_commit(gh, plan, plan.fork, sha) + else: + message = "" + if args.auto_merge: + _check_auto_merge(gh) + _check_current(gh, plan) + with _source_repo(args, plan) as source: + if remote is None: + worktree, sha = _prepare_commit(args, plan, source) + message = _git(worktree, "log", "-1", "--format=%B") + _check_current(gh, plan) + # No force push: Git also enforces the fast-forward constraint remotely. + print(f"Ensuring canonical branch points to {plan.reviewed.commit}...", flush=True) + _git( + source, + "push", + plan.previous.url, + f"{plan.reviewed.commit}:refs/heads/{plan.previous.branch}", + ) + print(f"Canonical branch now contains reviewed revision {plan.reviewed.commit}.") + if remote is None: + print(f"Publishing follow-up branch {plan.fork}:{plan.branch}...", flush=True) + _git( + worktree, + "push", + f"https://github.com/{plan.fork}.git", + f"{sha}:refs/heads/{plan.branch}", + ) + _check_current(gh, plan) + if existing is None: + print("Creating the lock-only follow-up PR...", flush=True) + existing = gh.api( + f"repos/{gh.consumer_repo}/pulls", + method="POST", + payload={ + "title": plan.title, + "body": _pr_body(plan, message), + "base": plan.base_branch, + "head": f"{plan.fork.split('/')[0]}:{plan.branch}", + "head_repo": plan.fork.split("/")[1], + }, + ) + print(f"Lock-only promotion PR: {existing['html_url']}") + existing = _ensure_ci_skip(gh, plan, existing) + if existing.get("merged"): + return existing + if args.auto_merge: + _arm_auto_merge(gh, plan, existing, message) + if args.wait: + deadline = time.monotonic() + args.timeout + while True: + current = gh.get(f"repos/{gh.consumer_repo}/pulls/{existing['number']}") + if current.get("merged"): + _verify_remote_commit(gh, plan, gh.consumer_repo, current["merge_commit_sha"]) + _report_merged(current) + break + if current["state"] == "closed" or current["head"]["sha"] != existing["head"]["sha"]: + if args.auto_merge and current.get("auto_merge"): + _disable_auto_merge(gh, current) + raise ValueError( + "Follow-up PR closed or changed while waiting; inspect before proceeding." + ) + if time.monotonic() >= deadline: + raise TimeoutError( + "Promotion PR is still pending; rerun promote to verify after merge." + ) + time.sleep(min(15, max(0, deadline - time.monotonic()))) + return existing + + +def main(argv: list[str] | None = None) -> int: + """Run the maintainer CLI; dry-run unless publication is explicitly requested.""" + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + promote = commands.add_parser("promote", help="Promote a merged consumer source update.") + promote.add_argument("--source-pr", "--trtllm-pr", dest="source_pr", required=True) + promote.add_argument("--vendor", required=True, help="Vendor key in the lock file.") + promote.add_argument("--consumer-repo", default="NVIDIA/TensorRT-LLM") + promote.add_argument("--base-branch", default="main") + promote.add_argument("--upstream-repo", required=True, help="Upstream OWNER/REPO.") + promote.add_argument("--upstream-branch", default="main") + promote.add_argument("--legacy-prims-ts", action="store_true", help=argparse.SUPPRESS) + promote.add_argument( + "--canonical-repo", + required=True, + help="Explicit canonical source fork OWNER/REPO (not the developer's temporary fork).", + ) + promote.add_argument( + "--canonical-branch", + required=True, + help="Expected canonical branch; specify a dated branch when applicable.", + ) + provenance = promote.add_mutually_exclusive_group(required=True) + provenance.add_argument("--upstream-pr", action="append", default=[]) + provenance.add_argument( + "--from-pr-description", + action="store_true", + help="Read contributor metadata and automatically match upstream commits.", + ) + provenance.add_argument( + "--unpaired-reason", + help="Explicit exception for changes with no upstream PR; retain them during refresh.", + ) + promote.add_argument("--map-upstream", action="append", default=[], metavar="COMMIT=PR") + promote.add_argument( + "--auto-match", + action="store_true", + help="Match --upstream-pr commits automatically instead of prompting.", + ) + promote.add_argument("--repo", type=Path, default=Path(__file__).resolve().parents[2]) + promote.add_argument( + "--fork", help="Consumer publishing fork OWNER/REPO; defaults to remote 'fork'." + ) + promote.add_argument( + "--source-repo", + "--flashinfer-repo", + dest="source_repo", + type=Path, + help="Local source repository containing both immutable SHAs.", + ) + promote.add_argument("--worktree", type=Path, help="New/preserved follow-up worktree path.") + promote.add_argument( + "--publish", + action="store_true", + help="Publish the lock-only PR and post the no-code-change /bot skip request.", + ) + promote.add_argument("--auto-merge", action="store_true") + promote.add_argument( + "--wait", action="store_true", help="Wait for merge and verify final commit metadata." + ) + promote.add_argument( + "--timeout", type=int, default=3600, help="Maximum --wait duration in seconds." + ) + args = parser.parse_args(argv) + if (args.auto_merge or args.wait) and not args.publish: + parser.error("--auto-merge and --wait require --publish") + if args.timeout <= 0: + parser.error("--timeout must be positive") + args.repo = args.repo.resolve() + try: + if not vendor._NAME_PATTERN.fullmatch(args.vendor): + raise ValueError("Invalid vendor key.") + _repo_name(f"https://github.com/{args.upstream_repo}") + gh = GitHub( + args.consumer_repo, + args.vendor, + args.upstream_repo, + args.base_branch, + args.upstream_branch, + ) + resolutions = None + if args.from_pr_description: + if __package__: + from . import metadata + else: + import metadata + number = _pr_number(args.source_pr, gh.consumer_repo) + pr = gh.get(f"repos/{gh.consumer_repo}/pulls/{number}") + entry = metadata.parse(pr.get("body") or "", gh.vendor_name, gh.upstream_repo) + args.upstream_pr = [str(number) for number in entry["upstream_prs"]] + args.unpaired_reason = entry["unpaired_reason"] + resolutions = entry["resolutions"] + if args.map_upstream and (args.auto_match or args.from_pr_description): + raise ValueError("Explicit --map-upstream cannot accompany automatic matching.") + plan = _make_plan( + args, + gh, + auto_match=args.auto_match or args.from_pr_description, + resolutions=resolutions, + ) + mode = ( + "ALREADY COMPLETED" if plan.completed_pr else "PUBLISH" if args.publish else "DRY RUN" + ) + print(f"{mode}: {plan.canonical_repo}:{plan.previous.branch}", flush=True) + print(f"Source: {plan.previous.commit} -> {plan.reviewed.commit}") + print(f"Follow-up: {plan.fork}:{plan.branch}") + print(_record_text(plan.record)) + if args.publish: + _publish(args, gh, plan) + elif plan.completed_pr is not None: + _report_merged(plan.completed_pr) + else: + print(f"On publication: post {_CI_SKIP_COMMAND}") + print(f"Squash auto-merge: {'enabled' if args.auto_merge else 'off'}") + print("No remote writes or consumer worktree changes. Use --publish to proceed.") + return 0 + except ( + RuntimeError, + ValueError, + OSError, + KeyError, + subprocess.TimeoutExpired, + vendor.VendorError, + ) as error: + print(f"Promotion stopped: {error}", file=sys.stderr) + print( + "Any completed pushes/PRs are preserved. Inspect the error before retrying.", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/vendor_sources.py b/scripts/vendor_sources.py index 392d6b8565ac..8e3f9350ed49 100644 --- a/scripts/vendor_sources.py +++ b/scripts/vendor_sources.py @@ -1,1644 +1,12 @@ #!/usr/bin/env python3 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Manage source trees vendored from immutable Git commits.""" - -from __future__ import annotations - -import argparse -import contextlib -import dataclasses -import fnmatch -import hashlib -import os -import re -import shutil -import stat -import subprocess -import sys -import tempfile -import urllib.parse -from collections.abc import Iterator, Mapping, Sequence -from pathlib import Path, PurePosixPath - -import yaml - -_REPO_ROOT = Path(__file__).resolve().parent.parent -_DEFAULT_LOCK = _REPO_ROOT / "3rdparty" / "vendor_sources.lock.yaml" -_SCHEMA_VERSION = 1 -_COMMIT_PATTERN = re.compile(r"[0-9a-f]{40}\Z") -_NAME_PATTERN = re.compile(r"[a-z0-9][a-z0-9._-]*\Z") -_DIGEST_PREFIX = "sha256-tree-v1:" -_PATCH_DIGEST_PREFIX = "sha256:" -_GIT_TIMEOUT_SECONDS = 60 -_GIT_LOCAL_ENVIRONMENT_VARIABLES = { - "GIT_ALTERNATE_OBJECT_DIRECTORIES", - "GIT_ATTR_GLOBAL", - "GIT_ATTR_SOURCE", - "GIT_ATTR_SYSTEM", - "GIT_CEILING_DIRECTORIES", - "GIT_COMMON_DIR", - "GIT_CONFIG", - "GIT_CONFIG_COUNT", - "GIT_CONFIG_PARAMETERS", - "GIT_DIR", - "GIT_GRAFT_FILE", - "GIT_IMPLICIT_WORK_TREE", - "GIT_INDEX_FILE", - "GIT_NO_REPLACE_OBJECTS", - "GIT_OBJECT_DIRECTORY", - "GIT_PREFIX", - "GIT_REPLACE_REF_BASE", - "GIT_SHALLOW_FILE", - "GIT_WORK_TREE", -} -_VENDOR_FIELDS = { - "url", - "branch", - "tag", - "commit", - "source", - "destination", - "include", - "patch", - "patch_digest", - "digest", -} - - -class VendorError(RuntimeError): - """A user-actionable vendoring failure.""" - - -class UpstreamUnavailable(VendorError): - """The recorded upstream could not be accessed.""" - - -class _UniqueKeyLoader(yaml.SafeLoader): - """Safe YAML loader that rejects duplicate mapping keys.""" - - -def _construct_unique_mapping( - loader: _UniqueKeyLoader, - node: yaml.MappingNode, - deep: bool = False, -) -> dict[object, object]: - if any( - isinstance(key_node, yaml.ScalarNode) and key_node.value == "<<" - for key_node, _ in node.value - ): - raise yaml.constructor.ConstructorError( - "while constructing a mapping", - node.start_mark, - "YAML merge keys are not allowed in the vendor lock", - node.start_mark, - ) - loader.flatten_mapping(node) - result: dict[object, object] = {} - for key_node, value_node in node.value: - key = loader.construct_object(key_node, deep=deep) - try: - duplicate = key in result - except TypeError as error: - raise yaml.constructor.ConstructorError( - "while constructing a mapping", - node.start_mark, - "found an unhashable mapping key", - key_node.start_mark, - ) from error - if duplicate: - raise yaml.constructor.ConstructorError( - "while constructing a mapping", - node.start_mark, - f"found duplicate key {key!r}", - key_node.start_mark, - ) - result[key] = loader.construct_object(value_node, deep=deep) - return result - - -_UniqueKeyLoader.add_constructor( - yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, - _construct_unique_mapping, -) - - -@dataclasses.dataclass(frozen=True) -class FileEntry: - """One regular file in a normalized selected tree.""" - - data: bytes - executable: bool = False - - -Tree = dict[str, FileEntry] - - -@dataclasses.dataclass -class Vendor: - """Validated lock entry for one managed source tree.""" - - name: str - url: str - commit: str - source: str - destination: str - include: tuple[str, ...] - digest: str - branch: str | None = None - tag: str | None = None - patch: str | None = None - patch_digest: str | None = None - - def to_mapping(self) -> dict[str, object]: - """Serialize this entry in the canonical field order.""" - result: dict[str, object] = {"url": self.url} - if self.branch is not None: - result["branch"] = self.branch - if self.tag is not None: - result["tag"] = self.tag - result.update( - { - "commit": self.commit, - "source": self.source, - "destination": self.destination, - "include": list(self.include), - } - ) - if self.patch is not None: - result["patch"] = self.patch - result["patch_digest"] = self.patch_digest - result["digest"] = self.digest - return result - - -@dataclasses.dataclass -class LockFile: - """Validated collection of vendor entries and its consumer root.""" - - path: Path - root: Path - vendors: dict[str, Vendor] - - -def _require_mapping(value: object, description: str) -> Mapping[object, object]: - if not isinstance(value, Mapping): - raise VendorError(f"{description} must be a mapping.") - return value - - -def _validate_mapping_keys( - mapping: Mapping[object, object], allowed: set[str], description: str -) -> set[str]: - keys: set[str] = set() - for key in mapping: - if not isinstance(key, str): - raise VendorError(f"{description} keys must be strings; got {key!r}.") - keys.add(key) - unknown = keys - allowed - if unknown: - raise VendorError(f"{description} has unexpected fields: {sorted(unknown)}.") - return keys - - -def _require_string(value: object, description: str) -> str: - if not isinstance(value, str) or not value.strip(): - raise VendorError(f"{description} must be a non-empty string.") - return value - - -def _validate_relative_path(value: object, description: str) -> str: - path_text = _require_string(value, description) - if "\\" in path_text: - raise VendorError(f"{description} must use POSIX separators: {path_text!r}.") - path = PurePosixPath(path_text) - if ( - path.is_absolute() - or path_text == "." - or path_text.startswith(":") - or any(part in ("", ".", "..", ".git") for part in path.parts) - ): - raise VendorError(f"{description} must be a safe relative path: {path_text!r}.") - return path.as_posix() - - -def _validate_url(value: object, vendor_name: str) -> str: - url = _require_string(value, f"Vendor {vendor_name!r} url") - parsed = urllib.parse.urlsplit(url) - if parsed.password is not None or ( - parsed.scheme in ("http", "https") and parsed.username is not None - ): - raise VendorError( - f"Vendor {vendor_name!r} URL must not contain credentials; use Git credential helpers." - ) - return url - - -def _validate_include(value: object, vendor_name: str) -> tuple[str, ...]: - if not isinstance(value, list) or not value: - raise VendorError(f"Vendor {vendor_name!r} include must be a non-empty list.") - patterns: list[str] = [] - for index, item in enumerate(value): - pattern = _require_string(item, f"Vendor {vendor_name!r} include[{index}]") - if "\\" in pattern or pattern.startswith("/"): - raise VendorError(f"Vendor {vendor_name!r} has an unsafe include pattern: {pattern!r}.") - if any(part == ".." for part in PurePosixPath(pattern).parts): - raise VendorError(f"Vendor {vendor_name!r} has an unsafe include pattern: {pattern!r}.") - patterns.append(pattern) - if len(patterns) != len(set(patterns)): - raise VendorError(f"Vendor {vendor_name!r} include patterns must be unique.") - return tuple(patterns) - - -def _validate_digest(value: object, description: str) -> str: - digest = _require_string(value, description) - suffix = digest.removeprefix(_DIGEST_PREFIX) - if not digest.startswith(_DIGEST_PREFIX) or not re.fullmatch(r"[0-9a-f]{64}", suffix): - raise VendorError(f"{description} must use {_DIGEST_PREFIX}<64 lowercase hex digits>.") - return digest - - -def _validate_patch_digest(value: object, description: str) -> str: - digest = _require_string(value, description) - suffix = digest.removeprefix(_PATCH_DIGEST_PREFIX) - if not digest.startswith(_PATCH_DIGEST_PREFIX) or not re.fullmatch(r"[0-9a-f]{64}", suffix): - raise VendorError( - f"{description} must use {_PATCH_DIGEST_PREFIX}<64 lowercase hex digits>." - ) - return digest - - -def _patch_digest(content: bytes) -> str: - return f"{_PATCH_DIGEST_PREFIX}{hashlib.sha256(content).hexdigest()}" - - -def _validate_vendor(name: object, value: object) -> Vendor: - vendor_name = _require_string(name, "Vendor name") - if _NAME_PATTERN.fullmatch(vendor_name) is None: - raise VendorError( - f"Vendor name {vendor_name!r} must contain only lowercase letters, digits, '.', '_', or '-'." - ) - mapping = _require_mapping(value, f"Vendor {vendor_name!r}") - fields = _validate_mapping_keys(mapping, _VENDOR_FIELDS, f"Vendor {vendor_name!r}") - required = {"url", "commit", "source", "destination", "include", "digest"} - missing = required - fields - if missing: - raise VendorError(f"Vendor {vendor_name!r} is missing fields: {sorted(missing)}.") - branch = mapping.get("branch") - tag = mapping.get("tag") - if branch is not None and tag is not None: - raise VendorError(f"Vendor {vendor_name!r} cannot record both branch and tag.") - branch_text = ( - None if branch is None else _require_string(branch, f"Vendor {vendor_name!r} branch") - ) - tag_text = None if tag is None else _require_string(tag, f"Vendor {vendor_name!r} tag") - for kind, ref in (("branch", branch_text), ("tag", tag_text)): - if ref is not None: - invalid_ref = ( - ref.startswith(("refs/", ".", "/")) - or ref.endswith((".", "/", ".lock")) - or ".." in ref - or "@{" in ref - or "//" in ref - or any(char.isspace() or char in "~^:?*[\\" for char in ref) - ) - if invalid_ref: - raise VendorError(f"Vendor {vendor_name!r} {kind} must be a short Git name.") - commit = _require_string(mapping["commit"], f"Vendor {vendor_name!r} commit") - if _COMMIT_PATTERN.fullmatch(commit) is None: - raise VendorError(f"Vendor {vendor_name!r} commit must be a full lowercase 40-hex SHA.") - patch_value = mapping.get("patch") - patch = ( - None - if patch_value is None - else _validate_relative_path(patch_value, f"Vendor {vendor_name!r} patch") - ) - expected_patch = f"3rdparty/vendor_patches/{vendor_name}.patch" - if patch is not None and patch != expected_patch: - raise VendorError(f"Vendor {vendor_name!r} patch must be {expected_patch!r}.") - patch_digest_value = mapping.get("patch_digest") - if (patch is None) != (patch_digest_value is None): - raise VendorError( - f"Vendor {vendor_name!r} patch and patch_digest must either both be present or both be absent." - ) - patch_digest = ( - None - if patch_digest_value is None - else _validate_patch_digest(patch_digest_value, f"Vendor {vendor_name!r} patch_digest") - ) - include = _validate_include(mapping["include"], vendor_name) - return Vendor( - name=vendor_name, - url=_validate_url(mapping["url"], vendor_name), - branch=branch_text, - tag=tag_text, - commit=commit, - source=_validate_relative_path(mapping["source"], f"Vendor {vendor_name!r} source"), - destination=_validate_relative_path( - mapping["destination"], f"Vendor {vendor_name!r} destination" - ), - include=include, - patch=patch, - patch_digest=patch_digest, - digest=_validate_digest(mapping["digest"], f"Vendor {vendor_name!r} digest"), - ) - - -def _consumer_root(lock_path: Path) -> Path: - if lock_path.parent.name == "3rdparty": - return lock_path.parent.parent.resolve() - return lock_path.parent.resolve() - - -def _validate_destination_layout(lock: LockFile) -> None: - destinations: list[tuple[str, PurePosixPath]] = [] - try: - lock_relative = PurePosixPath(lock.path.relative_to(lock.root).as_posix()) - except ValueError as error: - raise VendorError(f"Vendor lock must be inside the consumer root: {lock.path}.") from error - patch_directory = PurePosixPath("3rdparty/vendor_patches") - for name, vendor in lock.vendors.items(): - path = PurePosixPath(vendor.destination) - for other_name, other_path in destinations: - if ( - path == other_path - or path.is_relative_to(other_path) - or other_path.is_relative_to(path) - ): - raise VendorError( - f"Vendor destinations overlap: {name!r} ({path}) and {other_name!r} ({other_path})." - ) - destinations.append((name, path)) - if ( - path == patch_directory - or path.is_relative_to(patch_directory) - or patch_directory.is_relative_to(path) - ): - raise VendorError( - f"Vendor {name!r} destination overlaps vendoring control files: {path}." - ) - if lock_relative == path or lock_relative.is_relative_to(path): - raise VendorError(f"Vendor {name!r} destination contains the vendor lock: {path}.") - if vendor.patch is not None: - patch_path = PurePosixPath(vendor.patch) - if ( - patch_path == path - or patch_path.is_relative_to(path) - or path.is_relative_to(patch_path) - ): - raise VendorError(f"Vendor {name!r} destination overlaps its patch path: {path}.") - - -def _load_lock( - path: Path, - *, - allow_missing: bool = False, - allow_patch_recovery: bool = False, -) -> LockFile: - resolved = path.resolve() - if not resolved.exists(): - if allow_missing: - return LockFile(path=resolved, root=_consumer_root(resolved), vendors={}) - raise VendorError(f"Vendor lock does not exist: {resolved}.") - try: - content = resolved.read_text(encoding="utf-8") - if any( - isinstance(token, (yaml.tokens.AliasToken, yaml.tokens.AnchorToken)) - for token in yaml.scan(content) - ): - raise VendorError("Vendor lock YAML aliases and anchors are not allowed.") - loaded = yaml.load(content, Loader=_UniqueKeyLoader) - except VendorError: - raise - except (OSError, UnicodeError, yaml.YAMLError) as error: - raise VendorError(f"Failed to parse vendor lock {resolved}: {error}") from error - mapping = _require_mapping(loaded, "Vendor lock") - _validate_mapping_keys(mapping, {"schema_version", "vendors"}, "Vendor lock") - if mapping.get("schema_version") != _SCHEMA_VERSION: - raise VendorError(f"Vendor lock schema_version must be {_SCHEMA_VERSION}.") - raw_vendors = _require_mapping(mapping.get("vendors"), "Vendor lock vendors") - vendors = { - _require_string(name, "Vendor name"): _validate_vendor(name, value) - for name, value in raw_vendors.items() - } - lock = LockFile(path=resolved, root=_consumer_root(resolved), vendors=vendors) - _validate_destination_layout(lock) - for vendor in vendors.values(): - _checked_root_path(lock.root, vendor.destination, f"Vendor {vendor.name!r} destination") - if vendor.patch is not None: - patch_path = _checked_root_path( - lock.root, vendor.patch, f"Vendor {vendor.name!r} patch" - ) - if patch_path.is_symlink() or (patch_path.exists() and not patch_path.is_file()): - raise VendorError( - f"Vendor {vendor.name!r} patch does not name a regular file: {vendor.patch}." - ) - if not patch_path.exists() and allow_patch_recovery: - continue - if not patch_path.is_file(): - raise VendorError( - f"Vendor {vendor.name!r} patch does not name a regular file: {vendor.patch}." - ) - actual_patch_digest = _patch_digest(patch_path.read_bytes()) - if actual_patch_digest != vendor.patch_digest and not allow_patch_recovery: - raise VendorError( - f"Vendor {vendor.name!r} patch digest mismatch: expected " - f"{vendor.patch_digest}, got {actual_patch_digest}." - ) - return lock - - -def _save_lock(lock: LockFile) -> None: - payload = { - "schema_version": _SCHEMA_VERSION, - "vendors": {name: vendor.to_mapping() for name, vendor in sorted(lock.vendors.items())}, - } - content = yaml.safe_dump( - payload, - sort_keys=False, - default_flow_style=False, - allow_unicode=True, - ).encode("utf-8") - _atomic_write(lock.path, content) - - -def _save_lock_checked(lock: LockFile, description: str) -> None: - try: - _save_lock(lock) - except (OSError, yaml.YAMLError) as error: - raise VendorError(f"Failed to save {description}: {error}") from error - - -def _checked_root_path(root: Path, relative_path: str, description: str) -> Path: - target = root.joinpath(*PurePosixPath(relative_path).parts) - try: - resolved = target.resolve(strict=False) - except OSError as error: - raise VendorError(f"Could not resolve {description}: {error}") from error - if not resolved.is_relative_to(root.resolve()): - raise VendorError(f"{description} escapes the consumer root: {relative_path!r}.") - current = root.resolve() - for part in PurePosixPath(relative_path).parts: - current = current / part - if current.is_symlink(): - raise VendorError(f"{description} traverses a symlink: {relative_path!r}.") - if not current.exists(): - break - return target - - -def _atomic_write(path: Path, content: bytes) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - file_descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) - temporary = Path(temporary_name) - try: - with os.fdopen(file_descriptor, "wb") as output: - output.write(content) - output.flush() - os.fsync(output.fileno()) - temporary.chmod(0o644) - os.replace(temporary, path) - finally: - temporary.unlink(missing_ok=True) - - -def _sync_directory(path: Path) -> None: - directory_descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) - try: - os.fsync(directory_descriptor) - finally: - os.close(directory_descriptor) - - -def _matches(path: str, patterns: Sequence[str]) -> bool: - """Match a full relative POSIX path, with ** spanning zero or more components.""" - path_parts = PurePosixPath(path).parts - for pattern in patterns: - # Each entry records whether that path prefix matches the pattern so far. - matched = [True] + [False] * len(path_parts) - for part in PurePosixPath(pattern).parts: - if part == "**": - for index in range(1, len(matched)): - matched[index] = matched[index] or matched[index - 1] - else: - matched = [False] + [ - matched[index] and fnmatch.fnmatchcase(path_part, part) - for index, path_part in enumerate(path_parts) - ] - if matched[-1]: - return True - return False - - -def _validate_tree_relative_path(path: str) -> str: - pure_path = PurePosixPath(path) - if ( - pure_path.is_absolute() - or any(part in ("", ".", "..", ".git") for part in pure_path.parts) - or pure_path.as_posix() != path - ): - raise VendorError(f"Managed tree contains an unsafe path: {path!r}.") - return path - - -def _tree_digest(tree: Mapping[str, FileEntry]) -> str: - digest = hashlib.sha256() - digest.update(b"sha256-tree-v1\0") - for path, entry in sorted(tree.items()): - encoded_path = path.encode("utf-8") - digest.update(len(encoded_path).to_bytes(8, "big")) - digest.update(encoded_path) - digest.update(b"x" if entry.executable else b"-") - digest.update(len(entry.data).to_bytes(8, "big")) - digest.update(entry.data) - return f"{_DIGEST_PREFIX}{digest.hexdigest()}" - - -def _read_directory_tree(directory: Path, patterns: Sequence[str]) -> Tree: - if not directory.exists(): - return {} - if directory.is_symlink() or not directory.is_dir(): - raise VendorError(f"Managed destination must be a regular directory: {directory}.") - tree: Tree = {} - for path in sorted(directory.rglob("*")): - relative = _validate_tree_relative_path(path.relative_to(directory).as_posix()) - if path.is_symlink(): - raise VendorError(f"Managed trees cannot contain symlinks: {relative}.") - if path.is_dir(): - continue - if not path.is_file(): - raise VendorError(f"Managed trees cannot contain special files: {relative}.") - if not _matches(relative, patterns): - continue - mode = path.stat().st_mode - tree[relative] = FileEntry(path.read_bytes(), bool(mode & stat.S_IXUSR)) - return tree - - -def _read_all_directory_files(directory: Path, patterns: Sequence[str]) -> Tree: - tree = _read_directory_tree(directory, patterns) - for path in sorted(directory.rglob("*")): - if path.is_dir(): - continue - relative = path.relative_to(directory).as_posix() - if not _matches(relative, patterns): - raise VendorError(f"Patch produced a file outside the vendor include set: {relative}.") - return tree - - -def _write_tree(directory: Path, tree: Mapping[str, FileEntry]) -> None: - directory.mkdir(parents=True, exist_ok=True) - for relative, entry in tree.items(): - _validate_tree_relative_path(relative) - target = directory.joinpath(*PurePosixPath(relative).parts) - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(entry.data) - target.chmod(0o755 if entry.executable else 0o644) - - -def _remove_path(path: Path) -> None: - if path.is_symlink() or path.is_file(): - path.unlink() - elif path.is_dir(): - shutil.rmtree(path) - - -@contextlib.contextmanager -def _replace_selected_destination_transaction( - lock: LockFile, vendor: Vendor, tree: Mapping[str, FileEntry] -) -> Iterator[None]: - destination = _checked_root_path( - lock.root, vendor.destination, f"Vendor {vendor.name!r} destination" - ) - backup = destination.parent / f".{destination.name}.vendor-backup" - staging: Path | None = None - destination_existed = False - replacement_installed = False - preserve_staging = False - try: - destination.parent.mkdir(parents=True, exist_ok=True) - if destination.exists(): - _read_directory_tree(destination, vendor.include) - if backup.exists() or backup.is_symlink(): - raise VendorError(f"Stale vendor backup blocks synchronization: {backup}.") - staging = Path( - tempfile.mkdtemp(prefix=f".{destination.name}.vendor-", dir=destination.parent) - ) - destination_existed = destination.exists() - if destination_existed: - shutil.copytree(destination, staging, dirs_exist_ok=True) - for path in sorted(staging.rglob("*"), reverse=True): - if path.is_dir(): - with contextlib.suppress(OSError): - path.rmdir() - continue - relative = path.relative_to(staging).as_posix() - if _matches(relative, vendor.include): - path.unlink() - _write_tree(staging, tree) - if destination_existed: - os.replace(destination, backup) - try: - os.replace(staging, destination) - except OSError: - if backup.exists() and not destination.exists(): - os.replace(backup, destination) - raise - replacement_installed = True - try: - yield - except BaseException as triggering_error: - try: - if replacement_installed and (destination.exists() or destination.is_symlink()): - os.replace(destination, staging) - if destination_existed and (backup.exists() or backup.is_symlink()): - os.replace(backup, destination) - except OSError as rollback_error: - preserve_staging = True - raise VendorError( - f"Failed to roll back destination for vendor {vendor.name!r} after " - f"{triggering_error}; recovery data remains at {backup} and {staging}: " - f"{rollback_error}" - ) from rollback_error - raise - else: - if backup.exists() or backup.is_symlink(): - _remove_path(backup) - except OSError as error: - raise VendorError( - f"Failed to replace destination for vendor {vendor.name!r}: {error}" - ) from error - finally: - if ( - staging is not None - and not preserve_staging - and (staging.exists() or staging.is_symlink()) - ): - try: - _remove_path(staging) - except OSError as error: - raise VendorError( - f"Failed to clean staging directory for vendor {vendor.name!r}: " - f"{staging}: {error}" - ) from error - - -def _replace_selected_destination( - lock: LockFile, vendor: Vendor, tree: Mapping[str, FileEntry] -) -> None: - with _replace_selected_destination_transaction(lock, vendor, tree): - pass - - -@contextlib.contextmanager -def _atomic_file_replacement(path: Path, content: bytes) -> Iterator[None]: - try: - if path.is_symlink() or (path.exists() and not path.is_file()): - raise VendorError(f"Generated patch path must be a regular file: {path}.") - existed = path.exists() - old_content = path.read_bytes() if existed else None - old_mode = stat.S_IMODE(path.stat().st_mode) if existed else None - _atomic_write(path, content) - except VendorError: - raise - except OSError as error: - raise VendorError(f"Failed to write generated patch {path}: {error}") from error - try: - yield - except BaseException: - try: - if old_content is None: - path.unlink(missing_ok=True) - else: - _atomic_write(path, old_content) - assert old_mode is not None - path.chmod(old_mode) - except OSError as error: - raise VendorError(f"Failed to roll back generated patch {path}: {error}") from error - raise - - -def _git_environment(*, isolated: bool, repository_ceiling: Path | None = None) -> dict[str, str]: - environment = os.environ.copy() - for variable in _GIT_LOCAL_ENVIRONMENT_VARIABLES: - environment.pop(variable, None) - environment.update( - { - "GIT_TERMINAL_PROMPT": "0", - "GIT_ATTR_NOSYSTEM": "1", - "GIT_LITERAL_PATHSPECS": "1", - "GIT_NO_REPLACE_OBJECTS": "1", - "GIT_HTTP_LOW_SPEED_LIMIT": "1", - "GIT_HTTP_LOW_SPEED_TIME": "30", - "GIT_AUTHOR_NAME": "TensorRT-LLM Vendor Tool", - "GIT_AUTHOR_EMAIL": "vendor-tool@nvidia.com", - "GIT_COMMITTER_NAME": "TensorRT-LLM Vendor Tool", - "GIT_COMMITTER_EMAIL": "vendor-tool@nvidia.com", - } - ) - if isolated: - environment["GIT_CONFIG_NOSYSTEM"] = "1" - environment["GIT_CONFIG_GLOBAL"] = os.devnull - if repository_ceiling is not None: - environment["GIT_CEILING_DIRECTORIES"] = str(repository_ceiling.resolve()) - environment.setdefault("GIT_SSH_COMMAND", "ssh -o BatchMode=yes -o ConnectTimeout=20") - return environment - - -def _run_git( - arguments: Sequence[str | Path], - *, - cwd: Path, - capture_bytes: bool = False, - isolated: bool = False, - input_bytes: bytes | None = None, - repository_ceiling: Path | None = None, -) -> bytes | str: - if input_bytes is not None and not capture_bytes: - raise ValueError("Git commands with byte input must capture byte output.") - command = ["git", *[str(argument) for argument in arguments]] - try: - result = subprocess.run( - command, - cwd=cwd, - env=_git_environment( - isolated=isolated, - repository_ceiling=repository_ceiling, - ), - check=True, - capture_output=True, - input=input_bytes, - text=not capture_bytes, - timeout=_GIT_TIMEOUT_SECONDS, - ) - except FileNotFoundError as error: - raise VendorError("Git is required for this command but was not found.") from error - except subprocess.TimeoutExpired as error: - raise VendorError( - f"Git command timed out after {_GIT_TIMEOUT_SECONDS} seconds: {' '.join(command)}" - ) from error - except subprocess.CalledProcessError as error: - stderr_value = error.stderr - if isinstance(stderr_value, bytes): - stderr_text = stderr_value.decode("utf-8", errors="replace") - else: - stderr_text = stderr_value or "" - raise VendorError( - f"Git command failed: {' '.join(command)}\n{stderr_text.strip()}" - ) from error - return result.stdout - - -def _read_git_blobs(repo: Path, object_ids: Sequence[bytes], source: str) -> list[bytes]: - batch_input = b"".join(object_id + b"\n" for object_id in object_ids) - output = _run_git( - ["-C", repo, "cat-file", "--batch"], - cwd=repo, - capture_bytes=True, - input_bytes=batch_input, - ) - assert isinstance(output, bytes) - blobs: list[bytes] = [] - position = 0 - for expected_object_id in object_ids: - header_end = output.find(b"\n", position) - if header_end < 0: - raise VendorError(f"Git produced truncated blob metadata for source {source!r}.") - header = output[position:header_end] - fields = header.split(b" ") - if len(fields) != 3: - raise VendorError(f"Git produced invalid blob metadata for source {source!r}.") - object_id, object_type, raw_size = fields - try: - size = int(raw_size) - except ValueError as error: - raise VendorError( - f"Git produced an invalid blob size for source {source!r}." - ) from error - if object_id != expected_object_id or object_type != b"blob" or size < 0: - raise VendorError(f"Git returned an unexpected object for source {source!r}.") - data_start = header_end + 1 - data_end = data_start + size - if data_end >= len(output) or output[data_end : data_end + 1] != b"\n": - raise VendorError(f"Git produced truncated blob data for source {source!r}.") - blobs.append(output[data_start:data_end]) - position = data_end + 1 - if position != len(output): - raise VendorError(f"Git produced trailing blob data for source {source!r}.") - return blobs - - -def _archive_tree(repo: Path, commit: str, source: str, patterns: Sequence[str]) -> Tree: - try: - listing = _run_git( - ["-C", repo, "ls-tree", "-r", "-z", "--full-tree", commit, "--", source], - cwd=repo, - capture_bytes=True, - ) - except VendorError as error: - raise VendorError( - f"Could not read source {source!r} at commit {commit}: {error}" - ) from error - assert isinstance(listing, bytes) - if listing and not listing.endswith(b"\0"): - raise VendorError(f"Git produced an invalid tree listing for source {source!r}.") - - source_path = PurePosixPath(source) - selected: list[tuple[str, bytes, bool]] = [] - seen_paths: set[str] = set() - for record in listing.split(b"\0")[:-1]: - metadata, separator, raw_path = record.partition(b"\t") - fields = metadata.split(b" ") - if not separator or len(fields) != 3: - raise VendorError(f"Git produced an invalid tree entry for source {source!r}.") - raw_mode, object_type, object_id = fields - try: - path = raw_path.decode("utf-8") - mode = raw_mode.decode("ascii") - except UnicodeError as error: - raise VendorError( - f"Git tree paths and modes must be valid UTF-8 for {source!r}." - ) from error - _validate_tree_relative_path(path) - try: - relative = PurePosixPath(path).relative_to(source_path).as_posix() - except ValueError as error: - raise VendorError(f"Git tree entry escaped source {source!r}: {path!r}.") from error - if relative in ("", "."): - continue - _validate_tree_relative_path(relative) - if object_type != b"blob" or mode not in ("100644", "100755"): - raise VendorError( - f"Selected upstream trees cannot contain links or special files: {relative}." - ) - if not _matches(relative, patterns): - continue - if relative in seen_paths: - raise VendorError( - f"Git produced a duplicate tree entry for source {source!r}: {relative}." - ) - if re.fullmatch(rb"[0-9a-f]{40}|[0-9a-f]{64}", object_id) is None: - raise VendorError(f"Git produced an invalid object ID for source {source!r}.") - seen_paths.add(relative) - selected.append((relative, object_id, mode == "100755")) - - blobs = _read_git_blobs(repo, [object_id for _, object_id, _ in selected], source) - tree = { - relative: FileEntry(data=data, executable=executable) - for (relative, _, executable), data in zip(selected, blobs, strict=True) - } - if not tree: - raise VendorError(f"Source {source!r} at {commit} selected no files.") - return tree - - -def _local_source_tree(vendor: Vendor, repo: Path) -> Tree: - repository = repo.resolve() - if not repository.is_dir(): - raise VendorError(f"Local upstream repository does not exist: {repository}.") - try: - _run_git( - ["-C", repository, "cat-file", "-e", f"{vendor.commit}^{{commit}}"], cwd=repository - ) - except VendorError as error: - raise VendorError( - f"Local repository {repository} does not contain locked commit {vendor.commit}." - ) from error - return _archive_tree(repository, vendor.commit, vendor.source, vendor.include) - - -@contextlib.contextmanager -def _remote_repository(vendor: Vendor) -> Iterator[Path]: - with tempfile.TemporaryDirectory(prefix="trtllm-vendor-fetch-") as temporary_directory: - repository = Path(temporary_directory) - try: - _run_git(["init", "--quiet"], cwd=repository) - _run_git(["remote", "add", "origin", vendor.url], cwd=repository) - _run_git(["ls-remote", "--quiet", "origin"], cwd=repository) - except VendorError as error: - raise UpstreamUnavailable( - f"Upstream unavailable for {vendor.name!r} at {vendor.url}: {error}" - ) from error - try: - try: - _run_git( - ["fetch", "--quiet", "--depth", "1", "origin", vendor.commit], cwd=repository - ) - except VendorError: - ref = None - if vendor.branch is not None: - ref = f"refs/heads/{vendor.branch}" - elif vendor.tag is not None: - ref = f"refs/tags/{vendor.tag}" - if ref is None: - raise - _run_git(["fetch", "--quiet", "origin", ref], cwd=repository) - _run_git(["cat-file", "-e", f"{vendor.commit}^{{commit}}"], cwd=repository) - except VendorError as error: - raise VendorError( - f"Upstream {vendor.url} is reachable but locked commit {vendor.commit} " - f"could not be fetched: {error}" - ) from error - yield repository - - -def _source_tree(vendor: Vendor, repo: Path | None) -> Tree: - if repo is not None: - return _local_source_tree(vendor, repo) - with _remote_repository(vendor) as remote: - return _archive_tree(remote, vendor.commit, vendor.source, vendor.include) - - -def _store_git_tree(repository: Path, tree: Mapping[str, FileEntry]) -> str: - _run_git(["read-tree", "--empty"], cwd=repository, isolated=True) - index_entries = bytearray() - for relative, entry in sorted(tree.items()): - _validate_tree_relative_path(relative) - object_id = _run_git( - ["hash-object", "-w", "--stdin", "--no-filters"], - cwd=repository, - capture_bytes=True, - isolated=True, - input_bytes=entry.data, - ).strip() - assert isinstance(object_id, bytes) - if re.fullmatch(rb"[0-9a-f]{40}", object_id) is None: - raise VendorError(f"Git produced an invalid blob object ID for {relative!r}.") - mode = b"100755" if entry.executable else b"100644" - index_entries.extend(mode + b" " + object_id + b"\t" + relative.encode("utf-8") + b"\0") - if index_entries: - _run_git( - ["update-index", "-z", "--index-info"], - cwd=repository, - capture_bytes=True, - isolated=True, - input_bytes=bytes(index_entries), - ) - object_id = _run_git(["write-tree"], cwd=repository, isolated=True) - assert isinstance(object_id, str) - object_id = object_id.strip() - if re.fullmatch(r"[0-9a-f]{40}", object_id) is None: - raise VendorError("Git produced an invalid tree object ID.") - return object_id - - -def _generate_patch( - baseline: Mapping[str, FileEntry], result_tree: Mapping[str, FileEntry] -) -> bytes: - with tempfile.TemporaryDirectory(prefix="trtllm-vendor-patch-") as temporary_directory: - repository = Path(temporary_directory) - _run_git(["init", "--quiet", "--object-format=sha1"], cwd=repository, isolated=True) - baseline_object = _store_git_tree(repository, baseline) - result_object = _store_git_tree(repository, result_tree) - # Keep selected attributes out of Git's index fallback during the tree diff. - _run_git(["read-tree", "--empty"], cwd=repository, isolated=True) - patch = _run_git( - [ - "-c", - f"core.attributesFile={os.devnull}", - "diff-tree", - "-p", - "-r", - "--binary", - "--full-index", - "--no-renames", - "--no-ext-diff", - "--no-textconv", - "--unified=3", - "--diff-algorithm=myers", - "--no-indent-heuristic", - baseline_object, - result_object, - ], - cwd=repository, - capture_bytes=True, - isolated=True, - ) - assert isinstance(patch, bytes) - if patch: - reproduced = _apply_patch_bytes(baseline, patch, ("**/*",)) - else: - reproduced = dict(baseline) - expected = dict(result_tree) - if reproduced != expected: - differing = sorted( - path - for path in set(reproduced) | set(expected) - if reproduced.get(path) != expected.get(path) - ) - raise VendorError( - f"Generated vendor patch does not exactly reproduce its result tree: {differing}." - ) - return patch - - -def _apply_patch( - baseline: Mapping[str, FileEntry], - patch_path: Path, - patterns: Sequence[str], -) -> Tree: - with tempfile.TemporaryDirectory(prefix="trtllm-vendor-apply-") as temporary_directory: - directory = Path(temporary_directory) - _write_tree(directory, baseline) - try: - # Stop before Git can inspect a parent that belongs to an enclosing worktree. - _run_git( - ["apply", "--check", "--binary", patch_path], - cwd=directory, - isolated=True, - repository_ceiling=directory.parent, - ) - _run_git( - ["apply", "--binary", patch_path], - cwd=directory, - isolated=True, - repository_ceiling=directory.parent, - ) - except VendorError as error: - raise VendorError(f"Failed to apply vendor patch {patch_path}: {error}") from error - return _read_all_directory_files(directory, patterns) - - -def _normal_tree(lock: LockFile, vendor: Vendor, repo: Path | None) -> Tree: - tree = _source_tree(vendor, repo) - if vendor.patch is None: - return tree - patch_path = _checked_root_path(lock.root, vendor.patch, f"Vendor {vendor.name!r} patch") - return _apply_patch(tree, patch_path, vendor.include) - - -def _current_tree(lock: LockFile, vendor: Vendor) -> Tree: - destination = _checked_root_path( - lock.root, vendor.destination, f"Vendor {vendor.name!r} destination" - ) - return _read_directory_tree(destination, vendor.include) - - -def _changed_paths( - baseline: Mapping[str, FileEntry], result_tree: Mapping[str, FileEntry] -) -> list[str]: - return sorted( - path - for path in set(baseline) | set(result_tree) - if baseline.get(path) != result_tree.get(path) - ) - - -def _select_vendors(lock: LockFile, name: str | None) -> list[Vendor]: - if name is None: - return [lock.vendors[key] for key in sorted(lock.vendors)] - try: - return [lock.vendors[name]] - except KeyError as error: - raise VendorError(f"Unknown vendor: {name!r}.") from error - - -def _require_vendor(lock: LockFile, name: str) -> Vendor: - return _select_vendors(lock, name)[0] - - -def _verify_offline(lock: LockFile, vendor: Vendor) -> None: - actual = _tree_digest(_current_tree(lock, vendor)) - if actual != vendor.digest: - raise VendorError( - f"Vendor {vendor.name!r} destination digest mismatch: " - f"expected {vendor.digest}, got {actual}." - ) - - -def _verify_source(lock: LockFile, vendor: Vendor, repo: Path | None) -> None: - normal = _normal_tree(lock, vendor, repo) - normal_digest = _tree_digest(normal) - if normal_digest != vendor.digest: - raise VendorError( - f"Vendor {vendor.name!r} locked materialization digest mismatch: " - f"expected {vendor.digest}, got {normal_digest}." - ) - current = _current_tree(lock, vendor) - if current != normal: - raise VendorError( - f"Vendor {vendor.name!r} destination differs from its upstream materialization." - ) - - -def _command_list(args: argparse.Namespace) -> None: - lock = _load_lock(args.lock) - for vendor in _select_vendors(lock, None): - state = "patched" if vendor.patch else "exact" - print(f"{vendor.name}\t{state}\t{vendor.commit}\t{vendor.destination}") - - -def _command_status(args: argparse.Namespace) -> None: - lock = _load_lock(args.lock) - failed = False - for vendor in _select_vendors(lock, args.name): - try: - _verify_offline(lock, vendor) - except VendorError as error: - failed = True - print(f"FAIL-LOCAL-INTEGRITY {vendor.name}: {error}") - else: - state = "patched" if vendor.patch is not None else "exact" - print(f"PASS-OFFLINE {vendor.name}: {state} at {vendor.commit}") - if failed: - raise VendorError("One or more vendor snapshots failed offline integrity checks.") - - -def _new_vendor_from_args(args: argparse.Namespace) -> Vendor: - include = args.include if args.include is not None else ["**/*"] - value: dict[str, object] = { - "url": args.url, - "commit": args.commit, - "source": args.source, - "destination": args.destination, - "include": include, - "digest": f"{_DIGEST_PREFIX}{'0' * 64}", - } - if args.branch is not None: - value["branch"] = args.branch - if args.tag is not None: - value["tag"] = args.tag - return _validate_vendor(args.name, value) - - -def _command_create(args: argparse.Namespace) -> None: - lock = _load_lock(args.lock, allow_missing=True) - if args.name in lock.vendors: - raise VendorError(f"Vendor {args.name!r} already exists.") - vendor = _new_vendor_from_args(args) - lock.vendors[vendor.name] = vendor - _validate_destination_layout(lock) - destination = _checked_root_path( - lock.root, vendor.destination, f"Vendor {vendor.name!r} destination" - ) - if args.adopt is None and destination.exists(): - raise VendorError( - f"Vendor {vendor.name!r} destination already exists; use --adopt exact or patched." - ) - source_tree = _source_tree(vendor, args.repo) - - destination_transaction: contextlib.AbstractContextManager[None] = contextlib.nullcontext() - patch_transaction: contextlib.AbstractContextManager[None] = contextlib.nullcontext() - if args.adopt is None: - result_tree = source_tree - destination_transaction = _replace_selected_destination_transaction( - lock, vendor, result_tree - ) - else: - if not destination.exists(): - raise VendorError(f"Cannot adopt missing destination for vendor {vendor.name!r}.") - result_tree = _current_tree(lock, vendor) - if args.adopt == "exact": - if result_tree != source_tree: - changed = _changed_paths(source_tree, result_tree) - raise VendorError( - f"Vendor {vendor.name!r} is not exact; differing files: {changed}. " - "Use --adopt patched to record the difference." - ) - else: - patch = _generate_patch(source_tree, result_tree) - if patch: - patch_relative = f"3rdparty/vendor_patches/{vendor.name}.patch" - patch_path = _checked_root_path( - lock.root, patch_relative, f"Vendor {vendor.name!r} patch" - ) - vendor.patch = patch_relative - vendor.patch_digest = _patch_digest(patch) - patch_transaction = _atomic_file_replacement(patch_path, patch) - vendor.digest = _tree_digest(result_tree) - with contextlib.ExitStack() as transaction: - transaction.enter_context(destination_transaction) - transaction.enter_context(patch_transaction) - _save_lock_checked(lock, f"lock state for new vendor {vendor.name!r}") - print(f"Created {vendor.name} at {vendor.commit} ({'patched' if vendor.patch else 'exact'}).") - - -def _command_sync(args: argparse.Namespace) -> None: - lock = _load_lock(args.lock) - vendor = _require_vendor(lock, args.name) - normal = _normal_tree(lock, vendor, args.repo) - digest = _tree_digest(normal) - if digest != vendor.digest: - raise VendorError( - f"Vendor {vendor.name!r} materializes to {digest}, but the lock records {vendor.digest}. " - "Refresh the patch or pin instead of silently changing the accepted digest." - ) - _replace_selected_destination(lock, vendor, normal) - print(f"Synchronized {vendor.name} to {vendor.commit}.") - - -def _command_check(args: argparse.Namespace) -> None: - if args.require_access and not args.upstream: - raise VendorError("--require-access requires --upstream.") - if args.repo is not None and args.name is None: - raise VendorError("--repo requires one vendor name.") - if args.repo is not None and args.upstream: - raise VendorError("Use either --repo or --upstream, not both.") - if args.repo is not None and args.offline: - raise VendorError("Use either --repo or --offline, not both.") - lock = _load_lock(args.lock) - vendors = _select_vendors(lock, args.name) - for vendor in vendors: - _verify_offline(lock, vendor) - print(f"PASS-OFFLINE {vendor.name}: local snapshot verified; upstream not verified.") - - if args.repo is not None: - vendor = vendors[0] - _verify_source(lock, vendor, args.repo) - print(f"PASS-LOCAL {vendor.name}: verified from {args.repo}.") - return - if not args.upstream: - return - for vendor in vendors: - try: - _verify_source(lock, vendor, None) - except UpstreamUnavailable as error: - if args.require_access: - raise VendorError(f"REMOTE-UNAVAILABLE {vendor.name}: {error}") from error - print(f"SKIP-REMOTE-UNAVAILABLE {vendor.name}: {error}", file=sys.stderr) - continue - print(f"PASS-REMOTE {vendor.name}: verified from {vendor.url}.") - - -def _update_patch(lock: LockFile, vendor: Vendor, repo: Path | None, *, create: bool) -> None: - if create and vendor.patch is not None: - raise VendorError(f"Vendor {vendor.name!r} already has a patch; use refresh.") - if not create and vendor.patch is None: - raise VendorError(f"Vendor {vendor.name!r} has no patch; use create.") - raw = _source_tree(vendor, repo) - current = _current_tree(lock, vendor) - patch = _generate_patch(raw, current) - old_patch = vendor.patch - old_patch_path = ( - None - if old_patch is None - else _checked_root_path(lock.root, old_patch, f"Vendor {vendor.name!r} patch") - ) - old_patch_content = ( - old_patch_path.read_bytes() - if old_patch_path is not None and old_patch_path.is_file() - else None - ) - written_patch_path: Path | None = None - if patch: - patch_relative = old_patch or f"3rdparty/vendor_patches/{vendor.name}.patch" - patch_path = _checked_root_path(lock.root, patch_relative, f"Vendor {vendor.name!r} patch") - _atomic_write(patch_path, patch) - written_patch_path = patch_path - vendor.patch = patch_relative - vendor.patch_digest = _patch_digest(patch) - else: - vendor.patch = None - vendor.patch_digest = None - vendor.digest = _tree_digest(current) - try: - _save_lock(lock) - except (OSError, yaml.YAMLError) as error: - if old_patch_path is not None and old_patch_content is not None: - _atomic_write(old_patch_path, old_patch_content) - elif written_patch_path is not None: - written_patch_path.unlink(missing_ok=True) - raise VendorError(f"Failed to save refreshed patch state: {error}") from error - if not patch and old_patch_path is not None: - old_patch_path.unlink(missing_ok=True) - print(f"{'Created' if create else 'Refreshed'} patch for {vendor.name}.") - - -def _command_patch(args: argparse.Namespace) -> None: - lock = _load_lock(args.lock, allow_patch_recovery=True) - vendor = _require_vendor(lock, args.name) - if args.action == "create": - _update_patch(lock, vendor, args.repo, create=True) - return - if args.action == "refresh": - _update_patch(lock, vendor, args.repo, create=False) - return - if vendor.patch is None: - raise VendorError(f"Vendor {vendor.name!r} has no patch to drop.") - raw = _source_tree(vendor, args.repo) - current = _current_tree(lock, vendor) - if current != raw: - raise VendorError( - f"Vendor {vendor.name!r} destination is not exact upstream; refusing to drop its patch." - ) - patch_path = _checked_root_path(lock.root, vendor.patch, f"Vendor {vendor.name!r} patch") - vendor.patch = None - vendor.patch_digest = None - vendor.digest = _tree_digest(raw) - _save_lock(lock) - patch_path.unlink(missing_ok=True) - print(f"Dropped patch for {vendor.name}.") - - -def _worktree_head(repo: Path) -> str: - result = _run_git(["-C", repo, "rev-parse", "HEAD"], cwd=repo) - assert isinstance(result, str) - return result.strip() - - -def _require_clean_source_worktree(repo: Path, source: str, patterns: Sequence[str]) -> None: - result = _run_git( - ["-C", repo, "status", "--porcelain", "--untracked-files=all", "--", source], - cwd=repo, - ) - assert isinstance(result, str) - if result.strip(): - raise VendorError(f"Local upstream source {source!r} has uncommitted changes.") - ignored = _run_git( - [ - "-C", - repo, - "ls-files", - "-z", - "--others", - "--ignored", - "--exclude-standard", - "--", - source, - ], - cwd=repo, - capture_bytes=True, - ) - assert isinstance(ignored, bytes) - if ignored and not ignored.endswith(b"\0"): - raise VendorError(f"Git produced an invalid ignored-file listing for source {source!r}.") - source_path = PurePosixPath(source) - selected: list[str] = [] - for raw_path in ignored.split(b"\0")[:-1]: - try: - path = raw_path.decode("utf-8") - except UnicodeError as error: - raise VendorError( - f"Ignored paths under local upstream source {source!r} must be valid UTF-8." - ) from error - _validate_tree_relative_path(path) - try: - relative = PurePosixPath(path).relative_to(source_path).as_posix() - except ValueError as error: - raise VendorError( - f"Git ignored-file entry escaped source {source!r}: {path!r}." - ) from error - if relative not in ("", ".") and _matches(relative, patterns): - selected.append(relative) - if selected: - raise VendorError( - f"Local upstream source {source!r} has ignored untracked files selected " - f"for export: {selected}." - ) - - -def _apply_patch_bytes( - baseline: Mapping[str, FileEntry], - patch: bytes, - patterns: Sequence[str], -) -> Tree: - with tempfile.TemporaryDirectory(prefix="trtllm-vendor-delta-") as temporary_directory: - patch_path = Path(temporary_directory) / "delta.patch" - patch_path.write_bytes(patch) - return _apply_patch(baseline, patch_path, patterns) - - -def _command_export(args: argparse.Namespace) -> None: - lock = _load_lock(args.lock) - vendor = _require_vendor(lock, args.name) - repository = args.repo.resolve() - if _worktree_head(repository) != vendor.commit: - raise VendorError( - f"Local upstream HEAD must equal locked commit {vendor.commit} before export." - ) - _require_clean_source_worktree(repository, vendor.source, vendor.include) - raw = _local_source_tree(vendor, repository) - normal = raw - if vendor.patch is not None: - patch_path = _checked_root_path(lock.root, vendor.patch, f"Vendor {vendor.name!r} patch") - normal = _apply_patch(raw, patch_path, vendor.include) - if _tree_digest(normal) != vendor.digest: - raise VendorError( - f"Vendor {vendor.name!r} normal materialization does not match its digest." - ) - current = _current_tree(lock, vendor) - delta = _generate_patch(normal, current) - if not delta: - raise VendorError(f"Vendor {vendor.name!r} has no downstream change to export.") - try: - exported = _apply_patch_bytes(raw, delta, vendor.include) - except VendorError as error: - raise VendorError( - "The local change overlaps the vendor compatibility patch and cannot be exported " - f"automatically: {error}" - ) from error - export_vendor = dataclasses.replace(vendor, destination=vendor.source) - export_lock = LockFile(path=lock.path, root=repository, vendors={vendor.name: export_vendor}) - _replace_selected_destination(export_lock, export_vendor, exported) - print( - f"Exported {vendor.name} changes to {repository / vendor.source}; review and commit them upstream." - ) - - -def _command_pin(args: argparse.Namespace) -> None: - lock = _load_lock(args.lock) - vendor = _require_vendor(lock, args.name) - repository = args.repo.resolve() - commit = args.commit or _worktree_head(repository) - value = vendor.to_mapping() - value["url"] = args.url or vendor.url - value["commit"] = commit - if args.branch is not None: - value["branch"] = args.branch - value.pop("tag", None) - elif args.tag is not None: - value["tag"] = args.tag - value.pop("branch", None) - value["digest"] = vendor.digest - proposed = _validate_vendor(vendor.name, value) - raw = _local_source_tree(proposed, repository) - current = _current_tree(lock, vendor) - drop_patch = False - if proposed.patch is None: - materialized = raw - else: - patch_path = _checked_root_path(lock.root, proposed.patch, f"Vendor {vendor.name!r} patch") - try: - materialized = _apply_patch(raw, patch_path, proposed.include) - except VendorError: - if raw != current: - raise - materialized = raw - drop_patch = True - if materialized != current: - if proposed.patch is not None and raw == current: - materialized = raw - drop_patch = True - else: - changed = _changed_paths(materialized, current) - raise VendorError( - f"New pin does not reproduce the destination for {vendor.name!r}; " - f"differing files: {changed}." - ) - old_patch = proposed.patch - if drop_patch: - proposed.patch = None - proposed.patch_digest = None - proposed.digest = _tree_digest(materialized) - lock.vendors[vendor.name] = proposed - patch_path: Path | None = None - if drop_patch and old_patch is not None: - patch_path = _checked_root_path(lock.root, old_patch, f"Vendor {vendor.name!r} patch") - if patch_path.is_symlink() or not patch_path.is_file(): - raise VendorError(f"Generated patch path must be a regular file: {patch_path}.") - _save_lock_checked(lock, f"new pin for vendor {vendor.name!r}") - if patch_path is not None: - try: - _sync_directory(lock.path.parent) - except OSError as error: - raise VendorError( - f"Failed to make the new pin durable for vendor {vendor.name!r}: {error}" - ) from error - try: - patch_path.unlink() - except OSError as error: - print( - f"warning: Pinned vendor {vendor.name!r}, but could not remove its absorbed " - f"patch {patch_path}: {error}. The lock is committed and valid; this patch is " - "now an unreferenced orphan. Delete that file manually.", - file=sys.stderr, - ) - print(f"Pinned {vendor.name} to {commit}; offline enforcement restored.") - - -def _command_remove(args: argparse.Namespace) -> None: - lock = _load_lock(args.lock) - vendor = _require_vendor(lock, args.name) - del lock.vendors[vendor.name] - _save_lock(lock) - if vendor.patch is not None and all( - other.patch != vendor.patch for other in lock.vendors.values() - ): - _checked_root_path(lock.root, vendor.patch, f"Vendor {vendor.name!r} patch").unlink( - missing_ok=True - ) - print(f"Removed {vendor.name} from the lock; destination bytes were preserved.") - - -def _add_repo_argument(parser: argparse.ArgumentParser, *, required: bool = False) -> None: - parser.add_argument( - "--repo", - type=Path, - required=required, - help="Local Git checkout containing the locked commit; never contacts the lock URL.", - ) - - -def _build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--lock", - type=Path, - default=_DEFAULT_LOCK, - help=f"Vendor lock path (default: {_DEFAULT_LOCK.relative_to(_REPO_ROOT)}).", - ) - commands = parser.add_subparsers(dest="command", required=True) - - list_parser = commands.add_parser("list", help="List configured vendors.") - list_parser.set_defaults(handler=_command_list) - - status = commands.add_parser("status", help="Show offline vendor status.") - status.add_argument("name", nargs="?") - status.set_defaults(handler=_command_status) - - create = commands.add_parser("create", help="Create or adopt a vendor entry.") - create.add_argument("name") - create.add_argument("--url", required=True) - reference = create.add_mutually_exclusive_group() - reference.add_argument("--branch") - reference.add_argument("--tag") - create.add_argument("--commit", required=True) - create.add_argument("--source", required=True) - create.add_argument("--destination", required=True) - create.add_argument("--include", action="append") - create.add_argument("--adopt", choices=("exact", "patched")) - _add_repo_argument(create) - create.set_defaults(handler=_command_create) - - sync = commands.add_parser("sync", help="Restore a locked vendor materialization.") - sync.add_argument("name") - _add_repo_argument(sync) - sync.set_defaults(handler=_command_sync) - - check = commands.add_parser("check", help="Verify vendor integrity.") - check.add_argument("name", nargs="?") - check_mode = check.add_mutually_exclusive_group() - check_mode.add_argument( - "--offline", - action="store_true", - help="Explicitly select the default no-network integrity check.", - ) - check_mode.add_argument( - "--upstream", action="store_true", help="Attempt upstream verification." - ) - check.add_argument( - "--require-access", - action="store_true", - help="Fail when an upstream is inaccessible; for trusted maintainer environments.", - ) - _add_repo_argument(check) - check.set_defaults(handler=_command_check) - - patch = commands.add_parser("patch", help="Create, refresh, or drop a downstream patch.") - patch.add_argument("name") - patch.add_argument("action", choices=("create", "refresh", "drop")) - _add_repo_argument(patch) - patch.set_defaults(handler=_command_patch) - - export = commands.add_parser("export", help="Export pending destination changes upstream.") - export.add_argument("name") - _add_repo_argument(export, required=True) - export.set_defaults(handler=_command_export) - - pin = commands.add_parser("pin", help="Pin a vendor to a new committed source revision.") - pin.add_argument("name") - pin.add_argument("--url") - pin_reference = pin.add_mutually_exclusive_group() - pin_reference.add_argument("--branch") - pin_reference.add_argument("--tag") - pin.add_argument("--commit") - _add_repo_argument(pin, required=True) - pin.set_defaults(handler=_command_pin) - - remove = commands.add_parser( - "remove", help="Remove a vendor lock entry, preserving its destination." - ) - remove.add_argument("name") - remove.set_defaults(handler=_command_remove) - return parser - - -def main(argv: Sequence[str] | None = None) -> int: - """Run the vendoring command-line interface.""" - parser = _build_parser() - args = parser.parse_args(argv) - if args.lock != _DEFAULT_LOCK: - args.lock = args.lock.resolve() - try: - args.handler(args) - except VendorError as error: - print(f"error: {error}", file=sys.stderr) - return 1 - return 0 +"""Compatibility entry point; use scripts/vendor/manage.py for new invocations.""" +if __package__: + from .vendor.manage import main +else: + from vendor.manage import main if __name__ == "__main__": raise SystemExit(main()) diff --git a/tests/unittest/others/test_maintain_prims_ts.py b/tests/unittest/others/test_maintain_prims_ts.py index 77d6d3cb9777..38b363ced110 100644 --- a/tests/unittest/others/test_maintain_prims_ts.py +++ b/tests/unittest/others/test_maintain_prims_ts.py @@ -18,16 +18,20 @@ # tests/unittest/scripts shadows the repository's scripts namespace. Load the # standalone tool as an isolated package, without editing sys.path. -_SCRIPTS = Path(__file__).resolve().parents[3] / "scripts" +_SCRIPTS = Path(__file__).resolve().parents[3] / "scripts" / "vendor" _SPEC = importlib.util.spec_from_file_location( "prims_ts_maintenance_test", - _SCRIPTS / "maintain_prims_ts.py", + _SCRIPTS / "promote.py", submodule_search_locations=[str(_SCRIPTS)], ) m = importlib.util.module_from_spec(_SPEC) sys.modules[_SPEC.name] = m _SPEC.loader.exec_module(m) +_VENDOR = "example-vendor" +_CONSUMER = "NVIDIA/TensorRT-LLM" +_UPSTREAM = "flashinfer-ai/flashinfer" + pytestmark = pytest.mark.cpu_only @@ -64,6 +68,7 @@ class World(m.GitHub): """GitHub facade whose immutable data comes from temporary Git repositories.""" def __init__(self, root: Path) -> None: + super().__init__(_CONSUMER, _VENDOR, _UPSTREAM) self.root = root self.source = _init(root / "flashinfer") _write(self.source, "prims_ts/kernel.py", "VALUE = 0\n") @@ -81,7 +86,7 @@ def __init__(self, root: Path) -> None: "--lock", str(self.consumer / m._LOCK), "create", - m._VENDOR, + _VENDOR, "--url", "https://github.com/maintainer/flashinfer.git", "--branch", @@ -108,7 +113,7 @@ def __init__(self, root: Path) -> None: "--lock", str(self.consumer / m._LOCK), "pin", - m._VENDOR, + _VENDOR, "--url", "https://github.com/dev/flashinfer.git", "--branch", @@ -144,7 +149,8 @@ def __init__(self, root: Path) -> None: def args(self, **overrides: object) -> argparse.Namespace: values = dict( - trtllm_pr="17", + source_pr="17", + legacy_prims_ts=False, canonical_repo="maintainer/flashinfer", canonical_branch="trtllm-prims-ts-dev", upstream_pr=["4829"], @@ -152,7 +158,7 @@ def args(self, **overrides: object) -> argparse.Namespace: map_upstream=[], repo=self.consumer, fork=None, - flashinfer_repo=self.source, + source_repo=self.source, worktree=self.root / "followup", publish=False, auto_merge=False, @@ -176,7 +182,7 @@ def commit_data(self, repo: Path, sha: str) -> dict: } def compare(self, repo: str, base: str, head: str, *, all_commits: bool = False) -> dict: - source = self.consumer if repo == m._TRTLLM else self.source + source = self.consumer if repo == _CONSUMER else self.source commits = _git(source, "rev-list", "--reverse", f"{base}..{head}").splitlines() return { "merge_base_commit": {"sha": _git(source, "merge-base", base, head)}, @@ -252,7 +258,7 @@ def api( raise RuntimeError("simulated comment failure after posting") return copy.deepcopy(comment) if not route: - root = m._FLASHINFER if name == "flashinfer" else m._TRTLLM + root = _UPSTREAM if name == "flashinfer" else _CONSUMER return { "source": {"full_name": root}, "allow_auto_merge": True, @@ -262,9 +268,9 @@ def api( return [{"type": "required_status_checks"}] if route.startswith("git/ref/heads/"): branch = route[len("git/ref/heads/") :] - if repository == m._TRTLLM: + if repository == _CONSUMER: return {"object": {"sha": self.main}} - if repository == m._FLASHINFER: + if repository == _UPSTREAM: return {"object": {"sha": self.upstream}} remote = self.canonical if name == "flashinfer" else self.fork result = subprocess.run( @@ -301,7 +307,7 @@ def api( return copy.deepcopy(self.prs[99]) if route.startswith("pulls/"): number = int(route.split("/")[1]) - if repository == m._FLASHINFER: + if repository == _UPSTREAM: return { "number": number, "base": {"ref": "main", "sha": self.upstream}, @@ -326,7 +332,7 @@ def api( def world(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> World: instance = World(tmp_path) monkeypatch.setattr(m, "_git", instance.routed_git) - monkeypatch.setattr(m, "GitHub", lambda: instance) + monkeypatch.setattr(m, "GitHub", lambda *args: instance) return instance @@ -337,6 +343,12 @@ def test_dry_run_has_no_writes_and_pins_merged_not_live_head( m.main( [ "promote", + "--vendor", + _VENDOR, + "--upstream-repo", + _UPSTREAM, + "--canonical-branch", + "trtllm-prims-ts-dev", "--trtllm-pr", "17", "--canonical-repo", @@ -365,7 +377,7 @@ def test_publish_signed_lock_only_and_repeat(world: World) -> None: assert _git(world.canonical, "rev-parse", "trtllm-prims-ts-dev") == world.reviewed head = pr["head"]["sha"] assert _git(world.consumer, "diff", "--name-only", f"{head}^", head) == m._LOCK - assert m._lock_at(world, m._TRTLLM, head).to_mapping() == plan.promoted + assert m._lock_at(world, _CONSUMER, head).to_mapping() == plan.promoted message = _git(world.consumer, "show", "-s", "--format=%B", head) assert m._record_from_message(message) == plan.record assert "Signed-off-by: Promotion Test " in message @@ -385,7 +397,7 @@ def test_auto_merge_message_and_immediate_merge(world: World, state: str) -> Non m._publish(args, world, plan) if state == "CLEAN": assert world.prs[99]["merged"] - m._verify_remote_commit(world, plan, m._TRTLLM, world.main) + m._verify_remote_commit(world, plan, _CONSUMER, world.main) m._publish(args, world, m._make_plan(args, world)) else: saved = world.prs[99]["auto_merge"] @@ -398,15 +410,102 @@ def test_resume_after_upstream_head_moves_retains_snapshot(world: World) -> None args = world.args(publish=True) original = m._make_plan(args, world) m._publish(args, world, original) - changed = world.get(f"repos/{m._FLASHINFER}/pulls/4829") + changed = world.get(f"repos/{_UPSTREAM}/pulls/4829") changed["head"]["sha"] = "b" * 40 - world.overrides[f"repos/{m._FLASHINFER}/pulls/4829"] = changed + world.overrides[f"repos/{_UPSTREAM}/pulls/4829"] = changed resumed = m._make_plan(args, world) assert resumed.record != original.record m._publish(args, world, resumed) assert resumed.record == original.record +def test_bot_attribution_survives_retry_and_squash_merge(world: World) -> None: + args = world.args(publish=True, auto_merge=True) + original = m._make_plan(args, world) + provenance = { + "attribution": { + world.first: {"method": "same-commit"}, + world.reviewed: { + "method": "author-assertion", + "reason": "downstream adaptation", + "refresh_policy": "retain-until-verified", + }, + }, + "contributor_metadata": { + "source_head": "a" * 40, + "metadata_digest": "b" * 64, + "pr_author": "contributor", + }, + } + original.record.update(provenance) + m._publish(args, world, original) + resumed = m._make_plan(args, world) + resumed.record.update(provenance) + m._publish(args, world, resumed) + # Model GitHub completing the already-armed auto-merge asynchronously. + followup = world.prs[99] + saved = followup["auto_merge"] + tree = _git(world.consumer, "rev-parse", f"{followup['head']['sha']}^{{tree}}") + world.main = _git( + world.consumer, + "commit-tree", + tree, + "-p", + world.main, + text=f"{saved['commit_title']}\n\n{saved['commit_message']}\n", + ) + followup.update(merged=True, merged_at="today", state="closed", merge_commit_sha=world.main) + completed = m._make_plan(args, world) + assert completed.completed_pr["merged"] + assert completed.record == original.record + + +def test_non_main_consumer_and_upstream_branches(world: World) -> None: + world.base_branch = "release/next" + world.upstream_branch = "develop" + source = world.get(f"repos/{_CONSUMER}/pulls/17") + source["base"]["ref"] = world.base_branch + upstream = world.get(f"repos/{_UPSTREAM}/pulls/4829") + upstream["base"]["ref"] = world.upstream_branch + world.overrides[f"repos/{_CONSUMER}/pulls/17"] = source + world.overrides[f"repos/{_UPSTREAM}/pulls/4829"] = upstream + world.overrides[f"repos/{_CONSUMER}/rules/branches/{world.base_branch}"] = [] + args = world.args(publish=True, auto_merge=True) + plan = m._make_plan(args, world) + m._publish(args, world, plan) + assert plan.record["consumer_base"] == "release/next" + assert plan.record["upstream_branch"] == "develop" + creates = [ + payload + for method, endpoint, payload in world.calls + if method == "POST" and endpoint.endswith("/pulls") + ] + assert creates[0]["base"] == "release/next" + + +@pytest.mark.parametrize( + "content", + ["vendors: [", "schema_version: 1\nvendors: &v {toy: *v}", "schema_version: 1\nvendors: []"], +) +def test_malformed_remote_lock_is_actionable(world: World, content: str) -> None: + world.overrides[f"repos/{_CONSUMER}/contents/{m._LOCK}?ref={world.merged}"] = { + "encoding": "base64", + "content": base64.b64encode(content.encode()).decode(), + } + with pytest.raises(ValueError): + m._lock_document_at(world, _CONSUMER, world.merged) + + +def test_remote_lock_accepts_github_wrapped_base64(world: World) -> None: + endpoint = f"repos/{_CONSUMER}/contents/{m._LOCK}?ref={world.merged}" + response = world.get(endpoint) + original = m._lock_document_at(world, _CONSUMER, world.merged) + encoded = response["content"] + response["content"] = "\n".join(encoded[i : i + 60] for i in range(0, len(encoded), 60)) + "\n" + world.overrides[endpoint] = response + assert m._lock_document_at(world, _CONSUMER, world.merged) == original + + def test_resume_after_push_before_pr_creation(world: World) -> None: args = world.args(publish=True) world.fail_create = True @@ -428,7 +527,7 @@ def test_auto_merge_tampering_disables_request(world: World) -> None: @pytest.mark.parametrize("field,value", [("merged", False), ("base", {"ref": "release"})]) def test_unmerged_or_wrong_base_rejected(world: World, field: str, value: object) -> None: - endpoint = f"repos/{m._TRTLLM}/pulls/17" + endpoint = f"repos/{_CONSUMER}/pulls/17" changed = world.get(endpoint) changed[field] = value world.overrides[endpoint] = changed @@ -463,7 +562,7 @@ def divergent(*args: object, **kwargs: object) -> dict: def test_upstream_closed_unmerged_rejected(world: World) -> None: - endpoint = f"repos/{m._FLASHINFER}/pulls/4829" + endpoint = f"repos/{_UPSTREAM}/pulls/4829" changed = world.get(endpoint) changed.update(state="closed", merged=False) world.overrides[endpoint] = changed @@ -474,11 +573,11 @@ def test_upstream_closed_unmerged_rejected(world: World) -> None: @pytest.mark.parametrize("setting", ["auto", "squash", "queue"]) def test_unsupported_auto_merge_settings_fail_before_push(world: World, setting: str) -> None: if setting == "queue": - world.overrides[f"repos/{m._TRTLLM}/rules/branches/main"] = [{"type": "merge_queue"}] + world.overrides[f"repos/{_CONSUMER}/rules/branches/main"] = [{"type": "merge_queue"}] else: - data = world.get(f"repos/{m._TRTLLM}") + data = world.get(f"repos/{_CONSUMER}") data[f"allow_{'auto_merge' if setting == 'auto' else 'squash_merge'}"] = False - world.overrides[f"repos/{m._TRTLLM}"] = data + world.overrides[f"repos/{_CONSUMER}"] = data args = world.args(publish=True, auto_merge=True) with pytest.raises(ValueError, match="allow|Merge queues"): m._publish(args, world, m._make_plan(args, world)) @@ -565,6 +664,12 @@ def test_cli_invalid_flags(flags: list[str]) -> None: m.main( [ "promote", + "--vendor", + _VENDOR, + "--upstream-repo", + _UPSTREAM, + "--canonical-branch", + "trtllm-prims-ts-dev", "--trtllm-pr", "17", "--canonical-repo", @@ -643,7 +748,7 @@ def tampered(gh: m.GitHub, repo: str, sha: str) -> dict: monkeypatch.setattr(m, "_lock_document_at", tampered) with pytest.raises(ValueError, match="branch/URL-only"): - m._verify_remote_commit(world, plan, m._TRTLLM, pr["head"]["sha"]) + m._verify_remote_commit(world, plan, _CONSUMER, pr["head"]["sha"]) def test_unmerged_parent_cannot_hide_unrelated_commits( @@ -656,13 +761,13 @@ def test_unmerged_parent_cannot_hide_unrelated_commits( def unrelated(repo: str, base: str, head: str, **kwargs: object) -> dict: result = original(repo, base, head, **kwargs) - if repo == m._TRTLLM: + if repo == _CONSUMER: result["merge_base_commit"]["sha"] = world.base return result monkeypatch.setattr(world, "compare", unrelated) - with pytest.raises(ValueError, match="not on TRT-LLM main"): - m._verify_remote_commit(world, plan, m._TRTLLM, pr["head"]["sha"]) + with pytest.raises(ValueError, match="not on the consumer base branch"): + m._verify_remote_commit(world, plan, _CONSUMER, pr["head"]["sha"]) def test_wait_checks_final_commit(world: World) -> None: @@ -684,7 +789,7 @@ def test_wait_timeout_preserves_enabled_auto_merge( def test_source_fetch_uses_reviewed_sha_when_no_local_checkout(world: World) -> None: - args = world.args(publish=True, flashinfer_repo=None) + args = world.args(publish=True, source_repo=None) m._publish(args, world, m._make_plan(args, world)) assert any(args[0] == "fetch" and args[-1] == world.reviewed for args in world.git_calls) @@ -713,8 +818,8 @@ def test_unpaired_promotion_publishes_and_resumes(world: World) -> None: m._publish(args, world, plan) m._publish(args, world, m._make_plan(args, world)) assert len(world.prs) == 1 - assert "No paired FlashInfer PR" in world.prs[99]["body"] - assert not any(f"repos/{m._FLASHINFER}/pulls/" in call[1] for call in world.calls) + assert "No paired upstream PR" in world.prs[99]["body"] + assert not any(f"repos/{_UPSTREAM}/pulls/" in call[1] for call in world.calls) changed = copy.deepcopy(plan.record) changed["changes"][0]["refresh_policy"] = "drop" with pytest.raises(ValueError, match="reason/policy"): @@ -753,7 +858,7 @@ def test_canonical_branch_name_on_developer_fork(world: World, same_fork: bool) return plan = m._make_plan(args, world) m._publish(args, world, plan) - actual = m.vendor._load_lock(args.worktree / m._LOCK).vendors[m._VENDOR].to_mapping() + actual = m.vendor._load_lock(args.worktree / m._LOCK).vendors[_VENDOR].to_mapping() assert actual == {**plan.reviewed.to_mapping(), "url": plan.previous.url} @@ -762,6 +867,12 @@ def test_unpaired_cli_dry_run(world: World, capsys: pytest.CaptureFixture) -> No m.main( [ "promote", + "--vendor", + _VENDOR, + "--upstream-repo", + _UPSTREAM, + "--canonical-branch", + "trtllm-prims-ts-dev", "--trtllm-pr", "17", "--canonical-repo", @@ -813,16 +924,16 @@ def test_completed_promotion_ignores_later_live_state( _git(world.consumer, "switch", "--detach", world.main) _write(world.consumer, "later.txt", "later main commit\n") world.main = _commit(world.consumer, "later main update") - world.overrides[f"repos/{m._TRTLLM}"] = {"allow_auto_merge": False} + world.overrides[f"repos/{_CONSUMER}"] = {"allow_auto_merge": False} world.overrides["repos/maintainer/flashinfer/git/ref/heads/trtllm-prims-ts-dev"] = { "object": {"sha": "b" * 40} } world.overrides["repos/dev/flashinfer"] = {} - world.overrides[f"repos/{m._FLASHINFER}/pulls/4829"] = {"state": "closed", "merged": False} + world.overrides[f"repos/{_UPSTREAM}/pulls/4829"] = {"state": "closed", "merged": False} original_compare = world.compare def compare(repo: str, base: str, head: str, **kwargs: object) -> dict: - assert repo == m._TRTLLM, "Completed retry must not query old source history" + assert repo == _CONSUMER, "Completed retry must not query old source history" return original_compare(repo, base, head, **kwargs) monkeypatch.setattr(world, "compare", compare) @@ -981,7 +1092,7 @@ def test_ci_request_rejects_changed_pr(world: World, change: str) -> None: def test_generated_body_passes_repository_checklist(world: World) -> None: - checker = runpy.run_path(str(_SCRIPTS.parent / ".github/scripts/pr_checklist_check.py")) + checker = runpy.run_path(str(_SCRIPTS.parent.parent / ".github/scripts/pr_checklist_check.py")) plan = m._make_plan(world.args(), world) body = m._pr_body(plan, f"{plan.title}\n\n{m._record_text(plan.record)}") assert checker["find_unresolved_tasks"](body) == [] diff --git a/tests/unittest/others/test_vendor_bot.py b/tests/unittest/others/test_vendor_bot.py new file mode 100644 index 000000000000..29ac3bd6daa0 --- /dev/null +++ b/tests/unittest/others/test_vendor_bot.py @@ -0,0 +1,560 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Hermetic tests for metadata, attribution, local state, and author feedback.""" + +from __future__ import annotations + +import argparse +import copy +import importlib +import importlib.util +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +_ROOT = Path(__file__).resolve().parents[3] +_PACKAGE = "vendor_bot_test" +_SPEC = importlib.util.spec_from_file_location( + _PACKAGE, + _ROOT / "scripts/vendor/__init__.py", + submodule_search_locations=[str(_ROOT / "scripts/vendor")], +) +_MODULE = importlib.util.module_from_spec(_SPEC) +sys.modules[_PACKAGE] = _MODULE +_SPEC.loader.exec_module(_MODULE) +b = importlib.import_module(f"{_PACKAGE}.bot") +m = importlib.import_module(f"{_PACKAGE}.metadata") +p = importlib.import_module(f"{_PACKAGE}.promote") + +pytestmark = pytest.mark.cpu_only + + +def _body(content: str) -> str: + return f"{m.BEGIN}\n```yaml\nschema_version: 1\nvendors:\n toy:\n{content}\n```\n{m.END}" + + +def test_metadata_only_needs_upstream_urls() -> None: + entry = m.parse( + _body(" upstream_prs:\n - https://github.com/org/library/pull/42"), + "toy", + "org/library", + ) + assert entry == {"upstream_prs": [42], "unpaired_reason": None, "resolutions": {}} + + +@pytest.mark.parametrize( + "content", + [ + " upstream_prs: []", + " upstream_prs: https://github.com/org/library/pull/42", + " upstream_prs: [https://github.com/other/library/pull/42]", + " upstream_prs: [https://github.com/org/library/pull/42, https://github.com/org/library/pull/42]", + " unpaired_reason: ''", + " unpaired_reason: test\n upstream_prs: []", + " canonical_repo: attacker/repo", + " upstream_prs: []\n upstream_prs: []", + " upstream_prs: [", + " upstream_prs: [https://github.com/org/library/pull/42]\n resolutions: []", + " upstream_prs: &prs [https://github.com/org/library/pull/42]", + " upstream_prs: &prs [*prs]", + ], +) +def test_invalid_metadata_rejected(content: str) -> None: + with pytest.raises(ValueError): + m.parse(_body(content), "toy", "org/library") + + +@pytest.mark.parametrize( + "body", ["", "missing", m.template("toy", "org/library") * 2, m.END + m.BEGIN] +) +def test_missing_or_duplicate_marked_blocks(body: str) -> None: + with pytest.raises(ValueError): + m.parse(body, "toy", "org/library") + + +def test_author_resolution_requires_explanation() -> None: + content = ( + " upstream_prs: [https://github.com/org/library/pull/42]\n resolutions:\n " + + "a" * 40 + + ":\n upstream_pr: https://github.com/org/library/pull/42\n reason: adapted ABI" + ) + entry = m.parse(_body(content), "toy", "org/library") + assert entry["resolutions"]["a" * 40] == {"upstream_pr": 42, "reason": "adapted ABI"} + with pytest.raises(ValueError): + m.parse(_body(content.replace("adapted ABI", "''")), "toy", "org/library") + + +def test_unpaired_and_template() -> None: + entry = m.parse( + _body(" unpaired_reason: downstream-only compatibility"), "toy", "org/library" + ) + assert entry["unpaired_reason"] == "downstream-only compatibility" + assert m.parse(m.template("toy", "org/library"), "toy", "org/library")["upstream_prs"] == [123] + + +def _git(repo: Path, *arguments: str) -> str: + result = subprocess.run( + ["git", "-C", str(repo), *arguments], capture_output=True, text=True, check=True + ) + return result.stdout.strip() + + +@pytest.fixture +def source(tmp_path: Path) -> Path: + repo = tmp_path / "source" + repo.mkdir() + _git(repo, "init", "-q", "-b", "main") + _git(repo, "config", "user.name", "Test") + _git(repo, "config", "user.email", "test@example.invalid") + _git(repo, "config", "core.hooksPath", str(tmp_path / "no-hooks")) + return repo + + +def _commit(repo: Path, text: str, filename: str = "code.py") -> str: + (repo / filename).write_text(text) + _git(repo, "add", filename) + _git(repo, "commit", "-q", "-m", "test") + return _git(repo, "rev-parse", "HEAD") + + +def _pr(base: str, head: str) -> dict: + return {"base": {"sha": base, "ref": "main"}, "head": {"sha": head}, "state": "open"} + + +def test_match_multiple_upstream_prs_without_manual_map(source: Path) -> None: + base = _commit(source, "VALUE = 0\n") + first = _commit(source, "VALUE = 1\n") + second = _commit(source, "VALUE = 2\n") + assignments, evidence = m.match( + source, base, second, {41: _pr(base, first), 42: _pr(first, second)} + ) + assert assignments == {first: 41, second: 42} + assert all(item["method"] == "same-commit" for item in evidence.values()) + + +def test_match_rebased_patch(source: Path) -> None: + base = _commit(source, "VALUE = 0\n") + downstream = _commit(source, "VALUE = 1\n") + _git(source, "checkout", "-q", "-b", "upstream", base) + context = _commit(source, "context\n", "other.txt") + upstream = _commit(source, "VALUE = 1\n") + assignments, evidence = m.match(source, base, downstream, {42: _pr(context, upstream)}) + assert assignments == {downstream: 42} + assert evidence[downstream]["method"] == "same-edits" + + +@pytest.mark.parametrize("different", [" VALUE = 1\n", "VALUE = 1 \n", "VALUE = 1\r\n"]) +def test_matching_preserves_whitespace(source: Path, different: str) -> None: + base = _commit(source, "VALUE = 0\n") + downstream = _commit(source, "VALUE = 1\n") + _git(source, "checkout", "-q", "-b", "upstream", base) + upstream = _commit(source, different) + with pytest.raises(ValueError, match="Author action required"): + m.match(source, base, downstream, {42: _pr(base, upstream)}) + + +def test_identical_edits_in_different_functions_do_not_match(source: Path) -> None: + original = "def first():\n return False\n\n\ndef second():\n return False\n" + base = _commit(source, original) + downstream = _commit(source, original.replace("return False", "return True", 1)) + _git(source, "checkout", "-q", "-b", "upstream", base) + upstream = _commit( + source, + original.replace("def second():\n return False", "def second():\n return True"), + ) + with pytest.raises(ValueError, match="Author action required"): + m.match(source, base, downstream, {42: _pr(base, upstream)}) + + +def test_matching_squashed_series_and_ambiguous_duplicates(source: Path) -> None: + base = _commit(source, "VALUE = 0\n") + first = _commit(source, "VALUE = 1\n") + second = _commit(source, "VALUE = 2\n") + _git(source, "checkout", "-q", "-b", "upstream", base) + squash = _commit(source, "VALUE = 2\n") + assignments, evidence = m.match(source, base, second, {42: _pr(base, squash)}) + assert assignments == {first: 42, second: 42} + assert evidence[first]["method"] == "source-series-aggregate" + with pytest.raises(ValueError, match="#42, #43"): + m.match(source, base, second, {42: _pr(base, squash), 43: _pr(base, squash)}) + + +def test_author_can_resolve_adapted_commit(source: Path) -> None: + base = _commit(source, "VALUE = 0\n") + downstream = _commit(source, "VALUE = 1\n") + _git(source, "checkout", "-q", "-b", "upstream", base) + upstream = _commit(source, "VALUE = 2\n") + assignments, evidence = m.match( + source, + base, + downstream, + {42: _pr(base, upstream)}, + {downstream: {"upstream_pr": 42, "reason": "downstream ABI adaptation"}}, + ) + assert assignments == {downstream: 42} + assert evidence[downstream]["refresh_policy"] == "retain-until-verified" + with pytest.raises(ValueError, match="outside"): + m.match(source, base, downstream, {42: _pr(base, upstream)}, {base: {"upstream_pr": 42}}) + + +class FakeGitHub(p.GitHub): + def __init__(self) -> None: + super().__init__("org/consumer", "toy", "org/library") + self.pr = { + "number": 7, + "user": {"login": "author"}, + "state": "open", + "merged": False, + "head": {"sha": "a" * 40}, + "base": {"ref": "main"}, + "body": "", + "updated_at": "2099-01-01T00:00:00Z", + } + self.comments: list[dict] = [] + self.reviews: list[dict] = [] + self.writes: list[tuple[str, str, dict]] = [] + + def api( + self, endpoint: str, *, method: str = "GET", payload: dict | None = None + ) -> dict | list: + if method != "GET": + self.writes.append((endpoint, method, payload)) + if endpoint == "user": + return {"login": "operator"} + if endpoint.endswith("/pulls/7"): + return copy.deepcopy(self.pr) + if "/pulls?" in endpoint: + return [copy.deepcopy(self.pr)] + if "/files?" in endpoint: + return [{"filename": p._LOCK}] + if "/reviews/" in endpoint and endpoint.endswith("/dismissals"): + number = int(endpoint.split("/")[-2]) + self.reviews[number - 1]["state"] = "DISMISSED" + return {} + if "/reviews" in endpoint: + if method == "GET": + return copy.deepcopy(self.reviews) + self.reviews.append( + { + "id": len(self.reviews) + 1, + "user": {"login": "operator"}, + "body": payload["body"], + "state": "CHANGES_REQUESTED", + } + ) + return self.reviews[-1] + if "/comments/" in endpoint: + number = int(endpoint.rsplit("/", 1)[1]) + self.comments[number - 1]["body"] = payload["body"] + return {} + if "/comments" in endpoint: + if method == "GET": + return copy.deepcopy(self.comments) + self.comments.append( + { + "id": len(self.comments) + 1, + "user": {"login": "operator"}, + "body": payload["body"], + } + ) + return self.comments[-1] + raise AssertionError((endpoint, method, payload)) + + +@pytest.fixture +def monitor(tmp_path: Path): + args = argparse.Namespace( + canonical_repo="operator/library", + canonical_branch="dev", + fork="operator/consumer", + repo=tmp_path, + workdir=tmp_path, + publish=True, + since=None, + ) + store = b.Store(tmp_path / "state.sqlite") + gh = FakeGitHub() + instance = b.Monitor(args, gh, store) + yield instance + store.close() + + +def test_feedback_requests_changes_once_and_clears_only_metadata_review(monitor: b.Monitor) -> None: + gh = monitor.gh + monitor._feedback(gh.pr, "Missing marked block") + monitor._feedback(gh.pr, "Missing marked block") + assert len(gh.comments) == len(gh.reviews) == 1 + assert "PR description" in gh.comments[0]["body"] + assert "resolutions:" in gh.comments[0]["body"] + assert gh.writes[1][2]["event"] == "REQUEST_CHANGES" + monitor._feedback(gh.pr, None) + assert gh.reviews[0]["state"] == "DISMISSED" + assert all(payload.get("event") != "APPROVE" for _, _, payload in gh.writes) + + +def test_bot_does_not_override_operator_manual_review(monitor: b.Monitor) -> None: + monitor.gh.reviews.append( + { + "id": 1, + "user": {"login": "operator"}, + "body": "Manual code review", + "state": "CHANGES_REQUESTED", + } + ) + monitor._feedback(monitor.gh.pr, "Missing metadata") + monitor._feedback(monitor.gh.pr, None) + assert monitor.gh.reviews[0]["state"] == "CHANGES_REQUESTED" + assert len(monitor.gh.reviews) == 1 + + +def test_dry_run_and_self_review(monitor: b.Monitor) -> None: + monitor.args.publish = False + monitor._feedback(monitor.gh.pr, "Missing metadata") + assert not monitor.gh.writes + monitor.args.publish = True + monitor.gh.pr["user"]["login"] = "operator" + monitor._feedback(monitor.gh.pr, "Missing metadata") + assert monitor.gh.comments + assert not monitor.gh.reviews + + +def test_feedback_rechecks_mutable_description(monitor: b.Monitor) -> None: + stale = copy.deepcopy(monitor.gh.pr) + monitor.gh.pr["body"] = "updated while checking" + monitor._feedback(stale, "Missing metadata") + assert not monitor.gh.writes + + +def test_discovery_and_persistent_operator_scope(monitor: b.Monitor) -> None: + assert monitor._discover() == [7] + assert monitor._discover() == [7] + assert not monitor.gh.writes + changed = copy.copy(monitor.args) + changed.canonical_repo = "different/library" + with pytest.raises(ValueError, match="different operator"): + b.Monitor(changed, monitor.gh, monitor.store) + + +def _vendor(commit: str, url: str, branch: str): + return m.manage._validate_vendor( + "toy", + { + "url": url, + "branch": branch, + "commit": commit, + "source": "src", + "destination": "vendored/toy", + "include": ["**/*.py"], + "digest": "sha256-tree-v1:" + "0" * 64, + }, + ) + + +def test_monitor_reuses_matching_and_publishes_once_after_merge( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, +) -> None: + gh = monitor.gh + previous = _vendor("a" * 40, "https://github.com/operator/library.git", "dev") + reviewed = _vendor("b" * 40, "https://github.com/author/library.git", "temporary") + gh.pr["body"] = _body(" upstream_prs: [https://github.com/org/library/pull/42]") + monkeypatch.setattr(monitor, "_inputs", lambda pr: (previous, reviewed)) + matches, publications = [], [] + + def resolve(*args): + matches.append(True) + return {reviewed.commit: 42}, {reviewed.commit: {"method": "same-edits"}} + + def make_plan(args, github): + assert args.auto_merge and not args.wait + assert args.map_upstream == [f"{reviewed.commit}=42"] + return p.Promotion( + 7, + "c" * 40, + previous, + reviewed, + "operator/library", + "operator/consumer", + {}, + consumer_repo="org/consumer", + ) + + def publish(args, github, plan): + publications.append(plan) + assert plan.record["attribution"][reviewed.commit]["method"] == "same-edits" + return {"number": 8} + + original = gh.get + monkeypatch.setattr( + gh, + "get", + lambda endpoint: { + "number": 8, + "merged": True, + "merge_commit_sha": "d" * 40, + "html_url": "https://github.com/org/consumer/pull/8", + } + if endpoint.endswith("/pulls/8") + else original(endpoint), + ) + monkeypatch.setattr(m, "resolve", resolve) + monkeypatch.setattr(p, "_make_plan", make_plan) + monkeypatch.setattr(p, "_publish", publish) + monkeypatch.setattr(p, "_verify_remote_commit", lambda *args: "verified") + monitor.inspect(7) + monitor.inspect(7) + assert len(matches) == 1 + assert not publications + gh.pr.update(merged=True, state="closed") + monitor.inspect(7) + monitor.inspect(7) + assert len(publications) == 1 + assert monitor.store.get("complete:7").endswith("/8") + + +def test_monitor_reports_attribution_to_author_and_retries_after_correction( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, +) -> None: + previous = _vendor("a" * 40, "https://github.com/operator/library.git", "dev") + reviewed = _vendor("b" * 40, "https://github.com/author/library.git", "temporary") + monkeypatch.setattr(monitor, "_inputs", lambda pr: (previous, reviewed)) + monitor.gh.pr["body"] = _body(" upstream_prs: [https://github.com/org/library/pull/42]") + + def unresolved(*args): + raise ValueError("Author action required: no exact match for " + reviewed.commit) + + monkeypatch.setattr(m, "resolve", unresolved) + monitor.inspect(7) + assert "@author" in monitor.gh.comments[0]["body"] + assert reviewed.commit in monitor.gh.comments[0]["body"] + assert monitor.gh.reviews[0]["state"] == "CHANGES_REQUESTED" + monkeypatch.setattr(m, "resolve", lambda *args: ({reviewed.commit: 42}, {})) + monitor.inspect(7) + assert monitor.gh.reviews[0]["state"] == "DISMISSED" + + +def test_forbidden_dismissal_informs_author_without_approving( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monitor._feedback(monitor.gh.pr, "Missing block") + original = monitor.gh.api + + def forbidden(endpoint, **kwargs): + if endpoint.endswith("/dismissals"): + raise RuntimeError("HTTP 403") + return original(endpoint, **kwargs) + + monkeypatch.setattr(monitor.gh, "api", forbidden) + monitor._feedback(monitor.gh.pr, None) + assert "lacks dismissal permission" in monitor.gh.comments[0]["body"] + assert monitor.gh.reviews[0]["state"] == "CHANGES_REQUESTED" + writes = len(monitor.gh.writes) + monitor._feedback(monitor.gh.pr, None) + assert len(monitor.gh.writes) == writes + monkeypatch.setattr(monitor.gh, "api", original) + monitor._feedback(monitor.gh.pr, None) + assert "lacks dismissal permission" not in monitor.gh.comments[0]["body"] + assert monitor.gh.reviews[0]["state"] == "DISMISSED" + + +def test_old_promotion_messages_remain_readable() -> None: + record = { + "version": 1, + "vendor": "flashinfer-prims-ts", + "trtllm_pr": "https://github.com/NVIDIA/TensorRT-LLM/pull/17", + "trtllm_merge": "a" * 40, + } + message = ( + "----- BEGIN PRIMTS PROMOTION V1 -----\n" + + json.dumps(record) + + "\n----- END PRIMTS PROMOTION V1 -----" + ) + normalized = p._record_from_message(message) + assert normalized["consumer_pr"].endswith("/17") + assert normalized["upstream_repository"] == "flashinfer-ai/flashinfer" + + +def test_workdir_locks_and_refuses_unrelated_data(tmp_path: Path) -> None: + workdir = tmp_path / "state" + b._initialize_workdir(workdir) + with b._locked(workdir): + assert b._running(workdir) + with pytest.raises(ValueError, match="already owns"): + with b._locked(workdir): + pass + assert not b._running(workdir) + unrelated = tmp_path / "unrelated" + unrelated.mkdir() + (unrelated / "data").write_text("preserve") + with pytest.raises(ValueError, match="empty workdir"): + b._initialize_workdir(unrelated) + assert (unrelated / "data").read_text() == "preserve" + + +def test_workdir_refuses_checkout(source: Path) -> None: + with pytest.raises(ValueError, match="outside a Git worktree"): + b._initialize_workdir(source / "bot-state") + + +def test_daemon_launch_status_stop_without_systemd(tmp_path: Path) -> None: + binary = tmp_path / "bin" + binary.mkdir() + gh = binary / "gh" + gh.write_text( + "#!/usr/bin/env python3\nimport json, sys\n" + "print(json.dumps({'login': 'operator'} if sys.argv[-1] == 'user' else []))\n" + ) + gh.chmod(0o700) + workdir = tmp_path / "state" + env = { + **os.environ, + "PATH": f"{binary}:{os.environ['PATH']}", + "GH_CONFIG_DIR": str(tmp_path / "private-auth"), + } + command = [sys.executable, str(_ROOT / "scripts/vendor/bot.py")] + arguments = [ + "run", + "--workdir", + str(workdir), + "--vendor", + "toy", + "--upstream-repo", + "org/library", + "--canonical-repo", + "operator/library", + "--canonical-branch", + "dev", + "--fork", + "operator/consumer", + "--daemon", + ] + try: + result = subprocess.run( + command + arguments, env=env, capture_output=True, text=True, timeout=15 + ) + assert result.returncode == 0, result.stderr + assert "Started daemon PID" in result.stdout + status = subprocess.run( + command + ["status", "--workdir", str(workdir)], + env=env, + capture_output=True, + text=True, + check=True, + ) + assert json.loads(status.stdout)["running"] + assert "private-auth" not in (workdir / "bot.log").read_text() + finally: + subprocess.run( + command + ["stop", "--workdir", str(workdir)], env=env, capture_output=True, timeout=5 + ) + deadline = time.monotonic() + 10 + while b._running(workdir) and time.monotonic() < deadline: + time.sleep(0.1) + assert not b._running(workdir) diff --git a/tests/unittest/others/test_vendor_sources.py b/tests/unittest/others/test_vendor_sources.py index 3872728cd895..87346a9bb07e 100644 --- a/tests/unittest/others/test_vendor_sources.py +++ b/tests/unittest/others/test_vendor_sources.py @@ -34,7 +34,7 @@ _REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent -_VENDOR_SOURCES = _REPO_ROOT / "scripts" / "vendor_sources.py" +_VENDOR_SOURCES = _REPO_ROOT / "scripts" / "vendor" / "manage.py" _VENDOR_NAME = "example" _SOURCE = "python/example" _DESTINATION = "src/example" From 66b0a3f8f173020238eba766ee5042ee9b560b2e Mon Sep 17 00:00:00 2001 From: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> Date: Wed, 23 Sep 2026 08:56:16 +0000 Subject: [PATCH 03/11] [None][chore] remove legacy vendor tool entry points Signed-off-by: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> --- .github/CODEOWNERS | 1 - 3rdparty/vendor-sources.md | 6 ++--- AGENTS.md | 10 ++++---- scripts/maintain_prims_ts.py | 25 ------------------- scripts/vendor/README.md | 9 +++---- scripts/vendor/bot.py | 1 - scripts/vendor/promote.py | 10 ++------ scripts/vendor_sources.py | 12 --------- .../unittest/others/test_maintain_prims_ts.py | 7 +++--- 9 files changed, 17 insertions(+), 64 deletions(-) delete mode 100644 scripts/maintain_prims_ts.py delete mode 100644 scripts/vendor_sources.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a72a658cf868..bab1ef5b3a7f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -475,7 +475,6 @@ /requirements-grpc-smg.txt @NVIDIA/trt-llm-oss-compliance /requirements-openengine.txt @NVIDIA/trt-llm-oss-compliance /requirements.txt @NVIDIA/trt-llm-oss-compliance -/scripts/vendor_sources.py @NVIDIA/trt-llm-infra-devs @NVIDIA/trt-llm-oss-compliance /scripts/vendor/ @NVIDIA/trt-llm-infra-devs @NVIDIA/trt-llm-oss-compliance /setup.py @NVIDIA/trt-llm-oss-compliance /tensorrt_llm/_torch/attention/backends/prims_ts/** @NVIDIA/trt-llm-torch-attention-devs @NVIDIA/trt-llm-oss-compliance diff --git a/3rdparty/vendor-sources.md b/3rdparty/vendor-sources.md index 9fccaa06fee7..d80ebfeb8f52 100644 --- a/3rdparty/vendor-sources.md +++ b/3rdparty/vendor-sources.md @@ -302,9 +302,9 @@ authentication, configurable private workdirs, daemon launch/stop, and recovery. No GitHub App or systemd setup is required. All remote writes are opt-in with `--publish`; operator settings and credentials are not committed to this repo. -The old `scripts/vendor_sources.py` and `scripts/maintain_prims_ts.py` paths are -compatibility entry points. Existing PrimTS promotions can still be resumed -through their original command and branch naming. +Use the entry points under `scripts/vendor/` directly. The old top-level scripts +and PrimTS-specific CLI aliases are not retained; callers must migrate to the +generic commands and supply their vendor and repository configuration explicitly. ## License and attribution diff --git a/AGENTS.md b/AGENTS.md index 7a25d28964d3..e9bc5eb08206 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -203,11 +203,11 @@ Manual publication uses `--publish --auto-merge`; dry-run is the default. `--daemon`, and opt-in `--publish`. Do not launch a publishing daemon unless the user requested ongoing automation. Never commit operator credentials or runtime state. The bot asks the PR author to correct metadata or ambiguous attribution; -it never approves code merely because metadata passes. The old script paths -remain compatibility entry points. See `scripts/vendor/README.md` and -`3rdparty/vendor-sources.md`. Preserve structured promotion provenance and DCO -sign-off in the final squash message. CI skip applies only to verified lock-only -promotions, not changes to these tools. +it never approves code merely because metadata passes. Use the entry points +under `scripts/vendor/`; no legacy script wrappers are provided. See +`scripts/vendor/README.md` and `3rdparty/vendor-sources.md`. Preserve structured +promotion provenance and DCO sign-off in the final squash message. CI skip applies +only to verified lock-only promotions, not changes to these tools. ### Triggering CI diff --git a/scripts/maintain_prims_ts.py b/scripts/maintain_prims_ts.py deleted file mode 100644 index 15ee285fbf38..000000000000 --- a/scripts/maintain_prims_ts.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Compatibility entry point for the original PrimTS promotion command.""" - -import sys - -if __package__: - from .vendor.promote import main -else: - from vendor.promote import main - -if __name__ == "__main__": - arguments = sys.argv[1:] - if arguments and arguments[0] == "promote": - arguments[1:1] = [ - "--vendor", - "flashinfer-prims-ts", - "--upstream-repo", - "flashinfer-ai/flashinfer", - "--canonical-branch", - "trtllm-prims-ts-dev", - "--legacy-prims-ts", - ] - raise SystemExit(main(arguments)) diff --git a/scripts/vendor/README.md b/scripts/vendor/README.md index 16911a6422c0..680491f8eb43 100644 --- a/scripts/vendor/README.md +++ b/scripts/vendor/README.md @@ -17,11 +17,10 @@ repositories and canonical **branches**, not tag promotion or periodic rebases. | `python scripts/vendor/promote.py promote` | Promote an immutable, merged source update and prepare a lock-only PR | | `python scripts/vendor/bot.py run` | Monitor source-update PRs, validate metadata, resolve attribution, and promote | -`scripts/vendor_sources.py` remains a compatibility entry point. The old -`scripts/maintain_prims_ts.py` command supplies the original PrimTS defaults and -branch naming for historical operation retries. New invocations should use the -generic entry points. The separate legacy Triton packager does not use this lock -and is not migrated by this change. +Use these entry points directly; no backward compatibility is promised for the +old top-level scripts or PrimTS-specific CLI aliases. Supply vendor and repository +configuration explicitly. The separate legacy Triton packager does not use this +lock and is not migrated by this change. ## Workflow diff --git a/scripts/vendor/bot.py b/scripts/vendor/bot.py index f69dbebd2712..835ad7a87ae8 100644 --- a/scripts/vendor/bot.py +++ b/scripts/vendor/bot.py @@ -343,7 +343,6 @@ def inspect(self, number: int) -> None: auto_merge=True, wait=False, timeout=3600, - legacy_prims_ts=False, ) plan = promote._make_plan(arguments, self.gh) # Attribution evidence is versioned with the durable promotion record. diff --git a/scripts/vendor/promote.py b/scripts/vendor/promote.py index cac87271bb6d..a20a7bfc335a 100644 --- a/scripts/vendor/promote.py +++ b/scripts/vendor/promote.py @@ -283,13 +283,11 @@ class Promotion: record: dict consumer_repo: str = "NVIDIA/TensorRT-LLM" base_branch: str = "main" - legacy_prims_ts: bool = False completed_pr: dict | None = None @property def branch(self) -> str: - name = "prims-ts" if self.legacy_prims_ts else self.previous.name - return f"chore/{name}-promote-{self.number}-{self.reviewed.commit[:12]}" + return f"chore/{self.previous.name}-promote-{self.number}-{self.reviewed.commit[:12]}" @property def title(self) -> str: @@ -382,7 +380,6 @@ def _make_plan( record, consumer_repo=gh.consumer_repo, base_branch=gh.base_branch, - legacy_prims_ts=args.legacy_prims_ts, ) # A completed operation is historical: later promotions, deleted temporary # forks, or changed upstream PR state must not make it run again or fail. @@ -974,13 +971,12 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest="command", required=True) promote = commands.add_parser("promote", help="Promote a merged consumer source update.") - promote.add_argument("--source-pr", "--trtllm-pr", dest="source_pr", required=True) + promote.add_argument("--source-pr", required=True) promote.add_argument("--vendor", required=True, help="Vendor key in the lock file.") promote.add_argument("--consumer-repo", default="NVIDIA/TensorRT-LLM") promote.add_argument("--base-branch", default="main") promote.add_argument("--upstream-repo", required=True, help="Upstream OWNER/REPO.") promote.add_argument("--upstream-branch", default="main") - promote.add_argument("--legacy-prims-ts", action="store_true", help=argparse.SUPPRESS) promote.add_argument( "--canonical-repo", required=True, @@ -1014,8 +1010,6 @@ def main(argv: list[str] | None = None) -> int: ) promote.add_argument( "--source-repo", - "--flashinfer-repo", - dest="source_repo", type=Path, help="Local source repository containing both immutable SHAs.", ) diff --git a/scripts/vendor_sources.py b/scripts/vendor_sources.py deleted file mode 100644 index 8e3f9350ed49..000000000000 --- a/scripts/vendor_sources.py +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Compatibility entry point; use scripts/vendor/manage.py for new invocations.""" - -if __package__: - from .vendor.manage import main -else: - from vendor.manage import main - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/unittest/others/test_maintain_prims_ts.py b/tests/unittest/others/test_maintain_prims_ts.py index 38b363ced110..38097727218c 100644 --- a/tests/unittest/others/test_maintain_prims_ts.py +++ b/tests/unittest/others/test_maintain_prims_ts.py @@ -150,7 +150,6 @@ def __init__(self, root: Path) -> None: def args(self, **overrides: object) -> argparse.Namespace: values = dict( source_pr="17", - legacy_prims_ts=False, canonical_repo="maintainer/flashinfer", canonical_branch="trtllm-prims-ts-dev", upstream_pr=["4829"], @@ -349,7 +348,7 @@ def test_dry_run_has_no_writes_and_pins_merged_not_live_head( _UPSTREAM, "--canonical-branch", "trtllm-prims-ts-dev", - "--trtllm-pr", + "--source-pr", "17", "--canonical-repo", "maintainer/flashinfer", @@ -670,7 +669,7 @@ def test_cli_invalid_flags(flags: list[str]) -> None: _UPSTREAM, "--canonical-branch", "trtllm-prims-ts-dev", - "--trtllm-pr", + "--source-pr", "17", "--canonical-repo", "maintainer/flashinfer", @@ -873,7 +872,7 @@ def test_unpaired_cli_dry_run(world: World, capsys: pytest.CaptureFixture) -> No _UPSTREAM, "--canonical-branch", "trtllm-prims-ts-dev", - "--trtllm-pr", + "--source-pr", "17", "--canonical-repo", "maintainer/flashinfer", From 974209ee5bb4433da57481c03f0068e63981207c Mon Sep 17 00:00:00 2001 From: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> Date: Wed, 23 Sep 2026 09:26:43 +0000 Subject: [PATCH 04/11] [None][fix] recover vendor promotions without SQLite state Signed-off-by: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> --- AGENTS.md | 3 + scripts/vendor/README.md | 61 +++- scripts/vendor/bot.py | 143 ++++++---- scripts/vendor/promote.py | 143 ++++++++-- .../unittest/others/test_maintain_prims_ts.py | 263 +++++++++++++++++- tests/unittest/others/test_vendor_bot.py | 81 +++++- 6 files changed, 604 insertions(+), 90 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e9bc5eb08206..4f95a397c011 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -208,6 +208,9 @@ under `scripts/vendor/`; no legacy script wrappers are provided. See `scripts/vendor/README.md` and `3rdparty/vendor-sources.md`. Preserve structured promotion provenance and DCO sign-off in the final squash message. CI skip applies only to verified lock-only promotions, not changes to these tools. +Git/GitHub promotion commits are authoritative; SQLite is a disposable cache. +After cache loss, stop the old process and relaunch with the same operator/config. +Cold discovery backfills merged PRs; `--since` explicitly limits that history. ### Triggering CI diff --git a/scripts/vendor/README.md b/scripts/vendor/README.md index 680491f8eb43..b9178a510209 100644 --- a/scripts/vendor/README.md +++ b/scripts/vendor/README.md @@ -31,11 +31,16 @@ flowchart TD match -->|missing or ambiguous| feedback[Request changes with author-facing instructions] feedback -->|author corrects block, pin, or targeted resolution| metadata match -->|valid and source PR merged| verify[Verify ancestry, current lock, and materialization] - verify --> ff[Fast-forward configured canonical source branch] + verify --> durable[Commit provenance and publish the promotion branch] + durable --> ff[Fast-forward configured canonical source branch] ff --> promotion[Create or reuse URL/branch-only promotion PR] promotion --> ci[Request CI skip and enable squash auto-merge] ci -->|required checks and owner reviews| merged[Verify final lock and squash provenance] merged --> rebase[Other source-update authors rebase onto the canonical pin] + restart[Restart after local cache loss] --> recover[Rediscover PRs and verify Git provenance] + recover -->|pending recorded operation| verify + recover -->|no record and pin still pending| metadata + recover -->|completed promotion| merged ``` Promotion preserves the source SHA, selected files, compatibility patch, and @@ -105,10 +110,10 @@ When there is no upstream PR for the update, replace `upstream_prs` with a nonempty `unpaired_reason`. Do not combine those alternatives. Unpaired changes are retained during future refresh until equivalence is established separately. -The source head and normalized metadata are rechecked before feedback and -publication. Metadata corrections rerun validation. Historical promotion -records remain authoritative; editing a merged description does not rewrite an -already completed promotion's provenance. +Before a new operation is committed, the source head and normalized metadata +are rechecked before feedback and publication. Metadata corrections rerun +validation. Durable promotion records remain authoritative; later description +edits or upstream PR changes do not rewrite a recorded operation's provenance. ## Launch the local monitor @@ -146,19 +151,49 @@ The daemon survives terminal closure but does not automatically restart after a crash or reboot; a supervisor is optional. Stop before changing launch settings. The workdir must be empty on first use, owned by the current user, and outside a -Git worktree. It holds private SQLite state, a bare source cache, process-lock +Git worktree. It holds a disposable SQLite cache, a bare source cache, process-lock and stop files, and isolated promotion worktrees. Runtime state is not committed. An existing workdir is bound to its operator and vendor configuration; use a different directory for another operator/vendor. Authentication is inherited from the launching environment and is not persisted in that state. -The default interval is 120 seconds (`--interval`, minimum 30). First launch -discovers existing open PRs and merges occurring from startup onward. To include -earlier merges, set `--since 2026-01-01T00:00:00Z` on first launch. The saved -cutoff and tracked PRs survive restarts; retries discover interrupted pushes, -PR creation, CI requests, and auto-merge without duplicating them. Old historical -operations should be resumed with their original CLI/settings, not backfilled -indiscriminately into a newly configured monitor. +The default interval is 120 seconds (`--interval`, minimum 30). A cold start +discovers existing open PRs and backfills merged PRs from all history. This scan +can be expensive; subsequent polls use a cached cursor and tracked candidates. +An explicit `--since 2026-01-01T00:00:00Z` restricts historical discovery; use the +same cutoff on every launch, including recovery. Historical updates belonging +to another canonical branch, or superseded pins without a recoverable promotion, +are not promoted or sent author feedback. + +### Recovery after cache loss + +Git/GitHub are the source of truth; SQLite is not required to reconstruct a +promotion. Stop the old process before replacing or discarding its database. +Relaunch with the **same account, repositories, canonical branch, and cutoff**. +The bot accepts an existing workdir containing only recognized bot artifacts +even if SQLite is missing. It preserves those artifacts and refuses unrelated +files and symlinks. A new empty workdir also works; existing registered Git +worktrees are validated and reused, or recreated when their directories are gone. + +Recovery checks the deterministic promotion PR/branch and its signed-off commit +before parsing the source PR description or consulting upstream PR heads. It +verifies the operation identity, lock-only diff, and provenance, then resumes +the recorded operation. Completed promotions are verified against their squash +commit without requiring the old source fork or mutable PR metadata. Existing +feedback, reviews, CI-skip comments, and auto-merge settings are discovered from +GitHub rather than duplicated from missing local IDs. + +Before the first canonical-branch push, the signed promotion commit is pushed to +the configured publishing fork. Therefore loss after any remote publication +step retains a durable provenance record. A locally committed but not yet pushed +operation can also be recovered from the consumer repository's Git branch. +Before any provenance commit exists, attribution is recomputed from current +remote inputs; a previous successful match in SQLite is never authoritative. + +Recovery assumes a single publishing operator and accessible, unmodified Git/GitHub +artifacts. It is not recovery from deleted remote history or a replacement +publishing account/fork. Tampered commits, unexpected branch moves, closed-unmerged +promotion PRs, and unrelated worktree edits still stop the operation for inspection. One process monitors one vendor; separate workdirs can monitor different vendors. `--consumer-repo`, `--base-branch`, and `--upstream-branch` also support consumers diff --git a/scripts/vendor/bot.py b/scripts/vendor/bot.py index 835ad7a87ae8..e01ac4ad96d1 100644 --- a/scripts/vendor/bot.py +++ b/scripts/vendor/bot.py @@ -12,6 +12,7 @@ import json import logging import os +import re import signal import sqlite3 import subprocess @@ -34,7 +35,7 @@ class Store: - """Transactional runtime state; no authentication material is persisted.""" + """Disposable reconciliation cache; Git/GitHub own promotion provenance and state.""" def __init__(self, path: Path) -> None: self.connection = sqlite3.connect(path) @@ -100,8 +101,7 @@ def __init__(self, args: argparse.Namespace, gh: promote.GitHub, store: Store) - if store.get("since") is None: store.put( "since", - args.since - or datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + args.since or "1970-01-01T00:00:00Z", ) def _discover(self) -> list[int]: @@ -284,11 +284,52 @@ def _inputs(self, pr: dict) -> tuple[manage.Vendor, manage.Vendor] | None: if target_is_canonical: return None # Periodic refresh/direct canonical update: separate workflow. if not canonical: + if pr.get("merged"): + return None # Historical update belonging to another canonical branch. raise ValueError( "Finish the preceding vendor promotion, then rebase this source-update PR onto the canonical pin." ) return previous, reviewed + def _arguments(self, number: int, reviewed: manage.Vendor) -> argparse.Namespace: + return argparse.Namespace( + source_pr=str(number), + canonical_repo=self.args.canonical_repo, + canonical_branch=self.args.canonical_branch, + upstream_pr=[], + unpaired_reason=None, + map_upstream=[], + repo=self.args.repo, + fork=self.args.fork, + source_repo=None, + worktree=self.args.workdir / f"promotion-{number}-{reviewed.commit[:12]}", + publish=self.args.publish, + auto_merge=True, + wait=False, + timeout=3600, + ) + + def _publish(self, arguments: argparse.Namespace, plan: promote.Promotion) -> None: + number = plan.number + if plan.completed_pr is not None: + self.store.put(f"complete:{number}", plan.completed_pr["html_url"]) + _LOG.info( + "PR #%s: verified completed promotion %s", number, plan.completed_pr["html_url"] + ) + return + if not self.args.publish: + _LOG.info("PR #%s: promotion ready (dry run): %s", number, plan.branch) + return + followup = promote._publish(arguments, self.gh, plan) + self.store.put(f"promotion:{number}", followup["number"]) + latest = self.gh.get(f"{self.prefix}/pulls/{followup['number']}") + if latest.get("merged"): + promote._verify_remote_commit( + self.gh, plan, self.gh.consumer_repo, latest["merge_commit_sha"] + ) + self.store.put(f"complete:{number}", latest["html_url"]) + _LOG.info("PR #%s: promotion %s", number, latest["html_url"]) + def inspect(self, number: int) -> None: """Reconcile one source-update PR and its deterministic promotion operation.""" if self.store.get(f"complete:{number}"): @@ -301,21 +342,27 @@ def inspect(self, number: int) -> None: if inputs is None: return previous, reviewed = inputs + except (ValueError, manage.VendorError) as error: + self._feedback(pr, str(error)) + self.store.put(f"problem:{number}", str(error)) + return + arguments = self._arguments(number, reviewed) + if pr.get("merged"): + plan = promote._plan_identity(arguments, self.gh) + if promote._recover_plan(arguments, self.gh, plan): + self._publish(arguments, plan) + return + current = promote._lock_at(self.gh, self.gh.consumer_repo, plan.main_sha) + if current.to_mapping() != reviewed.to_mapping(): + _LOG.info("PR #%s: historical pin is no longer pending promotion", number) + return + promote._check_current(self.gh, plan) + try: entry = metadata.parse(pr.get("body") or "", self.gh.vendor_name, self.gh.upstream_repo) - signature = metadata.fingerprint( - { - "previous": previous.to_mapping(), - "reviewed": reviewed.to_mapping(), - "metadata": entry, - } - ) - key = f"matching:{number}:{signature}" - saved = self.store.get(key) cache = self.args.workdir / "sources.git" - if saved is None: - assignments, evidence = metadata.resolve(self.gh, previous, reviewed, entry, cache) - saved = {"assignments": assignments, "evidence": evidence} - self.store.put(key, saved) + # Until provenance is committed, decisions follow current remote + # inputs, not whichever upstream heads a previous process cached. + assignments, evidence = metadata.resolve(self.gh, previous, reviewed, entry, cache) except (ValueError, manage.VendorError) as error: self._feedback(pr, str(error)) self.store.put(f"problem:{number}", str(error)) @@ -328,44 +375,21 @@ def inspect(self, number: int) -> None: current = self.gh.get(f"{self.prefix}/pulls/{number}") if current.get("body") != pr.get("body") or current["head"]["sha"] != pr["head"]["sha"]: return - arguments = argparse.Namespace( - source_pr=str(number), - canonical_repo=self.args.canonical_repo, - canonical_branch=self.args.canonical_branch, - upstream_pr=[str(item) for item in entry["upstream_prs"]], - unpaired_reason=entry["unpaired_reason"], - map_upstream=[f"{sha}={target}" for sha, target in saved["assignments"].items()], - repo=self.args.repo, - fork=self.args.fork, - source_repo=cache if not entry["unpaired_reason"] and cache.exists() else None, - worktree=self.args.workdir / f"promotion-{number}-{reviewed.commit[:12]}", - publish=self.args.publish, - auto_merge=True, - wait=False, - timeout=3600, - ) + arguments.upstream_pr = [str(item) for item in entry["upstream_prs"]] + arguments.unpaired_reason = entry["unpaired_reason"] + arguments.map_upstream = [f"{sha}={target}" for sha, target in assignments.items()] + arguments.source_repo = cache if cache.exists() else None plan = promote._make_plan(arguments, self.gh) # Attribution evidence is versioned with the durable promotion record. # Reuse the original published record on retries rather than replacing it. if plan.completed_pr is None: - plan.record["attribution"] = saved["evidence"] + plan.record["attribution"] = evidence plan.record["contributor_metadata"] = { "source_head": pr["head"]["sha"], "metadata_digest": metadata.fingerprint(entry), "pr_author": pr["user"]["login"], } - if not self.args.publish: - _LOG.info("PR #%s: promotion ready (dry run): %s", number, plan.branch) - return - followup = promote._publish(arguments, self.gh, plan) - self.store.put(f"promotion:{number}", followup["number"]) - latest = self.gh.get(f"{self.prefix}/pulls/{followup['number']}") - if latest.get("merged"): - promote._verify_remote_commit( - self.gh, plan, self.gh.consumer_repo, latest["merge_commit_sha"] - ) - self.store.put(f"complete:{number}", latest["html_url"]) - _LOG.info("PR #%s: promotion %s", number, latest["html_url"]) + self._publish(arguments, plan) def poll(self) -> None: """Process candidates serially; preserve failures for safe later retries.""" @@ -387,8 +411,6 @@ def poll(self) -> None: def _initialize_workdir(path: Path) -> None: - if path.exists() and any(path.iterdir()) and not (path / _STATE_FILE).exists(): - raise ValueError("Choose an empty workdir or an existing vendor-bot workdir.") path.mkdir(parents=True, exist_ok=True, mode=0o700) if path.stat().st_uid != os.getuid(): raise ValueError("Workdir must be owned by the current user.") @@ -401,6 +423,28 @@ def _initialize_workdir(path: Path) -> None: raise ValueError( "Bot workdir must be outside a Git worktree to keep runtime state private." ) + files = { + _STATE_FILE, + "state.sqlite-wal", + "state.sqlite-shm", + "state.sqlite-journal", + "bot.log", + "bot.lock", + "stop", + } + for entry in path.iterdir(): + directory = entry.name == "sources.git" or re.fullmatch( + r"promotion-[0-9]+-[0-9a-f]{12}", entry.name + ) + if ( + entry.is_symlink() + or entry.stat().st_uid != os.getuid() + or not (entry.is_dir() if directory else entry.name in files and entry.is_file()) + ): + raise ValueError( + "Choose an empty workdir or an existing vendor-bot workdir; unexpected entry: " + + entry.name + ) path.chmod(0o700) @@ -489,7 +533,10 @@ def _parser() -> argparse.ArgumentParser: ) run.add_argument( "--since", - help="First-run merged-PR cutoff, UTC YYYY-MM-DDTHH:MM:SSZ; default startup time.", + help=( + "Optional historical discovery cutoff, UTC YYYY-MM-DDTHH:MM:SSZ; " + "default all history. Reuse the same cutoff after cache loss." + ), ) run.add_argument( "--publish", diff --git a/scripts/vendor/promote.py b/scripts/vendor/promote.py index a20a7bfc335a..1f9b1c8e920c 100644 --- a/scripts/vendor/promote.py +++ b/scripts/vendor/promote.py @@ -49,7 +49,7 @@ def _run( strip: bool = True, binary_output: bool = False, ) -> str: - # Do not use vendor_sources._run_git here: its synthetic identity is for + # Do not use manage._run_git here: its synthetic identity is for # patch generation. Real promotion commits must use the maintainer's identity. env = os.environ.copy() for key in vendor._GIT_LOCAL_ENVIRONMENT_VARIABLES: @@ -313,21 +313,8 @@ def _check_current(gh: GitHub, plan: Promotion) -> None: ) -def _make_plan( - args: argparse.Namespace, - gh: GitHub, - *, - auto_match: bool = False, - resolutions: dict[str, dict] | None = None, -) -> Promotion: - if args.unpaired_reason is not None: - if not args.unpaired_reason.strip() or args.upstream_pr or args.map_upstream: - raise ValueError( - "--unpaired-reason requires a nonempty reason and cannot be combined " - "with --upstream-pr or --map-upstream." - ) - elif not args.upstream_pr: - raise ValueError("Provide --upstream-pr or an explicit --unpaired-reason.") +def _plan_identity(args: argparse.Namespace, gh: GitHub) -> Promotion: + """Identify an operation from immutable consumer history, without live attribution.""" number = _pr_number(args.source_pr, gh.consumer_repo) pr = gh.get(f"repos/{gh.consumer_repo}/pulls/{number}") if not pr.get("merged") or pr["base"]["ref"] != gh.base_branch: @@ -370,7 +357,7 @@ def _make_plan( "source_url": reviewed.url, "source_branch": reviewed.branch, } - plan = Promotion( + return Promotion( number, _main_sha(gh), previous, @@ -381,6 +368,27 @@ def _make_plan( consumer_repo=gh.consumer_repo, base_branch=gh.base_branch, ) + + +def _make_plan( + args: argparse.Namespace, + gh: GitHub, + *, + auto_match: bool = False, + resolutions: dict[str, dict] | None = None, +) -> Promotion: + if args.unpaired_reason is not None: + if not args.unpaired_reason.strip() or args.upstream_pr or args.map_upstream: + raise ValueError( + "--unpaired-reason requires a nonempty reason and cannot be combined " + "with --upstream-pr or --map-upstream." + ) + elif not args.upstream_pr: + raise ValueError("Provide --upstream-pr or an explicit --unpaired-reason.") + plan = _plan_identity(args, gh) + previous, reviewed, record = plan.previous, plan.reviewed, plan.record + canonical_repo, fork = plan.canonical_repo, plan.fork + source_repo = _repo_name(reviewed.url) # A completed operation is historical: later promotions, deleted temporary # forks, or changed upstream PR state must not make it run again or fail. existing = _find_followup(gh, plan) @@ -491,6 +499,68 @@ def _find_followup(gh: GitHub, plan: Promotion) -> dict | None: return prs[0] if prs else None +def _local_branch(repo: Path, branch: str) -> str | None: + reference = f"refs/heads/{branch}" + for line in _git( + repo, "for-each-ref", "--format=%(refname) %(objectname)", reference + ).splitlines(): + name, sha = line.split() + if name == reference: + return _sha(sha) + return None + + +def _recover_plan(args: argparse.Namespace, gh: GitHub, plan: Promotion) -> bool: + """Restore verified provenance from Git/GitHub before consulting mutable metadata.""" + existing = _find_followup(gh, plan) + if existing: + existing = gh.get(f"repos/{gh.consumer_repo}/pulls/{existing['number']}") + sha = existing["merge_commit_sha"] if existing.get("merged") else existing["head"]["sha"] + _verify_remote_commit(gh, plan, gh.consumer_repo, sha) + if existing.get("merged"): + plan.completed_pr = existing + return True + else: + remote = gh.optional(f"repos/{plan.fork}/git/ref/heads/{plan.branch}") + if remote: + _verify_remote_commit(gh, plan, plan.fork, remote["object"]["sha"]) + else: + sha = _local_branch(args.repo, plan.branch) + if sha is None: + return False + _git( + args.repo, + "fetch", + "--quiet", + "--no-tags", + f"https://github.com/{plan.consumer_repo}.git", + plan.main_sha, + ) + if _git(args.repo, "merge-base", sha, plan.main_sha) == sha: + return False + parents = _git(args.repo, "show", "-s", "--format=%P", sha).split() + if ( + len(parents) != 1 + or _git(args.repo, "diff", "--name-only", parents[0], sha) != _LOCK + ): + raise ValueError("Local promotion commit must change only the vendor lock.") + _git(args.repo, "merge-base", "--is-ancestor", parents[0], plan.main_sha) + _check_document_change( + _local_lock_document(args.repo, parents[0]), + _local_lock_document(args.repo, sha), + plan, + ) + message = _git(args.repo, "show", "-s", "--format=%B", sha) + _adopt_record(plan, message) + if not re.search(r"^Signed-off-by: .+ <.+>$", message, re.MULTILINE): + raise ValueError("Local promotion commit has no DCO sign-off.") + _check_source_repo(gh, plan.canonical_repo) + _check_source_repo(gh, _repo_name(plan.reviewed.url)) + _check_fork(gh, plan.fork, gh.consumer_repo) + _check_current(gh, plan) + return True + + def _adopt_record(plan: Promotion, message: str) -> None: """Keep the original upstream snapshot when resuming an already-published run.""" existing = _record_from_message(message) @@ -499,7 +569,7 @@ def _adopt_record(plan: Promotion, message: str) -> None: raise ValueError("Existing promotion metadata does not describe this source update.") _validate_snapshot(existing, plan.record["upstream_repository"]) if "changes" not in plan.record: - # Only the read-only, already-merged path restores an entire snapshot. + # A verified operation identity can restore its entire durable snapshot. plan.record = existing return expected = { @@ -628,6 +698,19 @@ def _prepare_commit(args: argparse.Namespace, plan: Promotion, source: Path) -> ) worktree = args.worktree or args.repo.parent / plan.branch.removeprefix("chore/") worktree = worktree.resolve() + branch_exists = _local_branch(args.repo, plan.branch) is not None + missing_worktree = False + if branch_exists and not worktree.exists(): + for block in _git(args.repo, "worktree", "list", "--porcelain", "-z").split("\0\0"): + fields = dict(line.split(" ", 1) for line in block.split("\0") if " " in line) + if fields.get("branch") != f"refs/heads/{plan.branch}": + continue + registered = Path(fields["worktree"]) + if registered.exists(): + worktree = registered + else: + missing_worktree = True + break if worktree.exists(): if _git(worktree, "branch", "--show-current") != plan.branch or _git( worktree, "rev-parse", "--path-format=absolute", "--git-common-dir" @@ -653,6 +736,12 @@ def _prepare_commit(args: argparse.Namespace, plan: Promotion, source: Path) -> print(f"Resuming incomplete promotion in {worktree}...", flush=True) else: print(f"Creating promotion worktree {worktree} (LFS downloads disabled)...", flush=True) + if branch_exists: + # Force is restricted to a registration whose directory is gone; + # existing worktrees are reused and validated above, never replaced. + flags = ("--force",) if missing_worktree else () + _git(args.repo, "worktree", "add", *flags, str(worktree), plan.branch) + return _prepare_commit(args, plan, source) _git(args.repo, "worktree", "add", "-b", plan.branch, str(worktree), plan.main_sha) before = _local_lock_document(worktree, "HEAD") if before["vendors"][plan.previous.name] != plan.reviewed.to_mapping(): @@ -907,6 +996,16 @@ def _publish(args: argparse.Namespace, gh: GitHub, plan: Promotion) -> dict: worktree, sha = _prepare_commit(args, plan, source) message = _git(worktree, "log", "-1", "--format=%B") _check_current(gh, plan) + # Persist provenance remotely before changing the canonical source branch. + if remote is None: + print(f"Publishing follow-up branch {plan.fork}:{plan.branch}...", flush=True) + _git( + worktree, + "push", + f"https://github.com/{plan.fork}.git", + f"{sha}:refs/heads/{plan.branch}", + ) + _check_current(gh, plan) # No force push: Git also enforces the fast-forward constraint remotely. print(f"Ensuring canonical branch points to {plan.reviewed.commit}...", flush=True) _git( @@ -916,14 +1015,6 @@ def _publish(args: argparse.Namespace, gh: GitHub, plan: Promotion) -> dict: f"{plan.reviewed.commit}:refs/heads/{plan.previous.branch}", ) print(f"Canonical branch now contains reviewed revision {plan.reviewed.commit}.") - if remote is None: - print(f"Publishing follow-up branch {plan.fork}:{plan.branch}...", flush=True) - _git( - worktree, - "push", - f"https://github.com/{plan.fork}.git", - f"{sha}:refs/heads/{plan.branch}", - ) _check_current(gh, plan) if existing is None: print("Creating the lock-only follow-up PR...", flush=True) diff --git a/tests/unittest/others/test_maintain_prims_ts.py b/tests/unittest/others/test_maintain_prims_ts.py index 38097727218c..2c16e2234fa5 100644 --- a/tests/unittest/others/test_maintain_prims_ts.py +++ b/tests/unittest/others/test_maintain_prims_ts.py @@ -6,8 +6,10 @@ import argparse import base64 +import contextlib import copy import dataclasses +import importlib import importlib.util import runpy import subprocess @@ -27,6 +29,8 @@ m = importlib.util.module_from_spec(_SPEC) sys.modules[_SPEC.name] = m _SPEC.loader.exec_module(m) +sys.modules[f"{_SPEC.name}.promote"] = m +b = importlib.import_module(f"{_SPEC.name}.bot") _VENDOR = "example-vendor" _CONSUMER = "NVIDIA/TensorRT-LLM" @@ -146,6 +150,10 @@ def __init__(self, root: Path) -> None: self.fail_comment_before = False self.fail_comment_after = False self.git_calls: list[tuple[str, ...]] = [] + self.source_comments: list[dict] = [] + self.source_body = b.metadata.template(_VENDOR, _UPSTREAM).replace( + "/pull/123", "/pull/4829" + ) def args(self, **overrides: object) -> argparse.Namespace: values = dict( @@ -196,6 +204,7 @@ def routed_git(self, repo: Path, *args: str, text: str | None = None) -> str: "https://github.com/maintainer/TensorRT-LLM.git": str(self.fork), "https://github.com/maintainer/flashinfer.git": str(self.canonical), "https://github.com/dev/flashinfer.git": str(self.source), + "https://github.com/flashinfer-ai/flashinfer.git": str(self.source), } if args[:3] == ("remote", "get-url", "fork"): return "https://github.com/maintainer/TensorRT-LLM.git" @@ -241,6 +250,16 @@ def api( owner, name, *rest = endpoint[len(prefix) :].split("/") repository = f"{owner}/{name}" route = "/".join(rest) + if route.startswith("issues/17/comments"): + if method == "GET": + return copy.deepcopy(self.source_comments) + comment = { + "id": len(self.source_comments) + 1, + "body": payload["body"], + "user": {"login": "maintainer"}, + } + self.source_comments.append(comment) + return copy.deepcopy(comment) if route.startswith("issues/99/comments"): if method == "GET": page = int(route.rsplit("page=", 1)[1]) @@ -287,7 +306,16 @@ def api( if route.startswith("commits/"): return self.commit_data(self.consumer, route.split("/")[1]) if route.startswith("pulls?"): + if "head=" not in route: + if "state=open" in route: + return copy.deepcopy([pr for pr in self.prs.values() if pr["state"] == "open"]) + return [ + self.get(f"repos/{_CONSUMER}/pulls/17"), + *copy.deepcopy(list(self.prs.values())), + ] return copy.deepcopy(list(self.prs.values())) + if route.startswith("pulls/") and "/files?" in route: + return [{"filename": m._LOCK}] if route == "pulls" and method == "POST": if self.fail_create: raise RuntimeError("simulated PR creation failure") @@ -299,6 +327,8 @@ def api( state="open", merged=False, head={"sha": head}, + base={"ref": "main"}, + updated_at="2020-01-01T00:00:00Z", html_url="https://github.com/NVIDIA/TensorRT-LLM/pull/99", body=payload["body"], auto_merge=None, @@ -319,6 +349,11 @@ def api( return { "number": 17, "merged": True, + "state": "closed", + "merged_at": "2020-01-01T00:00:00Z", + "updated_at": "2020-01-01T00:00:00Z", + "user": {"login": "author"}, + "body": self.source_body, "base": {"ref": "main"}, "merge_commit_sha": self.merged, "head": {"sha": "a" * 40}, @@ -938,7 +973,7 @@ def compare(repo: str, base: str, head: str, **kwargs: object) -> dict: monkeypatch.setattr(world, "compare", compare) world.calls.clear() world.git_calls.clear() - args.flashinfer_repo = world.root / "missing-source" + args.source_repo = world.root / "missing-source" resumed = m._make_plan(args, world) assert resumed.completed_pr["merge_commit_sha"] == merged assert resumed.record == plan.record @@ -1110,3 +1145,229 @@ def run(command: list[str], **kwargs: object) -> subprocess.CompletedProcess: monkeypatch.setattr(m.subprocess, "run", run) assert m._git(Path("/test/repository"), "worktree", "add", "new") == "" assert m._git(Path("/test/repository"), "commit", "-s", "-F", "-", text="test") == "" + + +@pytest.mark.parametrize("loss", ["sqlite", "workdir"]) +@pytest.mark.parametrize( + "phase", + ["worktree", "stage", "commit", "branch", "canonical", "pr", "comment", "auto_merge", "merged"], +) +def test_bot_recovers_publication_after_cache_loss( + world: World, + monkeypatch: pytest.MonkeyPatch, + phase: str, + loss: str, +) -> None: + workdir = world.root / "bot-state" + b._initialize_workdir(workdir) + args = argparse.Namespace( + canonical_repo="maintainer/flashinfer", + canonical_branch="trtllm-prims-ts-dev", + fork="maintainer/TensorRT-LLM", + repo=world.consumer, + workdir=workdir, + publish=True, + since=None, + ) + database = workdir / b._STATE_FILE + store = b.Store(database) + monitor = b.Monitor(args, world, store) + if phase == "merged": + world.state = "CLEAN" + git, api = world.routed_git, world.api + + def crash_git(repo: Path, *arguments: str, text: str | None = None) -> str: + result = git(repo, *arguments, text=text) + boundary = ( + "worktree" + if arguments[:2] == ("worktree", "add") + else "stage" + if arguments[0] == "add" + else "commit" + if arguments[0] == "commit" + else "branch" + if arguments[:2] == ("push", "https://github.com/maintainer/TensorRT-LLM.git") + else "canonical" + if arguments[:2] == ("push", "https://github.com/maintainer/flashinfer.git") + else None + ) + if boundary == phase: + raise RuntimeError("simulated process loss") + return result + + def crash_api(endpoint: str, *, method: str = "GET", payload: dict | None = None): + result = api(endpoint, method=method, payload=payload) + if method == "POST": + boundary = ( + "pr" + if endpoint.endswith("/pulls") + else "comment" + if endpoint.endswith("/issues/99/comments") + else phase + if endpoint == "graphql" + and payload["query"].startswith("mutation") + and phase in ("auto_merge", "merged") + else None + ) + if boundary == phase: + raise RuntimeError("simulated process loss") + return result + + with monkeypatch.context() as crash: + crash.setattr(m, "_git", crash_git) + crash.setattr(world, "api", crash_api) + with pytest.raises(RuntimeError, match="simulated process loss"): + monitor.inspect(17) + # Publication of the provenance branch precedes the first canonical mutation. + if phase == "branch": + assert _git(world.canonical, "rev-parse", "trtllm-prims-ts-dev") == world.previous + committed = phase not in ("worktree", "stage") + expected_head = ( + m._local_branch(world.consumer, m._plan_identity(args=world.args(), gh=world).branch) + if committed + else None + ) + expected_record = ( + m._record_from_message(_git(world.consumer, "show", "-s", "--format=%B", expected_head)) + if committed + else None + ) + source_comments = copy.deepcopy(world.source_comments) + store.close() + if loss == "sqlite": + database.unlink() + else: + # Preserve the lost directory for assertions; Git's registration now + # refers to a missing path, as on workdir loss on the same machine. + workdir.rename(world.root / "lost-bot-state") + b._initialize_workdir(workdir) + + if committed: + world.source_body = "Description changed after the durable promotion commit." + world.overrides[f"repos/{_UPSTREAM}/pulls/4829"] = {"state": "closed", "merged": False} + + def no_rematching(*unused): + raise AssertionError("Durable promotion must not re-match mutable upstream PRs") + + monkeypatch.setattr(b.metadata, "resolve", no_rematching) + + store = b.Store(database) + resumed = b.Monitor(args, world, store) + assert 17 in resumed._discover() # The merged source PR predates startup by years. + resumed.poll() + assert store.get("problem:17") is None + assert store.get("problem:99") is None + assert world.source_comments == source_comments + assert len(world.prs) == len(world.comments) == 1 + assert _git(world.canonical, "rev-parse", "trtllm-prims-ts-dev") == world.reviewed + pr = world.prs[99] + if committed: + assert pr["head"]["sha"] == expected_head + assert sum(call[0] == "commit" for call in world.git_calls) == 1 + if not pr.get("merged"): + saved = copy.deepcopy(pr["auto_merge"]) + assert saved["merge_method"] == "squash" + writes = [ + (method, endpoint, payload) + for method, endpoint, payload in world.calls + if method != "GET" + and (endpoint != "graphql" or payload["query"].startswith("mutation")) + ] + resumed.inspect(17) + repeated = [ + (method, endpoint, payload) + for method, endpoint, payload in world.calls + if method != "GET" + and (endpoint != "graphql" or payload["query"].startswith("mutation")) + ] + assert writes == repeated + tree = _git(world.consumer, "rev-parse", f"{pr['head']['sha']}^{{tree}}") + world.main = _git( + world.consumer, + "commit-tree", + tree, + "-p", + world.main, + text=f"{saved['commit_title']}\n\n{saved['commit_message']}\n", + ) + pr.update(merged=True, merged_at="today", state="closed", merge_commit_sha=world.main) + resumed.inspect(17) + assert store.get("complete:17") == pr["html_url"] + store.close() + + # Even completion markers are only a cache: recover from the squash commit + # with an entirely new database and unavailable live source/metadata. + database.unlink() + world.source_body = "No metadata remains." + world.overrides["repos/dev/flashinfer"] = {} + calls_before = len(world.calls) + store = b.Store(database) + final = b.Monitor(args, world, store) + final.inspect(17) + assert store.get("complete:17") == pr["html_url"] + assert all(method == "GET" for method, _, _ in world.calls[calls_before:]) + assert m._lock_at(world, _CONSUMER, world.main).commit == world.reviewed + if committed: + assert ( + m._record_from_message(_git(world.consumer, "show", "-s", "--format=%B", world.main)) + == expected_record + ) + store.close() + + +@pytest.mark.parametrize("tamper", ["identity", "diff", "dco", "canonical", "closed"]) +def test_recovery_rejects_changed_artifacts_without_writes(world: World, tamper: str) -> None: + args = world.args(publish=True) + original = m._make_plan(args, world) + m._publish(args, world, original) + pr = world.prs[99] + endpoint = f"repos/{_CONSUMER}/commits/{pr['head']['sha']}" + commit = world.get(endpoint) + if tamper == "identity": + record = copy.deepcopy(original.record) + record["promoted"] = "a" * 40 + commit["commit"]["message"] = m._record_text(record) + elif tamper == "diff": + commit["files"].append({"filename": "unreviewed.py"}) + elif tamper == "dco": + commit["commit"]["message"] = m._record_text(original.record) + elif tamper == "canonical": + _git(world.canonical, "update-ref", "refs/heads/trtllm-prims-ts-dev", world.first) + else: + pr["state"] = "closed" + world.overrides[endpoint] = commit + world.calls.clear() + world.git_calls.clear() + with pytest.raises(ValueError): + m._recover_plan(args, world, m._plan_identity(args, world)) + assert all(method == "GET" for method, _, _ in world.calls) + assert not any(call[0] in ("push", "commit", "worktree") for call in world.git_calls) + + +def test_bot_cold_recovery_dry_run_and_superseded_history(world: World) -> None: + args = argparse.Namespace( + canonical_repo="maintainer/flashinfer", + canonical_branch="trtllm-prims-ts-dev", + fork="maintainer/TensorRT-LLM", + repo=world.consumer, + workdir=world.root, + publish=False, + since=None, + ) + # No published operation exists and a later pin has superseded this update. + world.main = world.base + world.source_body = "No metadata" + with contextlib.closing(b.Store(":memory:")) as store: + b.Monitor(args, world, store).inspect(17) + assert not world.source_comments + assert all(method == "GET" for method, _, _ in world.calls) + + world.main = world.merged + m._publish(world.args(publish=True), world, m._make_plan(world.args(), world)) + world.calls.clear() + world.git_calls.clear() + with contextlib.closing(b.Store(":memory:")) as store: + b.Monitor(args, world, store).inspect(17) + assert all(method == "GET" for method, _, _ in world.calls) + assert not world.git_calls + assert world.prs[99]["auto_merge"] is None diff --git a/tests/unittest/others/test_vendor_bot.py b/tests/unittest/others/test_vendor_bot.py index 29ac3bd6daa0..a9ce2c1f6dbf 100644 --- a/tests/unittest/others/test_vendor_bot.py +++ b/tests/unittest/others/test_vendor_bot.py @@ -5,6 +5,7 @@ from __future__ import annotations import argparse +import contextlib import copy import importlib import importlib.util @@ -340,6 +341,48 @@ def test_discovery_and_persistent_operator_scope(monitor: b.Monitor) -> None: b.Monitor(changed, monitor.gh, monitor.store) +def test_cold_discovery_backfills_old_merged_prs_and_honors_cutoff( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, +) -> None: + gh = monitor.gh + gh.pr.update( + state="closed", + merged=True, + merged_at="2020-01-01T00:00:00Z", + updated_at="2020-01-01T00:00:00Z", + ) + api = gh.api + + def filtered(endpoint: str, **kwargs): + if "/pulls?state=open" in endpoint: + return [] + return api(endpoint, **kwargs) + + monkeypatch.setattr(gh, "api", filtered) + assert monitor._discover() == [7] + with contextlib.closing(b.Store(":memory:")) as store: + restarted = b.Monitor(monitor.args, gh, store) + assert restarted._discover() == [7] + restricted = copy.copy(monitor.args) + restricted.since = "2021-01-01T00:00:00Z" + with contextlib.closing(b.Store(":memory:")) as store: + assert b.Monitor(restricted, gh, store)._discover() == [] + assert not gh.writes + + +def test_feedback_deduplicates_with_empty_sqlite(monitor: b.Monitor) -> None: + monitor._feedback(monitor.gh.pr, "Missing block") + writes = copy.deepcopy(monitor.gh.writes) + with contextlib.closing(b.Store(":memory:")) as store: + restarted = b.Monitor(monitor.args, monitor.gh, store) + restarted._feedback(monitor.gh.pr, "Missing block") + assert monitor.gh.writes == writes + restarted._feedback(monitor.gh.pr, None) + assert len(monitor.gh.comments) == len(monitor.gh.reviews) == 1 + assert monitor.gh.reviews[0]["state"] == "DISMISSED" + + def _vendor(commit: str, url: str, branch: str): return m.manage._validate_vendor( "toy", @@ -355,7 +398,7 @@ def _vendor(commit: str, url: str, branch: str): ) -def test_monitor_reuses_matching_and_publishes_once_after_merge( +def test_monitor_revalidates_matching_until_committed_and_publishes_once_after_merge( monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -384,6 +427,11 @@ def make_plan(args, github): consumer_repo="org/consumer", ) + monkeypatch.setattr(p, "_plan_identity", lambda args, gh: argparse.Namespace(main_sha="c" * 40)) + monkeypatch.setattr(p, "_recover_plan", lambda *args: False) + monkeypatch.setattr(p, "_lock_at", lambda *args: reviewed) + monkeypatch.setattr(p, "_check_current", lambda *args: None) + def publish(args, github, plan): publications.append(plan) assert plan.record["attribution"][reviewed.commit]["method"] == "same-edits" @@ -408,7 +456,7 @@ def publish(args, github, plan): monkeypatch.setattr(p, "_verify_remote_commit", lambda *args: "verified") monitor.inspect(7) monitor.inspect(7) - assert len(matches) == 1 + assert len(matches) == 2 assert not publications gh.pr.update(merged=True, state="closed") monitor.inspect(7) @@ -503,6 +551,35 @@ def test_workdir_refuses_checkout(source: Path) -> None: b._initialize_workdir(source / "bot-state") +def test_workdir_recovers_without_sqlite_but_preserves_artifacts(tmp_path: Path) -> None: + workdir = tmp_path / "state" + b._initialize_workdir(workdir) + (workdir / "bot.log").write_text("previous log\n") + (workdir / "bot.lock").touch() + (workdir / "sources.git").mkdir() + (workdir / ("promotion-7-" + "a" * 12)).mkdir() + b._initialize_workdir(workdir) + assert (workdir / "bot.log").read_text() == "previous log\n" + with b._locked(workdir): + with pytest.raises(ValueError, match="already owns"): + with b._locked(workdir): + pass + with contextlib.closing(b.Store(workdir / b._STATE_FILE)) as store: + assert store.get("tracked") is None + + +@pytest.mark.parametrize("name", ["state.sqlite", "bot.log", "bot.lock", "sources.git"]) +def test_workdir_recovery_rejects_symlinks(tmp_path: Path, name: str) -> None: + workdir = tmp_path / "state" + workdir.mkdir() + target = tmp_path / "private-data" + target.write_text("preserve") + (workdir / name).symlink_to(target) + with pytest.raises(ValueError, match="unexpected entry"): + b._initialize_workdir(workdir) + assert target.read_text() == "preserve" + + def test_daemon_launch_status_stop_without_systemd(tmp_path: Path) -> None: binary = tmp_path / "bin" binary.mkdir() From 1f482b4c4c94ffa9fe87a93b71a25b6731433520 Mon Sep 17 00:00:00 2001 From: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:54:50 +0000 Subject: [PATCH 05/11] [None][feat] Improve vendor bot monitoring and source attribution Add INFO progress reporting and daemon status diagnostics. Normalize contributor metadata line endings and compare only lock-selected source files, ignoring Git hunk context. Use exact matches first, accept unambiguous patch-line similarity of at least 90%, and report scores with durable attribution evidence. Cover monitoring, matching, and recovery behavior with regression tests. Signed-off-by: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> --- AGENTS.md | 3 + scripts/vendor/README.md | 104 ++- scripts/vendor/bot.py | 461 ++++++++-- scripts/vendor/metadata.py | 211 ++++- scripts/vendor/promote.py | 39 +- .../unittest/others/test_maintain_prims_ts.py | 127 ++- tests/unittest/others/test_vendor_bot.py | 795 +++++++++++++++++- 7 files changed, 1631 insertions(+), 109 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4f95a397c011..88ae8e124655 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -211,6 +211,9 @@ only to verified lock-only promotions, not changes to these tools. Git/GitHub promotion commits are authoritative; SQLite is a disposable cache. After cache loss, stop the old process and relaunch with the same operator/config. Cold discovery backfills merged PRs; `--since` explicitly limits that history. +For monitoring, use INFO poll summaries/30-second active-operation heartbeats +and `bot.py status --workdir ...`. Check the last poll outcome and next deadline; +the process lock and heartbeat alone do not establish successful reconciliation. ### Triggering CI diff --git a/scripts/vendor/README.md b/scripts/vendor/README.md index b9178a510209..e4021d8cb5d2 100644 --- a/scripts/vendor/README.md +++ b/scripts/vendor/README.md @@ -27,7 +27,7 @@ lock and is not migrated by this change. ```mermaid flowchart TD source[Source-update PR changes a vendor pin] --> metadata[Author lists upstream PRs in description] - metadata --> match[Bot validates metadata and matches immutable commits] + metadata --> match[Bot validates metadata and matches lock-selected file changes] match -->|missing or ambiguous| feedback[Request changes with author-facing instructions] feedback -->|author corrects block, pin, or targeted resolution| metadata match -->|valid and source PR merged| verify[Verify ancestry, current lock, and materialization] @@ -86,12 +86,53 @@ vendors: ```` -No complete commit map is required. The bot compares the exact source delta -against immutable upstream PR snapshots. Matching uses commit identity, -whitespace- and context-preserving edits, and whole-series aggregate changes for squash/split -cases. Commit titles, cherry-pick trailers, and fuzzy similarity are not proof. +No complete commit map is required. The bot compares only files selected by the +vendor lock's `source` directory and `include` patterns, using the same selection +rules as vendoring, on both sides of each comparison. For `flashinfer-prims-ts`, +this currently selects `flashinfer/attention/prims_ts/**/*.py`: differences in +external tests, benchmarks, or unselected documentation do not prevent a match. +Changing `source` or `include` during the source update requires a separate +selection change; automatic attribution refuses to silently narrow its scope. + +The selected source delta is compared against immutable upstream PR snapshots. +Matching uses commit identity, zero-context edits, and whole-series aggregate +changes for squash/split cases. +Edit matching ignores unchanged neighboring lines, line numbers, blob IDs, and +Git-generated hunk labels. It preserves changed-line whitespace, file paths/modes, +binary data, and hunk boundaries. Function definitions that are actually added or +removed remain part of the comparison; only labels in hunk headers are ignored. +Identical edits at different locations in the same file can therefore match; +this establishes textual attribution, not semantic equivalence. Exact matches +take priority; multiple exact matches remain ambiguous. Merged upstream PRs can still be paired using their original head history. +When a source commit has no exact match, the bot compares its normalized patch +lines against each upstream commit and PR aggregate. A unique upstream PR with +similarity **>= 0.9** is accepted. The metric is Python +`SequenceMatcher(autojunk=False)`: `2 * matched_lines / (source_lines + upstream_lines)`. +Whitespace remains significant, repeated lines are not discarded, and empty +patches never establish similarity. The unrounded score determines acceptance; +comments display rounded percentages. Whole-source-series matching still requires +exact equality, so aggregate similarity cannot hide an unmatched individual commit. + +Comments show accepted non-exact matches with their scores and threshold. Below +the threshold, or when multiple PRs qualify, author-action feedback shows the +closest upstream PRs and scores. Similarity matches record the score, metric, +threshold, compared immutable revisions, and `retain-until-verified` refresh policy +in promotion provenance. They count as attribution for promotion, not code-review +approval or proof that changes can be dropped during refresh. Commit titles and +cherry-pick trailers are not evidence. Authors still resolve ambiguous or +below-threshold matches explicitly. + +Commits with no lock-selected changes need no upstream attribution or author +resolution. Empty selected patches never count as an upstream match; omit PRs +that cover only excluded changes. The promotion record saves `attribution_scope` +and marks excluded-only commits `outside-vendor-scope`, in an unpaired group with +`refresh_policy: retain`. All source commits still undergo the linear-history +and ancestry checks, and the promoted SHA and vendored-tree verification remain +unchanged. Scoped attribution does not prove whole-commit equivalence or authorize +dropping changes to excluded files during a future source-branch refresh. + If attribution is unresolved, feedback lists the full source SHAs and candidate PRs. **The PR author**, not the operator, resolves it by adding missing upstream PRs, removing unrelated PRs, correcting/rebasing the pinned source, or adding a @@ -165,6 +206,59 @@ same cutoff on every launch, including recovery. Historical updates belonging to another canonical branch, or superseded pins without a recoverable promotion, are not promoted or sent author feedback. +### Monitoring health + +`--log-level INFO` is the default, in both foreground and daemon mode. Logs use +UTC timestamps. INFO includes poll start/end, discovery counts, elapsed time, +PR metadata/feedback and promotion transitions, and the next polling deadline. +Unchanged PR outcomes are not logged repeatedly. `WARNING`/`ERROR` suppress the +INFO heartbeat and summaries; DEBUG is not needed for normal monitoring. Git +and promotion subprocess output still goes to the daemon log independently of +the Python logging threshold. + +During an active operation, a log-only heartbeat runs every 30 seconds, including +while the worker waits for GitHub or Git. It reports the phase, PR number (when +known), operation age, and counters. A heartbeat proves only that the reporting +thread is alive: unchanged counters and growing operation age may indicate a +slow or stuck operation. It does not advance the progress or completion timestamp. +Idle intervals are described by `Next poll at ...`, not continuous heartbeat noise. + +`status` reads the local cache without creating or repairing a database: + +```bash +python scripts/vendor/bot.py status --workdir /path/to/private/vendor-bot-state +tail -n 100 /path/to/private/vendor-bot-state/bot.log +``` + +The JSON separates process liveness from work completion: + +| Field | Meaning | +| --- | --- | +| `running`, `pid`, `publish` | Process lock held, recorded PID, and whether remote writes are enabled; not a health verdict | +| `operation`, `progress_at` | Current phase/PR/start time and last worker checkpoint | +| `counts` | Current poll's scanned records, tracked lock PRs, relevant vendor updates, processed PRs, author-blocked PRs, and errors | +| `last_completed_poll` | Last fully scanned/processed poll, even if individual PRs needed attention | +| `last_successful_poll` | Last completed poll with no blocked PRs or processing errors; not a code-review approval or guarantee that promotions have merged | +| `last_poll_summary` | Duration, counts, and outcome: `ok`, `needs_attention`, `failed`, or `interrupted` | +| `last_error` | Historical error timestamp, PR, and bounded single-line diagnostic; retained after recovery | +| `next_poll_at` | Next scheduled poll, including exponential backoff after discovery failures | +| `state_available`, `state_error` | Whether cached telemetry could be read; missing/corrupt state does not mean a held process lock is healthy | + +`scanned` counts PR records visited, not unique PRs (the initial open/history +scans overlap). `tracked` includes lock changes for other vendors; `relevant` +counts the selected vendor's eligible updates before promotion-state checks. +Blocked PRs are author/metadata issues; errors are failed processing attempts. +Logs include their PR numbers and reasons. Diagnostic summaries redact configured +GitHub tokens, recognizable token formats, URL credentials, and authorization +headers, and omit multiline YAML excerpts; no PR bodies or authentication +configuration are intentionally logged. + +For monitoring, check the latest poll outcome and deadline along with the log. +Do not interpret `running: true` or an old `last_error` alone as health/failure. +Stopped processes have no current PID, operation, or next deadline in `status`, +but retain historical poll/error information. Telemetry is disposable and never +authorizes publication; restart recovery still verifies Git/GitHub provenance. + ### Recovery after cache loss Git/GitHub are the source of truth; SQLite is not required to reconstruct a diff --git a/scripts/vendor/bot.py b/scripts/vendor/bot.py index e01ac4ad96d1..f66da0682902 100644 --- a/scripts/vendor/bot.py +++ b/scripts/vendor/bot.py @@ -32,6 +32,23 @@ _LOG = logging.getLogger("vendor-bot") _STATE_FILE = "state.sqlite" _STOP = threading.Event() +_HEARTBEAT_SECONDS = 30 + + +def _now() -> str: + return datetime.datetime.now(datetime.timezone.utc).isoformat() + + +def _summary(error: object) -> str: + """Keep diagnostics single-line; exclude YAML excerpts and authentication secrets.""" + text = str(error).splitlines()[0] if str(error) else type(error).__name__ + for name in ("GH_TOKEN", "GITHUB_TOKEN", "GH_ENTERPRISE_TOKEN", "GITHUB_ENTERPRISE_TOKEN"): + if os.environ.get(name): + text = text.replace(os.environ[name], "[REDACTED]") + text = re.sub(r"https?://[^\s/]+@", "https://[REDACTED]@", text) + text = re.sub(r"\b(?:gh[pousr]_[A-Za-z0-9_]+|github_pat_[A-Za-z0-9_]+)", "[REDACTED]", text) + text = re.sub(r"(?i)(authorization\s*[:=]\s*).+", r"\1[REDACTED]", text) + return text[:500] class Store: @@ -59,6 +76,81 @@ def close(self) -> None: self.connection.close() +class Progress: + """Main-thread status checkpoints with an independent, log-only heartbeat.""" + + def __init__(self, store: Store) -> None: + self.store = store + previous = store.get("health", {}) + self.state = { + "operation": None, + "progress_at": None, + "poll_started_at": None, + "last_completed_poll": previous.get("last_completed_poll"), + "last_successful_poll": previous.get("last_successful_poll"), + "last_poll_summary": previous.get("last_poll_summary"), + "last_error": previous.get("last_error"), + "next_poll_at": None, + "counts": {}, + } + self.lock = threading.Lock() + self.last_checkpoint = time.monotonic() + + def update(self, **fields: object) -> None: + with self.lock: + self.state.update(fields) + self.store.put("health", self.state) + self.last_checkpoint = time.monotonic() + + def operation(self, phase: str, number: int | None = None) -> None: + self.update( + operation={"phase": phase, "pr": number, "started_at": _now()}, progress_at=_now() + ) + + def count(self, name: str, amount: int = 1) -> None: + with self.lock: + self.state["counts"][name] = self.state["counts"].get(name, 0) + amount + if time.monotonic() - self.last_checkpoint >= _HEARTBEAT_SECONDS: + self.update(progress_at=_now()) + + def error(self, error: object, number: int | None = None) -> None: + self.update(last_error={"at": _now(), "pr": number, "message": _summary(error)}) + + def heartbeat(self) -> None: + with self.lock: + operation = self.state["operation"] + if operation is None or operation["phase"] in ("waiting", "stopped"): + return + age = ( + datetime.datetime.now(datetime.timezone.utc) + - datetime.datetime.fromisoformat(operation["started_at"]) + ).total_seconds() + _LOG.info( + "Heartbeat: phase=%s pr=%s operation_age=%.1fs counts=%s; " + "worker alive, operation completion not yet confirmed", + operation["phase"], + operation["pr"], + age, + self.state["counts"], + ) + + @contextlib.contextmanager + def heartbeats(self) -> Iterator[None]: + finished = threading.Event() + + def report() -> None: + while not finished.wait(_HEARTBEAT_SECONDS): + self.heartbeat() + + thread = threading.Thread(target=report, name="vendor-bot-heartbeat", daemon=True) + thread.start() + try: + yield + finally: + finished.set() + thread.join() + + def _pages(gh: promote.GitHub, endpoint: str) -> Iterator[dict]: separator = "&" if "?" in endpoint else "?" page = 1 @@ -75,8 +167,15 @@ def _pages(gh: promote.GitHub, endpoint: str) -> Iterator[dict]: class Monitor: """One operator-configured vendor; run other vendors in separate workdirs.""" - def __init__(self, args: argparse.Namespace, gh: promote.GitHub, store: Store) -> None: + def __init__( + self, + args: argparse.Namespace, + gh: promote.GitHub, + store: Store, + progress: Progress | None = None, + ) -> None: self.args, self.gh, self.store = args, gh, store + self.progress = progress or Progress(store) self.actor = gh.get("user")["login"] self.prefix = f"repos/{gh.consumer_repo}" self.marker = f"" @@ -115,10 +214,22 @@ def _discover(self) -> list[int]: if not self.store.get("bootstrapped", False): endpoints.insert(0, f"{self.prefix}/pulls?state=open") since = self.store.get("cursor", self.store.get("since")) + _LOG.info( + "Discovery started: since=%s cold_start=%s", + since, + not self.store.get("bootstrapped", False), + ) for endpoint in endpoints: + self.progress.operation( + "discover_open" if "state=open" in endpoint else "discover_history" + ) for pr in _pages(self.gh, endpoint): + if _STOP.is_set() or (self.args.workdir / "stop").exists(): + _STOP.set() + return sorted(tracked) if "state=all" in endpoint and pr["updated_at"] < since: break + self.progress.count("scanned") if pr["base"]["ref"] != self.gh.base_branch: continue number = pr["number"] @@ -139,9 +250,24 @@ def _discover(self) -> list[int]: self.store.put("tracked", sorted(tracked)) self.store.put("bootstrapped", True) self.store.put("cursor", cutoff) + self.progress.update(progress_at=_now()) + _LOG.info( + "Discovery completed: scanned=%s tracked_lock_prs=%s", + self.progress.state["counts"].get("scanned", 0), + len(tracked), + ) return sorted(tracked) - def _feedback(self, pr: dict, problem: str | None) -> None: + def _transition(self, number: int, state: str, reason: str = "") -> None: + observation = {"state": state, "reason": _summary(reason) if reason else ""} + if self.store.get(f"observation:{number}") != observation: + _LOG.info("PR #%s: %s%s", number, state, f": {observation['reason']}" if reason else "") + self.store.put(f"observation:{number}", observation) + self.store.put(f"problem:{number}", observation["reason"] if state == "blocked" else None) + + def _feedback( + self, pr: dict, problem: str | None, *, attribution: dict[str, dict] | None = None + ) -> None: number = pr["number"] message = ( f"@{pr['user']['login']} — vendor promotion needs author action:\n\n{problem}\n\n" @@ -157,18 +283,29 @@ def _feedback(self, pr: dict, problem: str | None) -> None: else "Vendor promotion metadata and source attribution are valid. " "This is **not** a code-review approval. Promotion still requires the source PR to merge." ) + if attribution: + details = metadata.similarity_feedback(attribution, self.gh.upstream_repo) + if details: + message += "\n\n" + details body = f"{self.marker}\n{message}" dismissal_notice = ( "\n\nAn authorized reviewer must dismiss the automated metadata " "review; this account lacks dismissal permission." ) if not self.args.publish: - _LOG.info("PR #%s: %s", number, problem or "metadata valid (dry run)") + self._transition( + number, + "blocked" if problem else "metadata_valid", + problem or "dry run; no feedback published", + ) return # Check immutable head and description immediately before feedback. A # concurrent author edit will be retried on the next poll. current = self.gh.get(f"{self.prefix}/pulls/{number}") if current["head"]["sha"] != pr["head"]["sha"] or current.get("body") != pr.get("body"): + self._transition( + number, "deferred", "source head or description changed; retry next poll" + ) return comments = list(_pages(self.gh, f"{self.prefix}/issues/{number}/comments")) existing = [ @@ -187,11 +324,18 @@ def _feedback(self, pr: dict, problem: str | None) -> None: method="PATCH", payload={"body": body}, ) + _LOG.info("PR #%s: updated metadata feedback comment", number) else: self.gh.api( f"{self.prefix}/issues/{number}/comments", method="POST", payload={"body": body} ) + _LOG.info("PR #%s: posted metadata feedback comment", number) if current["state"] != "open" or current["user"]["login"] == self.actor: + self._transition( + number, + "blocked" if problem else "metadata_valid", + problem or "feedback reconciled; not a code-review approval", + ) return reviews = [ item @@ -222,6 +366,7 @@ def _feedback(self, pr: dict, problem: str | None) -> None: ), }, ) + _LOG.info("PR #%s: requested metadata changes", number) elif not problem: for review in automated: # Never replace a metadata objection with APPROVE or dismiss a @@ -235,6 +380,7 @@ def _feedback(self, pr: dict, problem: str | None) -> None: "event": "DISMISS", }, ) + _LOG.info("PR #%s: dismissed resolved metadata review %s", number, review["id"]) except RuntimeError as error: if "HTTP 403" not in str(error): raise @@ -243,6 +389,9 @@ def _feedback(self, pr: dict, problem: str | None) -> None: number, review["id"], ) + self._transition( + number, "blocked", "Authorized human must dismiss metadata review" + ) if existing and existing[0]["body"] != body + dismissal_notice: self.gh.api( f"{self.prefix}/issues/comments/{existing[0]['id']}", @@ -256,8 +405,16 @@ def _feedback(self, pr: dict, problem: str | None) -> None: method="PATCH", payload={"body": body}, ) + self._transition( + number, + "blocked" if problem else "metadata_valid", + problem or "feedback reconciled; not a code-review approval", + ) def _inputs(self, pr: dict) -> tuple[manage.Vendor, manage.Vendor] | None: + for change in _pages(self.gh, f"{self.prefix}/pulls/{pr['number']}/files"): + if change["filename"] == promote._LOCK and change.get("status") in ("added", "removed"): + return None # Creating/removing the entire lock is not a source update. if pr.get("merged"): revision = promote._sha(pr["merge_commit_sha"]) commit = self.gh.get(f"{self.prefix}/commits/{revision}") @@ -313,13 +470,13 @@ def _publish(self, arguments: argparse.Namespace, plan: promote.Promotion) -> No number = plan.number if plan.completed_pr is not None: self.store.put(f"complete:{number}", plan.completed_pr["html_url"]) - _LOG.info( - "PR #%s: verified completed promotion %s", number, plan.completed_pr["html_url"] - ) + self._transition(number, "completed", plan.completed_pr["html_url"]) return if not self.args.publish: - _LOG.info("PR #%s: promotion ready (dry run): %s", number, plan.branch) + self._transition(number, "promotion_ready", f"dry run: {plan.branch}") return + self.progress.operation("publishing_promotion", number) + self._transition(number, "promoting", "create or resume verified lock-only promotion") followup = promote._publish(arguments, self.gh, plan) self.store.put(f"promotion:{number}", followup["number"]) latest = self.gh.get(f"{self.prefix}/pulls/{followup['number']}") @@ -328,62 +485,81 @@ def _publish(self, arguments: argparse.Namespace, plan: promote.Promotion) -> No self.gh, plan, self.gh.consumer_repo, latest["merge_commit_sha"] ) self.store.put(f"complete:{number}", latest["html_url"]) - _LOG.info("PR #%s: promotion %s", number, latest["html_url"]) + self._transition( + number, "completed" if latest.get("merged") else "promotion_pending", latest["html_url"] + ) def inspect(self, number: int) -> None: """Reconcile one source-update PR and its deterministic promotion operation.""" - if self.store.get(f"complete:{number}"): + completed = self.store.get(f"complete:{number}") + if completed: + self._transition(number, "completed", completed) return + self.progress.operation("inspecting", number) pr = self.gh.get(f"{self.prefix}/pulls/{number}") if pr["state"] == "closed" and not pr.get("merged"): + self._transition(number, "ignored", "closed without merging") return try: inputs = self._inputs(pr) if inputs is None: + self._transition( + number, "ignored", "not a promotable update for this canonical vendor" + ) return previous, reviewed = inputs + self.progress.count("relevant") except (ValueError, manage.VendorError) as error: self._feedback(pr, str(error)) - self.store.put(f"problem:{number}", str(error)) return arguments = self._arguments(number, reviewed) if pr.get("merged"): + self.progress.operation("recovering_promotion", number) plan = promote._plan_identity(arguments, self.gh) if promote._recover_plan(arguments, self.gh, plan): + self._transition(number, "recovered", "verified durable promotion provenance") self._publish(arguments, plan) return current = promote._lock_at(self.gh, self.gh.consumer_repo, plan.main_sha) if current.to_mapping() != reviewed.to_mapping(): - _LOG.info("PR #%s: historical pin is no longer pending promotion", number) + self._transition(number, "ignored", "historical pin is no longer pending promotion") return promote._check_current(self.gh, plan) try: + self.progress.operation("validating_metadata", number) entry = metadata.parse(pr.get("body") or "", self.gh.vendor_name, self.gh.upstream_repo) cache = self.args.workdir / "sources.git" # Until provenance is committed, decisions follow current remote # inputs, not whichever upstream heads a previous process cached. - assignments, evidence = metadata.resolve(self.gh, previous, reviewed, entry, cache) + _, evidence = metadata.resolve(self.gh, previous, reviewed, entry, cache) except (ValueError, manage.VendorError) as error: self._feedback(pr, str(error)) - self.store.put(f"problem:{number}", str(error)) return - self._feedback(pr, None) + self.progress.operation("reconciling_feedback", number) + self._feedback(pr, None, attribution=evidence) if not pr.get("merged"): return # Re-read mutable description/head before any promotion. Immutable lock # checks inside promote also run before each remote write. current = self.gh.get(f"{self.prefix}/pulls/{number}") if current.get("body") != pr.get("body") or current["head"]["sha"] != pr["head"]["sha"]: + self._transition( + number, "deferred", "source head or description changed; retry next poll" + ) return arguments.upstream_pr = [str(item) for item in entry["upstream_prs"]] arguments.unpaired_reason = entry["unpaired_reason"] - arguments.map_upstream = [f"{sha}={target}" for sha, target in assignments.items()] arguments.source_repo = cache if cache.exists() else None - plan = promote._make_plan(arguments, self.gh) + self.progress.operation("planning_promotion", number) + plan = promote._make_plan( + arguments, + self.gh, + auto_match=bool(entry["upstream_prs"]), + resolutions=entry["resolutions"], + ) # Attribution evidence is versioned with the durable promotion record. # Reuse the original published record on retries rather than replacing it. if plan.completed_pr is None: - plan.record["attribution"] = evidence plan.record["contributor_metadata"] = { "source_head": pr["head"]["sha"], "metadata_digest": metadata.fingerprint(entry), @@ -393,21 +569,67 @@ def inspect(self, number: int) -> None: def poll(self) -> None: """Process candidates serially; preserve failures for safe later retries.""" - for number in self._discover(): - if _STOP.is_set() or (self.args.workdir / "stop").exists(): - _STOP.set() - break - try: - self.inspect(number) - except ( - RuntimeError, - ValueError, - OSError, - KeyError, - subprocess.TimeoutExpired, - ) as error: - self.store.put(f"problem:{number}", str(error)) - _LOG.error("PR #%s paused: %s", number, error) + started = time.monotonic() + self.progress.update( + poll_started_at=_now(), + next_poll_at=None, + counts=dict(scanned=0, tracked=0, relevant=0, processed=0, blocked=0, errors=0), + ) + _LOG.info("Poll started") + completed = False + try: + candidates = self._discover() + self.progress.count("tracked", len(candidates)) + for number in candidates: + if _STOP.is_set() or (self.args.workdir / "stop").exists(): + _STOP.set() + break + try: + self.inspect(number) + if self.store.get(f"problem:{number}"): + self.progress.count("blocked") + except ( + RuntimeError, + ValueError, + OSError, + KeyError, + subprocess.TimeoutExpired, + ) as error: + self.progress.count("errors") + self.progress.error(error, number) + self._transition(number, "blocked", _summary(error)) + _LOG.error("PR #%s paused: %s", number, _summary(error)) + self.progress.count("processed") + completed = not _STOP.is_set() + except (RuntimeError, OSError, ValueError, KeyError, subprocess.TimeoutExpired) as error: + self.progress.count("errors") + self.progress.error(error) + raise + finally: + counts = self.progress.state["counts"].copy() + outcome = ( + "failed" + if not completed and not _STOP.is_set() + else "interrupted" + if _STOP.is_set() + else "needs_attention" + if counts["errors"] or counts["blocked"] + else "ok" + ) + summary = { + "outcome": outcome, + "finished_at": _now(), + "duration_seconds": round(time.monotonic() - started, 3), + **counts, + } + fields = {"last_poll_summary": summary, "progress_at": _now()} + if completed: + fields["last_completed_poll"] = summary["finished_at"] + self.store.put("last_poll", summary["finished_at"]) + if outcome == "ok": + fields["last_successful_poll"] = summary["finished_at"] + self.progress.update(**fields) + _LOG.info("Poll finished: %s", summary) def _initialize_workdir(path: Path) -> None: @@ -473,40 +695,128 @@ def _serve(args: argparse.Namespace) -> None: (args.workdir / "stop").unlink(missing_ok=True) with contextlib.closing(Store(args.workdir / _STATE_FILE)) as store: store.put("process", {"pid": None, "publish": args.publish}) - gh = promote.GitHub( - args.consumer_repo, - args.vendor, - args.upstream_repo, - args.base_branch, - args.upstream_branch, - ) - monitor = Monitor(args, gh, store) - store.put("process", {"pid": os.getpid(), "publish": args.publish}) - _LOG.info( - "Started vendor %s as %s; remote writes %s", - args.vendor, - monitor.actor, - args.publish, - ) - failures = 0 - while not _STOP.is_set(): + progress = Progress(store) + progress.operation("starting") + with progress.heartbeats(): try: - monitor.poll() + gh = promote.GitHub( + args.consumer_repo, + args.vendor, + args.upstream_repo, + args.base_branch, + args.upstream_branch, + ) + monitor = Monitor(args, gh, store, progress) + store.put("process", {"pid": os.getpid(), "publish": args.publish}) + _LOG.info( + "Started vendor %s as %s; remote writes %s; interval=%ss", + args.vendor, + monitor.actor, + args.publish, + args.interval, + ) failures = 0 - store.put("last_poll", datetime.datetime.now(datetime.timezone.utc).isoformat()) - except (RuntimeError, OSError, ValueError, subprocess.TimeoutExpired) as error: - failures += 1 - _LOG.error("Poll paused: %s", error) - if args.once: - raise - if args.once: - break - deadline = time.monotonic() + min(args.interval * 2 ** min(failures, 5), 3600) - while time.monotonic() < deadline and not _STOP.wait(1): - if (args.workdir / "stop").exists(): - _STOP.set() - break - store.put("process", {"pid": None, "publish": args.publish}) + while not _STOP.is_set(): + try: + monitor.poll() + failures = 0 + except ( + RuntimeError, + OSError, + ValueError, + KeyError, + subprocess.TimeoutExpired, + ) as error: + failures += 1 + _LOG.error("Poll paused: %s", _summary(error)) + if args.once: + raise + if args.once or _STOP.is_set(): + break + delay = min(args.interval * 2 ** min(failures, 5), 3600) + deadline = time.monotonic() + delay + progress.operation("waiting") + progress.update( + next_poll_at=( + datetime.datetime.now(datetime.timezone.utc) + + datetime.timedelta(seconds=delay) + ).isoformat() + ) + _LOG.info( + "Next poll at %s (in %ss; consecutive_poll_failures=%s)", + progress.state["next_poll_at"], + delay, + failures, + ) + while time.monotonic() < deadline and not _STOP.wait(1): + if (args.workdir / "stop").exists(): + _STOP.set() + break + except ( + RuntimeError, + OSError, + ValueError, + KeyError, + subprocess.TimeoutExpired, + ) as error: + progress.error(error) + raise + finally: + progress.operation("stopped") + progress.update(next_poll_at=None) + store.put("process", {"pid": None, "publish": args.publish}) + _LOG.info("Stopped vendor %s", args.vendor) + + +def _status(workdir: Path) -> dict: + """Read cached telemetry without creating or repairing the state database.""" + active = _running(workdir) + result = { + "running": active, + "stop_requested": active and (workdir / "stop").exists(), + "pid": None, + "publish": None, + "operation": None, + } + database = workdir / _STATE_FILE + if not database.exists(): + result["state_available"] = False + return result + try: + with contextlib.closing( + sqlite3.connect(f"{database.as_uri()}?mode=ro", uri=True, timeout=1) + ) as connection: + rows = connection.execute( + "SELECT key, value FROM state WHERE key IN ('health', 'process', 'settings')" + ).fetchall() + state = {key: json.loads(value) for key, value in rows} + if any(not isinstance(value, dict) for value in state.values()): + raise ValueError("Malformed status cache; expected JSON objects.") + health, process = state.get("health", {}), state.get("process", {}) + for key in ( + "operation", + "progress_at", + "poll_started_at", + "last_completed_poll", + "last_successful_poll", + "last_poll_summary", + "last_error", + "next_poll_at", + "counts", + ): + result[key] = health.get(key) + result.update( + state_available=True, + pid=process.get("pid") if active else None, + publish=process.get("publish"), + vendor=state.get("settings", {}).get("vendor"), + ) + if not active: + result["operation"] = None + result["next_poll_at"] = None + except (sqlite3.Error, ValueError, OSError) as error: + result.update(state_available=False, state_error=_summary(error)) + return result def _parser() -> argparse.ArgumentParser: @@ -531,6 +841,13 @@ def _parser() -> argparse.ArgumentParser: run.add_argument( "--interval", type=int, default=120, help="Polling interval, seconds (minimum 30)." ) + run.add_argument( + "--log-level", + type=str.upper, + choices=("DEBUG", "INFO", "WARNING", "ERROR"), + default="INFO", + help="Daemon/foreground log threshold (default: INFO).", + ) run.add_argument( "--since", help=( @@ -560,15 +877,17 @@ def main(argv: list[str] | None = None) -> int: """Launch/control a POSIX daemon without storing credentials or requiring systemd.""" args = _parser().parse_args(argv) args.workdir = args.workdir.expanduser().resolve() - logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + logging.basicConfig( + level=getattr(args, "log_level", "INFO"), format="%(asctime)sZ %(levelname)s %(message)s" + ) + logging.Formatter.converter = time.gmtime try: if args.command != "run": - active = _running(args.workdir) - if args.command == "stop" and active: + status = _status(args.workdir) + if args.command == "stop" and status["running"]: (args.workdir / "stop").touch(mode=0o600) - print( - json.dumps({"running": active, "stop_requested": args.command == "stop" and active}) - ) + status["stop_requested"] = True + print(json.dumps(status)) return 0 if args.interval < 30: raise ValueError("--interval must be at least 30 seconds.") @@ -625,7 +944,7 @@ def main(argv: list[str] | None = None) -> int: _serve(args) return 0 except (ValueError, RuntimeError, OSError, sqlite3.Error, subprocess.TimeoutExpired) as error: - _LOG.error("Bot stopped: %s", error) + _LOG.error("Bot stopped: %s", _summary(error)) return 1 diff --git a/scripts/vendor/metadata.py b/scripts/vendor/metadata.py index ddcaac7bd2df..89babbe6e734 100644 --- a/scripts/vendor/metadata.py +++ b/scripts/vendor/metadata.py @@ -11,7 +11,8 @@ import hashlib import json import re -from pathlib import Path +from difflib import SequenceMatcher +from pathlib import Path, PurePosixPath import yaml @@ -23,6 +24,8 @@ BEGIN = "" END = "" +SIMILARITY_THRESHOLD = 0.9 +_SIMILARITY_METRIC = "sequence-matcher-lines-v1" def template(vendor_name: str, upstream_repo: str) -> str: @@ -38,6 +41,7 @@ def parse(body: str, vendor_name: str, upstream_repo: str) -> dict: """Read exactly one marked block and validate the selected vendor's inputs.""" if len(body) > 65536: raise ValueError("PR description exceeds the supported metadata size.") + body = body.replace("\r\n", "\n").replace("\r", "\n") if body.count(BEGIN) != 1 or body.count(END) != 1: raise ValueError("PR description must contain exactly one vendor-promotion marked block.") start, end = body.index(BEGIN) + len(BEGIN), body.index(END) @@ -121,8 +125,40 @@ def fingerprint(value: dict) -> str: return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest() -def _patch(repo: Path, base: str, head: str) -> str: - """Normalize hunk coordinates, retaining whitespace, paths, modes, and binary data.""" +def _patch( + repo: Path, base: str, head: str, *, source: str = ".", include: tuple[str, ...] = ("**/*",) +) -> str: + """Compare lock-selected edits, retaining bytes, paths, modes, and hunk boundaries.""" + changed = promote._run( + [ + "git", + "-C", + str(repo), + "diff", + "--no-ext-diff", + "--no-textconv", + "--no-renames", + "--name-only", + "-z", + base, + head, + "--", + ], + strip=False, + binary_output=True, + ) + # Use the vendoring glob semantics rather than Git's different pathspec + # rules. Enumerate both sides without rename detection to include deletions + # and moves across the selection boundary; pass the results as literal paths. + selected = [] + for path in changed.split("\0"): + if not path or not PurePosixPath(path).is_relative_to(source): + continue + relative = PurePosixPath(path).relative_to(source).as_posix() + if relative != "." and manage._matches(relative, include): + selected.append(f":(top,literal){path}") + if not selected: + return "" diff = promote._run( [ "git", @@ -135,18 +171,20 @@ def _patch(repo: Path, base: str, head: str) -> str: "--binary", "--full-index", "--no-color", - "--unified=3", + "--unified=0", + "--inter-hunk-context=0", base, head, "--", + *selected, ], strip=False, binary_output=True, ) - # Blob IDs and hunk positions change on rebases. Keep surrounding lines and - # function context so identical edits at different code sites do not match. + # Hunk labels are Git-generated context, not changed code. Keep only the + # boundary marker; edit bytes, including changed function definitions, remain. return "\n".join( - re.sub(r"^@@ -[0-9]+(?:,[0-9]+)? \+[0-9]+(?:,[0-9]+)? @@", "@@", line) + re.sub(r"^@@ -[0-9]+(?:,[0-9]+)? \+[0-9]+(?:,[0-9]+)? @@.*$", "@@", line) for line in diff.split("\n") if not line.startswith("index ") ) @@ -156,41 +194,99 @@ def _commits(repo: Path, base: str, head: str) -> list[str]: return promote._git(repo, "rev-list", "--reverse", f"{base}..{head}").splitlines() +def _similarity(source_patch: str, upstream_patch: str) -> float: + """Score normalized patch lines without discarding whitespace or frequent lines.""" + if not source_patch or not upstream_patch: + return 0.0 + if source_patch == upstream_patch: + return 1.0 + return SequenceMatcher( + None, + source_patch.removesuffix("\n").split("\n"), + upstream_patch.removesuffix("\n").split("\n"), + autojunk=False, + ).ratio() + + +def _upstream_reference(number: int, repository: str | None) -> str: + label = f"upstream PR #{number}" + return f"[{label}](https://github.com/{repository}/pull/{number})" if repository else label + + +def similarity_feedback(evidence: dict[str, dict], upstream_repo: str | None = None) -> str: + """Describe accepted non-exact matches for feedback and durable provenance.""" + lines = [ + f"- `{sha}`: {_upstream_reference(item['upstream_pr'], upstream_repo)}, similarity " + f"**{item['similarity']:.2%}** — accepted " + f"(threshold {item['similarity_threshold']:.2%}; {item['method']})." + for sha, item in evidence.items() + if "similarity" in item + ] + if not lines: + return "" + return ( + "No exact match was found for the following commits; similarity matching accepted them:\n\n" + + "\n".join(lines) + + "\n\nScores compare normalized patch lines in lock-selected files, not semantic " + "equivalence. These changes retain `retain-until-verified` refresh policy." + ) + + def match( repo: Path, previous: str, reviewed: str, upstream_prs: dict[int, dict], resolutions: dict[str, dict] | None = None, -) -> tuple[dict[str, int], dict[str, dict]]: + *, + source: str = ".", + include: tuple[str, ...] = ("**/*",), + upstream_repo: str | None = None, +) -> tuple[dict[str, int | None], dict[str, dict]]: """Match a linear source range against fetched immutable upstream PR snapshots. - Exact edits are compared across all files, not only selected vendored files. + Only lock-selected files are compared. None assignments record commits with + no selected changes, not upstream equivalence for the rest of those commits. Ambiguities remain author-actionable; titles and cherry-pick trailers are not proof. """ commits = _commits(repo, previous, reviewed) if not commits: raise ValueError("Source update has no new commits to attribute.") + source_parents = {} for sha in commits: - if len(promote._git(repo, "show", "-s", "--format=%P", sha).split()) != 1: + parents = promote._git(repo, "show", "-s", "--format=%P", sha).split() + if len(parents) != 1: raise ValueError("Source history must be linear; rebase merge commits first.") + source_parents[sha] = parents[0] resolutions = resolutions or {} if set(resolutions) - set(commits): raise ValueError("Resolutions include commits outside the reviewed source delta.") candidates: dict[str, set[int]] = {sha: set() for sha in commits} - patches = {sha: _patch(repo, f"{sha}^", sha) for sha in commits} + patches = {sha: _patch(repo, f"{sha}^", sha, source=source, include=include) for sha in commits} + if any(not patches[sha] for sha in resolutions): + raise ValueError("Remove resolutions for commits with no lock-selected file changes.") + series = _patch(repo, previous, reviewed, source=source, include=include) methods: dict[tuple[str, int], str] = {} + comparisons: dict[int, list[tuple[str, str, str, str]]] = {} for number, pr in upstream_prs.items(): base, head = promote._sha(pr["base"]["sha"]), promote._sha(pr["head"]["sha"]) merge_base = promote._git(repo, "merge-base", base, head) upstream_commits = _commits(repo, merge_base, head) upstream_patches = set() + comparisons[number] = [] for sha in upstream_commits: parents = promote._git(repo, "show", "-s", "--format=%P", sha).split() if len(parents) == 1: - upstream_patches.add(_patch(repo, parents[0], sha)) - aggregate = _patch(repo, merge_base, head) + patch = _patch(repo, parents[0], sha, source=source, include=include) + upstream_patches.add(patch) + if patch: + comparisons[number].append(("similar-edits", parents[0], sha, patch)) + aggregate = _patch(repo, merge_base, head, source=source, include=include) + if aggregate: + comparisons[number].append(("similar-upstream-aggregate", merge_base, head, aggregate)) for sha in commits: + if not patches[sha]: + continue if sha in upstream_commits: candidates[sha].add(number) methods[sha, number] = "same-commit" @@ -202,13 +298,43 @@ def match( methods[sha, number] = "upstream-aggregate" # A whole reviewed series may be squashed into one upstream PR. Do not # invent individual correspondences: identify the shared group explicitly. - if aggregate and _patch(repo, previous, reviewed) == aggregate: + if aggregate and series == aggregate: for sha in commits: + if not patches[sha]: + continue candidates[sha].add(number) methods.setdefault((sha, number), "source-series-aggregate") + # Exact matches take priority across all listed PRs. Approximate attribution + # is considered only when none exists, and is not permission to drop changes. + similarities: dict[str, dict[int, dict]] = {} + for sha in commits: + if not patches[sha] or sha in resolutions or candidates[sha]: + continue + similarities[sha] = {} + for number, choices in comparisons.items(): + best = {"similarity": 0.0} + for method, base, head, patch in choices: + score = _similarity(patches[sha], patch) + if score > best["similarity"]: + best = { + "method": method, + "similarity": score, + "comparison": { + "source_base": source_parents[sha], + "source_head": sha, + "upstream_base": base, + "upstream_head": head, + }, + } + similarities[sha][number] = best + if best["similarity"] >= SIMILARITY_THRESHOLD: + candidates[sha].add(number) assignments, evidence, unresolved = {}, {}, [] for sha in commits: - if sha in resolutions: + if not patches[sha]: + assignments[sha] = None + evidence[sha] = {"method": "outside-vendor-scope", "refresh_policy": "retain"} + elif sha in resolutions: resolution = resolutions[sha] number = resolution["upstream_pr"] if number not in upstream_prs: @@ -223,25 +349,54 @@ def match( number = next(iter(candidates[sha])) assignments[sha] = number evidence[sha] = { - "method": methods[sha, number], "upstream_head": upstream_prs[number]["head"]["sha"], "upstream_base": upstream_prs[number]["base"]["sha"], } + if sha in similarities: + evidence[sha].update( + similarities[sha][number], + upstream_pr=number, + similarity_threshold=SIMILARITY_THRESHOLD, + similarity_metric=_SIMILARITY_METRIC, + refresh_policy="retain-until-verified", + ) + else: + evidence[sha]["method"] = methods[sha, number] else: options = ( ", ".join(f"#{number}" for number in sorted(candidates[sha])) or "no exact match" ) - unresolved.append(f"- `{sha}`: {options}") + detail = f"- `{sha}`: {options}" + if sha in similarities: + closest = sorted( + similarities[sha].items(), key=lambda item: (-item[1]["similarity"], item[0]) + )[:3] + scores = ( + ", ".join( + f"{_upstream_reference(number, upstream_repo)}: {item['similarity']:.2%}" + for number, item in closest + ) + or "no upstream candidates" + ) + outcome = "multiple PRs meet" if candidates[sha] else "no PR meets" + detail += ( + f"; best similarities: {scores}; {outcome} the " + f"{SIMILARITY_THRESHOLD:.2%} similarity threshold" + ) + unresolved.append(detail) + accepted = similarity_feedback(evidence, upstream_repo) if unresolved: raise ValueError( "Author action required: upstream attribution is unresolved. Add missing upstream PRs, " "update/rebase the source pin, or add a targeted resolution with an upstream_pr and " "reason for each commit below. No complete commit_map is required.\n" + "\n".join(unresolved) + + ("\n\n" + accepted if accepted else "") ) - if set(assignments.values()) != set(upstream_prs): + if set(assignments.values()) - {None} != set(upstream_prs): raise ValueError( - "Some listed upstream PRs cover no new source commits; remove unrelated PRs." + "Some listed upstream PRs cover no new lock-selected changes; remove unrelated PRs." + + ("\n\n" + accepted if accepted else "") ) return assignments, evidence @@ -252,8 +407,13 @@ def resolve( reviewed: manage.Vendor, entry: dict, cache: Path, -) -> tuple[dict[str, int], dict[str, dict]]: +) -> tuple[dict[str, int | None], dict[str, dict]]: """Fetch only Git data into a private bare cache and resolve declared PRs.""" + if previous.source != reviewed.source or set(previous.include) != set(reviewed.include): + raise ValueError( + "Automatic attribution requires unchanged lock source/include selection; " + "separate the selection change from the source update." + ) cache.mkdir(parents=True, exist_ok=True) if not (cache / "HEAD").exists(): promote._git(cache, "init", "--bare", "--quiet") @@ -299,4 +459,13 @@ def resolve( promote._sha(sha), ) prs[number] = pr - return match(cache, previous.commit, reviewed.commit, prs, entry["resolutions"]) + return match( + cache, + previous.commit, + reviewed.commit, + prs, + entry["resolutions"], + source=reviewed.source, + include=reviewed.include, + upstream_repo=gh.upstream_repo, + ) diff --git a/scripts/vendor/promote.py b/scripts/vendor/promote.py index 1f9b1c8e920c..cee4f4a7f02a 100644 --- a/scripts/vendor/promote.py +++ b/scripts/vendor/promote.py @@ -437,6 +437,7 @@ def _make_plan( } assignments, evidence = metadata.resolve(gh, previous, reviewed, entry, Path(directory)) record["attribution"] = evidence + record["attribution_scope"] = {"source": reviewed.source, "include": list(reviewed.include)} else: assignments = _map_commits(commits, prs, args.map_upstream, gh.upstream_repo) if prs else {} upstream_main = _sha( @@ -458,6 +459,26 @@ def _make_plan( "refresh_policy": "retain", } ) + outside_scope = [ + item["sha"] + for item in commits + if item["sha"] in assignments and assignments[item["sha"]] is None + ] + if outside_scope: + groups.append( + { + "commits": outside_scope, + "upstream_pr": None, + "upstream_base": None, + "upstream_head": None, + "upstream_merge": None, + "unpaired_reason": ( + "No changes to files selected by the vendor lock source/include. " + "No whole-commit upstream equivalence is claimed." + ), + "refresh_policy": "retain", + } + ) for upstream_number, upstream in prs.items(): groups.append( { @@ -568,6 +589,11 @@ def _adopt_record(plan: Promotion, message: str) -> None: if any(existing.get(key) != plan.record[key] for key in stable_keys): raise ValueError("Existing promotion metadata does not describe this source update.") _validate_snapshot(existing, plan.record["upstream_repository"]) + if "attribution_scope" in existing and existing["attribution_scope"] != { + "source": plan.reviewed.source, + "include": list(plan.reviewed.include), + }: + raise ValueError("Promotion attribution scope does not match the reviewed vendor lock.") if "changes" not in plan.record: # A verified operation identity can restore its entire durable snapshot. plan.record = existing @@ -596,6 +622,12 @@ def _adopt_record(plan: Promotion, message: str) -> None: def _validate_snapshot(record: dict, upstream_repo: str) -> None: _sha(record["upstream_base"]) _sha(record["upstream_observed_main"]) + if "attribution_scope" in record: + scope = record["attribution_scope"] + if not isinstance(scope, dict) or set(scope) != {"source", "include"}: + raise ValueError("Invalid promotion attribution scope.") + vendor._validate_relative_path(scope["source"], "Attribution source") + vendor._validate_include(scope["include"], record["vendor"]) commits = [] if not isinstance(record["changes"], list) or not record["changes"]: raise ValueError("Promotion metadata must contain source changes.") @@ -910,9 +942,10 @@ def _pr_body(plan: Promotion, message: str) -> str: unpaired = "" if any(group["upstream_pr"] is None for group in plan.record["changes"]): unpaired = ( - "No paired upstream PR exists for this source update. Its provenance " - "explicitly records it as unpaired, with `refresh_policy: retain`. A future " - "refresh must carry it forward until an upstream pairing or equivalence " + "Some source commits have no paired upstream PR. Their provenance " + "records an explicit reason and `refresh_policy: retain`, including commits " + "outside the lock-selected scope when applicable. A future refresh must " + "retain these commits until an upstream pairing or equivalence " "is explicitly established.\n\n" ) return ( diff --git a/tests/unittest/others/test_maintain_prims_ts.py b/tests/unittest/others/test_maintain_prims_ts.py index 2c16e2234fa5..09699b2dea65 100644 --- a/tests/unittest/others/test_maintain_prims_ts.py +++ b/tests/unittest/others/test_maintain_prims_ts.py @@ -494,6 +494,131 @@ def test_bot_attribution_survives_retry_and_squash_merge(world: World) -> None: assert completed.record == original.record +def test_scoped_attribution_preserves_excluded_tail_during_promotion_and_recovery( + world: World, +) -> None: + upstream = world.get(f"repos/{_UPSTREAM}/pulls/4829") + paired_head = world.reviewed + _write(world.source, "tests/test_kernel.py", "# downstream tests only\n") + world.reviewed = _commit(world.source, "excluded test changes") + assert ( + m.vendor.main( + [ + "--lock", + str(world.consumer / m._LOCK), + "pin", + _VENDOR, + "--commit", + world.reviewed, + "--repo", + str(world.source), + ] + ) + == 0 + ) + _git(world.consumer, "add", "-A") + _git(world.consumer, "commit", "-q", "--amend", "-s", "-m", "include tests-only source commit") + world.merged = _git(world.consumer, "rev-parse", "HEAD") + world.main = world.merged + world.overrides[f"repos/{_UPSTREAM}/pulls/4829"] = upstream + args = world.args(publish=True, auto_merge=True) + plan = m._make_plan(args, world, auto_match=True) + assert plan.record["attribution_scope"] == {"source": "prims_ts", "include": ["**/*.py"]} + assert plan.record["attribution"][world.reviewed] == { + "method": "outside-vendor-scope", + "refresh_policy": "retain", + } + unpaired, paired = plan.record["changes"] + assert unpaired["commits"] == [world.reviewed] + assert unpaired["upstream_pr"] is None + assert unpaired["refresh_policy"] == "retain" + assert paired["commits"] == [world.first, paired_head] + assert paired["upstream_pr"] == f"https://github.com/{_UPSTREAM}/pull/4829" + m._validate_snapshot(plan.record, _UPSTREAM) + m._publish(args, world, plan) + saved = world.prs[99]["auto_merge"] + assert m._record_from_message(saved["commit_message"]) == plan.record + assert "outside the lock-selected scope" in world.prs[99]["body"] + assert _git(world.canonical, "rev-parse", "trtllm-prims-ts-dev") == world.reviewed + assert not (args.worktree / "vendored/prims_ts/test_kernel.py").exists() + # Recovery must use the durable, lock-bound scope, not mutable PR metadata. + upstream["head"]["sha"] = "b" * 40 + recovered = m._plan_identity(args, world) + assert m._recover_plan(args, world, recovered) + assert recovered.record == plan.record + m._publish(args, world, recovered) + assert len(world.prs) == 1 + + +def test_similarity_provenance_survives_publication_and_recovery(world: World) -> None: + original_head = world.reviewed + additions = [f"VALUE_{index} = {index}\n" for index in range(16)] + _write(world.source, "prims_ts/kernel.py", "VALUE = 3\n" + "".join(additions)) + world.reviewed = _commit(world.source, "downstream source update") + _git(world.source, "checkout", "-q", "-b", "upstream-adapted", original_head) + _write( + world.source, "prims_ts/kernel.py", "VALUE = 3\n" + "".join(additions[:-1]) + "adapted\n" + ) + upstream_head = _commit(world.source, "upstream equivalent with adaptation") + _git(world.source, "checkout", "-q", "main") + upstream = world.get(f"repos/{_UPSTREAM}/pulls/4829") + upstream["head"]["sha"] = upstream_head + world.overrides[f"repos/{_UPSTREAM}/pulls/4829"] = upstream + _write(world.consumer, "vendored/prims_ts/kernel.py", "VALUE = 3\n" + "".join(additions)) + assert ( + m.vendor.main( + [ + "--lock", + str(world.consumer / m._LOCK), + "pin", + _VENDOR, + "--commit", + world.reviewed, + "--repo", + str(world.source), + ] + ) + == 0 + ) + _git(world.consumer, "add", "-A") + _git(world.consumer, "commit", "-q", "--amend", "-s", "-m", "merged source update") + world.merged = _git(world.consumer, "rev-parse", "HEAD") + world.main = world.merged + args = world.args(publish=True, auto_merge=True) + plan = m._make_plan(args, world, auto_match=True) + evidence = plan.record["attribution"][world.reviewed] + assert evidence["method"] == "similar-edits" + assert evidence["similarity"] == 0.95 + assert evidence["similarity_threshold"] == 0.9 + assert evidence["refresh_policy"] == "retain-until-verified" + assert evidence["comparison"]["upstream_head"] == upstream_head + m._publish(args, world, plan) + saved = world.prs[99]["auto_merge"] + assert m._record_from_message(saved["commit_message"]) == plan.record + upstream["head"]["sha"] = "b" * 40 + recovered = m._plan_identity(args, world) + assert m._recover_plan(args, world, recovered) + assert recovered.record == plan.record + m._publish(args, world, recovered) + assert len(world.prs) == 1 + + +@pytest.mark.parametrize( + "scope", + [ + {"source": "different", "include": ["**/*.py"]}, + {"source": "prims_ts", "include": ["*.py"]}, + {"source": "prims_ts"}, + ], +) +def test_recovery_rejects_mismatched_attribution_scope(world: World, scope: dict) -> None: + args = world.args() + record = m._make_plan(args, world).record + record["attribution_scope"] = scope + with pytest.raises(ValueError, match="attribution scope"): + m._adopt_record(m._plan_identity(args, world), m._record_text(record)) + + def test_non_main_consumer_and_upstream_branches(world: World) -> None: world.base_branch = "release/next" world.upstream_branch = "develop" @@ -852,7 +977,7 @@ def test_unpaired_promotion_publishes_and_resumes(world: World) -> None: m._publish(args, world, plan) m._publish(args, world, m._make_plan(args, world)) assert len(world.prs) == 1 - assert "No paired upstream PR" in world.prs[99]["body"] + assert "Some source commits have no paired upstream PR" in world.prs[99]["body"] assert not any(f"repos/{_UPSTREAM}/pulls/" in call[1] for call in world.calls) changed = copy.deepcopy(plan.record) changed["changes"][0]["refresh_policy"] = "drop" diff --git a/tests/unittest/others/test_vendor_bot.py b/tests/unittest/others/test_vendor_bot.py index a9ce2c1f6dbf..bff2d885370a 100644 --- a/tests/unittest/others/test_vendor_bot.py +++ b/tests/unittest/others/test_vendor_bot.py @@ -7,12 +7,15 @@ import argparse import contextlib import copy +import dataclasses import importlib import importlib.util import json +import logging import os import subprocess import sys +import threading import time from pathlib import Path @@ -39,15 +42,33 @@ def _body(content: str) -> str: return f"{m.BEGIN}\n```yaml\nschema_version: 1\nvendors:\n toy:\n{content}\n```\n{m.END}" -def test_metadata_only_needs_upstream_urls() -> None: +@pytest.mark.parametrize("line_ending", ["\n", "\r\n", "\r"], ids=["lf", "crlf", "cr"]) +def test_metadata_only_needs_upstream_urls(line_ending: str) -> None: entry = m.parse( - _body(" upstream_prs:\n - https://github.com/org/library/pull/42"), + _body(" upstream_prs:\n - https://github.com/org/library/pull/42").replace( + "\n", line_ending + ), "toy", "org/library", ) assert entry == {"upstream_prs": [42], "unpaired_reason": None, "resolutions": {}} +def test_metadata_mixed_line_endings() -> None: + body = _body(" upstream_prs:\n - https://github.com/org/library/pull/42") + mixed = body.replace("```yaml\n", "```yaml\r\n").replace("\n```\n", "\r```\r\n") + expected = m.parse(body, "toy", "org/library") + actual = m.parse(mixed, "toy", "org/library") + assert actual == expected + assert m.fingerprint(actual) == m.fingerprint(expected) + + +def test_metadata_size_limit_precedes_line_ending_normalization() -> None: + body = "\r\n" * 32768 + m.template("toy", "org/library") + with pytest.raises(ValueError, match="exceeds the supported metadata size"): + m.parse(body, "toy", "org/library") + + @pytest.mark.parametrize( "content", [ @@ -117,6 +138,7 @@ def source(tmp_path: Path) -> Path: def _commit(repo: Path, text: str, filename: str = "code.py") -> str: + (repo / filename).parent.mkdir(parents=True, exist_ok=True) (repo / filename).write_text(text) _git(repo, "add", filename) _git(repo, "commit", "-q", "-m", "test") @@ -149,6 +171,170 @@ def test_match_rebased_patch(source: Path) -> None: assert evidence[downstream]["method"] == "same-edits" +def test_match_only_lock_selected_files(source: Path) -> None: + base = _commit(source, "VALUE = 0\n", "library/kernels/code.py") + for name in ( + "tests/test_code.py", + "library/kernels/README.md", + "library/kernels_extra/code.py", + ): + path = source / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("downstream-only content\n") + _git(source, "add", "-A") + downstream = _commit(source, "VALUE = 1\n", "library/kernels/code.py") + _git(source, "checkout", "-q", "-b", "upstream", base) + upstream = _commit(source, "VALUE = 1\n", "library/kernels/code.py") + assignments, evidence = m.match( + source, + base, + downstream, + {42: _pr(base, upstream)}, + source="library/kernels", + include=("**/*.py",), + ) + assert assignments == {downstream: 42} + assert evidence[downstream]["method"] == "same-edits" + with pytest.raises(ValueError, match="no exact match"): + m.match(source, base, downstream, {42: _pr(base, upstream)}) + + +@pytest.mark.parametrize("include", [("*.py",), ("**/*.py",), ("nested/*.py",)]) +def test_patch_uses_lock_globs_and_literal_git_paths( + source: Path, include: tuple[str, ...] +) -> None: + base = _commit(source, "base\n", "outside.txt") + paths = ["root.py", "nested/kernel[1] *.py", "nested/README.md", "nested/deep/kernel.py"] + for name in paths: + path = source / "selected" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"{name}\n") + _git(source, "add", "-A") + head = _commit(source, "not selected\n", "selected-sibling/outside.py") + patch = m._patch(source, base, head, source="selected", include=include) + for name in paths: + assert (f"+{name}\n" in patch) == m.manage._matches(name, include) + assert "not selected" not in patch + assert m._patch(source, base, head, source="absent", include=include) == "" + + +@pytest.mark.parametrize("squash", [False, True]) +def test_excluded_commits_are_not_attributed_to_upstream(source: Path, squash: bool) -> None: + base = _commit(source, "VALUE = 0\n", "src/code.py") + before = _commit(source, "initial tests\n", "tests/test_code.py") + first = _commit(source, "VALUE = 1\n", "src/code.py") + second = _commit(source, "VALUE = 2\n", "src/code.py") + after = _commit(source, "more tests\n", "tests/test_code.py") + upstream = after + if squash: + _git(source, "checkout", "-q", "-b", "upstream", base) + upstream = _commit(source, "VALUE = 2\n", "src/code.py") + assignments, evidence = m.match( + source, + base, + after, + {42: _pr(base, upstream)}, + source="src", + include=("**/*.py",), + ) + assert assignments == {before: None, first: 42, second: 42, after: None} + for sha in (before, after): + assert evidence[sha] == {"method": "outside-vendor-scope", "refresh_policy": "retain"} + assert evidence[first]["method"] == ("source-series-aggregate" if squash else "same-commit") + + +def test_empty_selected_patches_never_match_an_upstream_pr(source: Path) -> None: + base = _commit(source, "VALUE = 0\n", "src/code.py") + head = _commit(source, "tests only\n", "tests/test_code.py") + assignments, _ = m.match(source, base, head, {}, source="src", include=("**/*.py",)) + assert assignments == {head: None} + with pytest.raises(ValueError, match="cover no new lock-selected changes"): + m.match(source, base, head, {42: _pr(base, head)}, source="src", include=("**/*.py",)) + with pytest.raises(ValueError, match="Remove resolutions"): + m.match( + source, + base, + head, + {42: _pr(base, head)}, + {head: {"upstream_pr": 42, "reason": "tests"}}, + source="src", + include=("**/*.py",), + ) + + +def test_selected_code_mismatch_is_not_hidden_by_identical_tests(source: Path) -> None: + base = _commit(source, "VALUE = 0\n", "src/code.py") + tests = _commit(source, "shared tests\n", "tests/test_code.py") + downstream = _commit(source, "VALUE = 1\n", "src/code.py") + _git(source, "checkout", "-q", "-b", "upstream", tests) + upstream = _commit(source, "VALUE = 2\n", "src/code.py") + with pytest.raises(ValueError, match="no exact match"): + m.match( + source, base, downstream, {42: _pr(base, upstream)}, source="src", include=("**/*.py",) + ) + + +@pytest.mark.parametrize("into_scope", [False, True]) +def test_patch_keeps_selected_side_of_move(source: Path, into_scope: bool) -> None: + selected, excluded = "src/code.py", "tests/test_code.py" + old, new = (excluded, selected) if into_scope else (selected, excluded) + base = _commit(source, "VALUE = 0\n", old) + (source / new).parent.mkdir(parents=True, exist_ok=True) + _git(source, "mv", old, new) + _git(source, "commit", "-q", "-m", "move") + head = _git(source, "rev-parse", "HEAD") + patch = m._patch(source, base, head, source="src", include=("**/*.py",)) + assert selected in patch + assert excluded not in patch + assert ("+VALUE = 0" if into_scope else "-VALUE = 0") in patch + assert ("new file mode" if into_scope else "deleted file mode") in patch + + +@pytest.mark.parametrize("change", [{"source": "different"}, {"include": ("*.py",)}]) +def test_resolve_rejects_selection_changes_before_fetch(tmp_path: Path, change: dict) -> None: + previous = _vendor("a" * 40, "https://github.com/operator/library.git", "dev") + reviewed = dataclasses.replace(previous, commit="b" * 40, **change) + cache = tmp_path / "cache" + with pytest.raises(ValueError, match="unchanged lock source/include"): + m.resolve(None, previous, reviewed, {}, cache) + assert not cache.exists() + + +@pytest.mark.parametrize( + ("original", "rebased"), + [ + ( + "def run():\n before = 0\n value = 0\n after = 0\n return value\n", + "def run():\n before = 2\n value = 0\n after = 3\n return value\n", + ), + ( + "def old_helper():\n pass\n\n\n@cache\ndef run():\n value = 0\n", + "def new_helper():\n pass\n\n\n@cache\ndef run():\n value = 0\n", + ), + ( + "def run():\n value = 0\n # gap\n second = 0\n", + "def run():\n value = 0\n" + " # gap\n" * 10 + " second = 0\n", + ), + ], + ids=["neighboring-lines", "neighboring-function", "hunk-spacing"], +) +def test_match_ignores_unchanged_context(source: Path, original: str, rebased: str) -> None: + # Repo-local diff settings must not reintroduce unchanged context. + _git(source, "config", "diff.interHunkContext", "20") + base = _commit(source, original) + downstream = _commit( + source, original.replace("value = 0", "value = 1").replace("second = 0", "second = 1") + ) + _git(source, "checkout", "-q", "-b", "upstream", base) + context = _commit(source, rebased) + upstream = _commit( + source, rebased.replace("value = 0", "value = 1").replace("second = 0", "second = 1") + ) + assignments, evidence = m.match(source, base, downstream, {42: _pr(context, upstream)}) + assert assignments == {downstream: 42} + assert evidence[downstream]["method"] == "same-edits" + + @pytest.mark.parametrize("different", [" VALUE = 1\n", "VALUE = 1 \n", "VALUE = 1\r\n"]) def test_matching_preserves_whitespace(source: Path, different: str) -> None: base = _commit(source, "VALUE = 0\n") @@ -159,7 +345,22 @@ def test_matching_preserves_whitespace(source: Path, different: str) -> None: m.match(source, base, downstream, {42: _pr(base, upstream)}) -def test_identical_edits_in_different_functions_do_not_match(source: Path) -> None: +def test_decorator_insertion_ignores_preceding_function_label(source: Path) -> None: + original = "def old_helper():\n pass\n\n\ndef run():\n return 1\n" + rebased = original.replace("old_helper", "new_helper") + base = _commit(source, original) + downstream = _commit(source, original.replace("def run():", "@cache\ndef run():")) + _git(source, "checkout", "-q", "-b", "upstream", base) + context = _commit(source, rebased) + upstream = _commit(source, rebased.replace("def run():", "@cache\ndef run():")) + assert "@@ def old_helper():" in _git(source, "diff", "--unified=0", base, downstream) + assert "@@ def new_helper():" in _git(source, "diff", "--unified=0", context, upstream) + assignments, evidence = m.match(source, base, downstream, {42: _pr(context, upstream)}) + assert assignments == {downstream: 42} + assert evidence[downstream]["method"] == "same-edits" + + +def test_hunk_labels_are_not_part_of_edit_identity(source: Path) -> None: original = "def first():\n return False\n\n\ndef second():\n return False\n" base = _commit(source, original) downstream = _commit(source, original.replace("return False", "return True", 1)) @@ -168,7 +369,30 @@ def test_identical_edits_in_different_functions_do_not_match(source: Path) -> No source, original.replace("def second():\n return False", "def second():\n return True"), ) - with pytest.raises(ValueError, match="Author action required"): + assignments, evidence = m.match(source, base, downstream, {42: _pr(base, upstream)}) + assert assignments == {downstream: 42} + assert evidence[downstream]["method"] == "same-edits" + with pytest.raises(ValueError, match="#41, #42"): + m.match(source, base, downstream, {41: _pr(base, downstream), 42: _pr(base, upstream)}) + + +@pytest.mark.parametrize("added", [True, False], ids=["added", "removed"]) +def test_changed_function_definitions_are_not_ignored(source: Path, added: bool) -> None: + original = "def first():\n pass\n\n\ndef second():\n pass\n" + base = _commit(source, "# base\n" if added else original) + downstream = _commit(source, "def first():\n pass\n") + _git(source, "checkout", "-q", "-b", "upstream", base) + upstream = _commit(source, "def second():\n pass\n") + with pytest.raises(ValueError, match="no exact match"): + m.match(source, base, downstream, {42: _pr(base, upstream)}) + + +def test_hunk_like_changed_text_is_not_ignored(source: Path) -> None: + base = _commit(source, "# base\n", "patch.txt") + downstream = _commit(source, "@@ -1 +1 @@ def first():\n", "patch.txt") + _git(source, "checkout", "-q", "-b", "upstream", base) + upstream = _commit(source, "@@ -1 +1 @@ def second():\n", "patch.txt") + with pytest.raises(ValueError, match="no exact match"): m.match(source, base, downstream, {42: _pr(base, upstream)}) @@ -203,6 +427,112 @@ def test_author_can_resolve_adapted_commit(source: Path) -> None: m.match(source, base, downstream, {42: _pr(base, upstream)}, {base: {"upstream_pr": 42}}) +@pytest.mark.parametrize("different", ["a \n", "a\r\n", "a\v\n"]) +def test_similarity_preserves_line_bytes(different: str) -> None: + assert m._similarity("a\n", different) == 0 + assert m._similarity("", "") == 0 + + +def test_similarity_keeps_repeated_lines() -> None: + first = "different\n" + "repeat\n" * 299 + second = "another\n" + "repeat\n" * 299 + assert m._similarity(first, second) == pytest.approx(299 / 300) + + +@pytest.mark.parametrize("changed_lines", [1, 2, 3]) +def test_similarity_threshold_and_provenance(source: Path, changed_lines: int) -> None: + base = _commit(source, "") + lines = [f"VALUE_{index} = {index}\n" for index in range(16)] + downstream = _commit(source, "".join(lines)) + _git(source, "checkout", "-q", "-b", "upstream", base) + upstream = _commit(source, "".join(lines[:-changed_lines]) + "adapted\n" * changed_lines) + expected = (20 - changed_lines) / 20 + if expected < 0.9: + with pytest.raises(ValueError, match="no exact match") as failure: + m.match(source, base, downstream, {42: _pr(base, upstream)}) + assert "upstream PR #42: 85.00%" in str(failure.value) + assert "no PR meets the 90.00% similarity threshold" in str(failure.value) + return + assignments, evidence = m.match(source, base, downstream, {42: _pr(base, upstream)}) + assert assignments == {downstream: 42} + assert evidence[downstream] == { + "method": "similar-edits", + "upstream_base": base, + "upstream_head": upstream, + "upstream_pr": 42, + "similarity": expected, + "similarity_threshold": 0.9, + "similarity_metric": "sequence-matcher-lines-v1", + "refresh_policy": "retain-until-verified", + "comparison": { + "source_base": base, + "source_head": downstream, + "upstream_base": base, + "upstream_head": upstream, + }, + } + assert f"**{expected:.2%}** — accepted" in m.similarity_feedback(evidence) + + +def test_multiple_similar_prs_remain_ambiguous(source: Path) -> None: + base = _commit(source, "") + lines = [f"VALUE_{index} = {index}\n" for index in range(16)] + downstream = _commit(source, "".join(lines)) + _git(source, "checkout", "-q", "-b", "upstream", base) + upstream = _commit(source, "".join(lines[:-1]) + "adapted\n") + with pytest.raises(ValueError, match="#41, #42") as failure: + m.match(source, base, downstream, {41: _pr(base, upstream), 42: _pr(base, upstream)}) + assert "#41: 95.00%" in str(failure.value) + assert "#42: 95.00%" in str(failure.value) + assert "multiple PRs meet the 90.00% similarity threshold" in str(failure.value) + + +def test_exact_matches_take_priority_over_similar_prs(source: Path) -> None: + base = _commit(source, "") + lines = [f"VALUE_{index} = {index}\n" for index in range(16)] + first = _commit(source, "".join(lines)) + second = _commit(source, "second file\n", "second.py") + _git(source, "checkout", "-q", "-b", "upstream", base) + approximate = _commit(source, "".join(lines[:-1]) + "adapted\n") + upstream = _commit(source, "second file\n", "second.py") + assignments, evidence = m.match( + source, base, second, {41: _pr(base, first), 42: _pr(base, upstream)} + ) + assert m._similarity(m._patch(source, base, first), m._patch(source, base, approximate)) == 0.95 + assert assignments == {first: 41, second: 42} + assert all("similarity" not in item for item in evidence.values()) + + +def test_approximate_series_cannot_hide_unmatched_individual_commits(source: Path) -> None: + base = _commit(source, "", "src/code.py") + lines = [f"VALUE_{index} = {index}\n" for index in range(16)] + first = _commit(source, "".join(lines[:8]), "src/code.py") + second = _commit(source, "".join(lines), "src/code.py") + excluded = _commit(source, "tests only\n", "tests/test_code.py") + _git(source, "checkout", "-q", "-b", "upstream", base) + upstream = _commit(source, "".join(lines[:-1]) + "adapted\n", "src/code.py") + assert m._similarity(m._patch(source, base, second), m._patch(source, base, upstream)) == 0.95 + with pytest.raises(ValueError, match="no exact match") as failure: + m.match( + source, base, excluded, {42: _pr(base, upstream)}, source="src", include=("**/*.py",) + ) + assert first in str(failure.value) and second in str(failure.value) + assert excluded not in str(failure.value) + + +def test_similarity_error_also_reports_accepted_matches(source: Path) -> None: + base = _commit(source, "") + lines = [f"VALUE_{index} = {index}\n" for index in range(16)] + first = _commit(source, "".join(lines)) + second = _commit(source, "unrelated\n" * 16, "second.py") + _git(source, "checkout", "-q", "-b", "upstream", base) + upstream = _commit(source, "".join(lines[:-1]) + "adapted\n") + with pytest.raises(ValueError, match="no exact match") as failure: + m.match(source, base, second, {42: _pr(base, upstream)}) + assert f"`{second}`: no exact match" in str(failure.value) + assert f"`{first}`: upstream PR #42, similarity **95.00%** — accepted" in str(failure.value) + + class FakeGitHub(p.GitHub): def __init__(self) -> None: super().__init__("org/consumer", "toy", "org/library") @@ -269,6 +599,7 @@ def api( @pytest.fixture def monitor(tmp_path: Path): + b._STOP.clear() args = argparse.Namespace( canonical_repo="operator/library", canonical_branch="dev", @@ -283,6 +614,7 @@ def monitor(tmp_path: Path): instance = b.Monitor(args, gh, store) yield instance store.close() + b._STOP.clear() def test_feedback_requests_changes_once_and_clears_only_metadata_review(monitor: b.Monitor) -> None: @@ -371,6 +703,53 @@ def filtered(endpoint: str, **kwargs): assert not gh.writes +@pytest.mark.parametrize("changed_lines", [1, 3], ids=["accepted", "below-threshold"]) +def test_monitor_posts_similarity_and_deduplicates_feedback( + source: Path, monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch, changed_lines: int +) -> None: + base = _commit(source, "", "src/code.py") + lines = [f"VALUE_{index} = {index}\n" for index in range(16)] + downstream = _commit(source, "".join(lines), "src/code.py") + _git(source, "checkout", "-q", "-b", "upstream", base) + upstream = _commit( + source, "".join(lines[:-changed_lines]) + "adapted\n" * changed_lines, "src/code.py" + ) + previous = _vendor(base, "https://github.com/operator/library.git", "dev") + reviewed = _vendor(downstream, "https://github.com/author/library.git", "temporary") + monkeypatch.setattr(monitor, "_inputs", lambda pr: (previous, reviewed)) + monitor.gh.pr["body"] = _body(" upstream_prs: [https://github.com/org/library/pull/42]") + monkeypatch.setattr( + m, + "resolve", + lambda *args: m.match( + source, + base, + downstream, + {42: _pr(base, upstream)}, + source=reviewed.source, + include=reviewed.include, + upstream_repo="org/library", + ), + ) + monitor.inspect(7) + body = monitor.gh.comments[0]["body"] + assert "https://github.com/org/library/pull/42" in body + assert downstream in body and "90.00%" in body + if changed_lines == 1: + assert "95.00%" in body and "accepted" in body + assert "not** a code-review approval" in body + assert not monitor.gh.reviews + else: + assert "85.00%" in body and "no PR meets" in body + assert monitor.gh.reviews[0]["state"] == "CHANGES_REQUESTED" + writes = copy.deepcopy(monitor.gh.writes) + with contextlib.closing(b.Store(":memory:")) as store: + restarted = b.Monitor(monitor.args, monitor.gh, store) + monkeypatch.setattr(restarted, "_inputs", lambda pr: (previous, reviewed)) + restarted.inspect(7) + assert monitor.gh.writes == writes + + def test_feedback_deduplicates_with_empty_sqlite(monitor: b.Monitor) -> None: monitor._feedback(monitor.gh.pr, "Missing block") writes = copy.deepcopy(monitor.gh.writes) @@ -413,9 +792,10 @@ def resolve(*args): matches.append(True) return {reviewed.commit: 42}, {reviewed.commit: {"method": "same-edits"}} - def make_plan(args, github): + def make_plan(args, github, *, auto_match, resolutions): assert args.auto_merge and not args.wait - assert args.map_upstream == [f"{reviewed.commit}=42"] + assert args.map_upstream == [] + assert auto_match and resolutions == {} return p.Promotion( 7, "c" * 40, @@ -423,7 +803,7 @@ def make_plan(args, github): reviewed, "operator/library", "operator/consumer", - {}, + {"attribution": {reviewed.commit: {"method": "same-edits"}}}, consumer_repo="org/consumer", ) @@ -625,7 +1005,22 @@ def test_daemon_launch_status_stop_without_systemd(tmp_path: Path) -> None: text=True, check=True, ) - assert json.loads(status.stdout)["running"] + state = json.loads(status.stdout) + assert state["running"] + assert state["pid"] is not None + assert state["publish"] is False + assert state["vendor"] == "toy" + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + state = b._status(workdir) + if state.get("next_poll_at"): + break + time.sleep(0.05) + assert state["last_poll_summary"]["outcome"] == "ok" + assert state["last_successful_poll"] == state["last_completed_poll"] + assert state["operation"]["phase"] == "waiting" + assert state["next_poll_at"] + assert "Poll finished" in (workdir / "bot.log").read_text() assert "private-auth" not in (workdir / "bot.log").read_text() finally: subprocess.run( @@ -635,3 +1030,387 @@ def test_daemon_launch_status_stop_without_systemd(tmp_path: Path) -> None: while b._running(workdir) and time.monotonic() < deadline: time.sleep(0.1) assert not b._running(workdir) + assert b._status(workdir)["pid"] is None + assert b._status(workdir)["next_poll_at"] is None + + +def test_log_level_defaults_to_info_and_accepts_lowercase(tmp_path: Path) -> None: + command = [ + "run", + "--workdir", + str(tmp_path), + "--vendor", + "toy", + "--upstream-repo", + "org/library", + "--canonical-repo", + "operator/library", + "--canonical-branch", + "dev", + "--fork", + "operator/consumer", + ] + assert b._parser().parse_args(command).log_level == "INFO" + assert b._parser().parse_args(command + ["--log-level", "debug"]).log_level == "DEBUG" + + +@pytest.mark.parametrize("publish", [False, True]) +def test_metadata_transitions_are_logged_once_and_clear_problem( + monitor: b.Monitor, + caplog: pytest.LogCaptureFixture, + publish: bool, +) -> None: + caplog.set_level(logging.INFO, logger="vendor-bot") + monitor.args.publish = publish + monitor._feedback(monitor.gh.pr, "Missing marked block") + monitor._feedback(monitor.gh.pr, "Missing marked block") + assert caplog.text.count("PR #7: blocked: Missing marked block") == 1 + assert monitor.store.get("problem:7") == "Missing marked block" + monitor._feedback(monitor.gh.pr, None) + monitor._feedback(monitor.gh.pr, None) + assert caplog.text.count("PR #7: metadata_valid") == 1 + assert monitor.store.get("problem:7") is None + if publish: + assert "posted metadata feedback comment" in caplog.text + assert "requested metadata changes" in caplog.text + assert "dismissed resolved metadata review" in caplog.text + else: + assert not monitor.gh.writes + + +def test_idle_poll_has_summary_and_success_timestamp( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + caplog.set_level(logging.INFO, logger="vendor-bot") + monkeypatch.setattr(monitor, "_discover", lambda: []) + monitor.poll() + health = monitor.store.get("health") + assert health["last_successful_poll"] == health["last_completed_poll"] + assert health["last_completed_poll"] == monitor.store.get("last_poll") + assert health["last_poll_summary"]["outcome"] == "ok" + assert health["last_poll_summary"]["processed"] == 0 + assert health["last_poll_summary"]["errors"] == 0 + assert "Poll started" in caplog.text and "Poll finished" in caplog.text + + +def test_pr_failure_is_not_a_successful_poll_and_recovers( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(monitor, "_discover", lambda: [7]) + + def fail(number: int) -> None: + raise RuntimeError("HTTP 429: rate limit") + + monkeypatch.setattr(monitor, "inspect", fail) + monitor.poll() + failed = monitor.store.get("health") + assert failed["last_completed_poll"] + assert failed["last_successful_poll"] is None + assert failed["last_poll_summary"]["outcome"] == "needs_attention" + assert failed["last_poll_summary"]["errors"] == 1 + assert failed["last_poll_summary"]["processed"] == 1 + assert failed["last_error"]["pr"] == 7 + assert "HTTP 429" in failed["last_error"]["message"] + monkeypatch.setattr( + monitor, "inspect", lambda number: monitor._transition(number, "metadata_valid") + ) + monitor.poll() + recovered = monitor.store.get("health") + assert recovered["last_successful_poll"] == recovered["last_completed_poll"] + assert recovered["last_poll_summary"]["outcome"] == "ok" + assert recovered["last_error"] == failed["last_error"] # Historical, not an active error. + assert monitor.store.get("problem:7") is None + + +def test_author_blocked_poll_is_not_reported_healthy( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, +) -> None: + previous = _vendor("a" * 40, "https://github.com/operator/library.git", "dev") + reviewed = _vendor("b" * 40, "https://github.com/author/library.git", "temporary") + monkeypatch.setattr(monitor, "_inputs", lambda pr: (previous, reviewed)) + monitor.poll() + health = monitor.store.get("health") + assert health["last_poll_summary"]["outcome"] == "needs_attention" + assert health["counts"]["blocked"] == 1 + assert health["counts"]["errors"] == 0 + assert health["counts"]["relevant"] == 1 + assert health["last_successful_poll"] is None + + +@pytest.mark.parametrize("error_type", [RuntimeError, KeyError]) +def test_discovery_failure_never_advances_completion( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, + error_type: type[Exception], +) -> None: + monitor.progress.update(last_completed_poll="old", last_successful_poll="old") + + def fail() -> list[int]: + raise error_type("discovery failed") + + monkeypatch.setattr(monitor, "_discover", fail) + with pytest.raises(error_type): + monitor.poll() + health = monitor.store.get("health") + assert health["last_completed_poll"] == health["last_successful_poll"] == "old" + assert health["last_poll_summary"]["outcome"] == "failed" + + +def test_stopping_discovery_does_not_advance_cursor_or_success( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, +) -> None: + original = monitor.gh.api + + def stop(endpoint: str, **kwargs): + if "/pulls?" in endpoint: + b._STOP.set() + return original(endpoint, **kwargs) + + monkeypatch.setattr(monitor.gh, "api", stop) + monitor.poll() + assert monitor.store.get("cursor") is None + assert monitor.store.get("bootstrapped") is None + health = monitor.store.get("health") + assert health["last_completed_poll"] is None + assert health["last_successful_poll"] is None + assert health["last_poll_summary"]["outcome"] == "interrupted" + + +def test_heartbeat_during_blocked_discovery_does_not_fake_progress( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + caplog.set_level(logging.INFO, logger="vendor-bot") + monkeypatch.setattr(b, "_HEARTBEAT_SECONDS", 0.01) + reported = threading.Event() + original = monitor.progress.heartbeat + + def heartbeat() -> None: + original() + reported.set() + + def wait_for_heartbeat(endpoint: str, **kwargs): + before = monitor.store.get("health") + assert reported.wait(2), "No heartbeat during blocked API call" + assert monitor.store.get("health") == before + return [] + + monkeypatch.setattr(monitor.progress, "heartbeat", heartbeat) + monkeypatch.setattr(monitor.gh, "api", wait_for_heartbeat) + with monitor.progress.heartbeats(): + monitor.poll() + assert "Heartbeat: phase=discover_open" in caplog.text + assert "operation completion not yet confirmed" in caplog.text + assert "Discovery completed" in caplog.text + assert not any(t.name == "vendor-bot-heartbeat" for t in threading.enumerate()) + + +def test_progress_checkpoint_and_restart_keep_history(monitor: b.Monitor) -> None: + progress = monitor.progress + progress.operation("discover_history") + progress.last_checkpoint = float("-inf") + progress.count("scanned", 123) + assert monitor.store.get("health")["counts"]["scanned"] == 123 + progress.error("previous error", 7) + progress.update(last_successful_poll="old", next_poll_at="stale") + restarted = b.Progress(monitor.store) + assert restarted.state["last_successful_poll"] == "old" + assert restarted.state["last_error"]["pr"] == 7 + assert restarted.state["next_poll_at"] is None + assert restarted.state["operation"] is None + + +def test_completed_promotion_clears_stale_problem_without_remote_writes(monitor: b.Monitor) -> None: + monitor.store.put("problem:7", "interrupted after completion checkpoint") + monitor.store.put("complete:7", "https://github.com/org/consumer/pull/8") + monitor.inspect(7) + assert monitor.store.get("problem:7") is None + assert monitor.store.get("observation:7")["state"] == "completed" + assert not monitor.gh.writes + + +@pytest.mark.parametrize("status", ["added", "removed", "modified"]) +def test_lock_creation_removal_is_not_a_promotion_but_other_404s_fail( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, + status: str, +) -> None: + original = monitor.gh.api + + def files(endpoint: str, **kwargs): + if "/files?" in endpoint: + return [{"filename": p._LOCK, "status": status}] + return original(endpoint, **kwargs) + + def missing(*args): + raise RuntimeError("gh failed: gh: Not Found (HTTP 404)") + + monkeypatch.setattr(monitor.gh, "api", files) + monkeypatch.setattr(p, "_main_sha", lambda gh: "a" * 40) + monkeypatch.setattr(p, "_lock_document_at", missing) + if status == "modified": + with pytest.raises(RuntimeError, match="HTTP 404"): + monitor._inputs(monitor.gh.pr) + else: + monitor.inspect(7) + assert monitor.store.get("observation:7")["state"] == "ignored" + assert monitor.store.get("problem:7") is None + assert not monitor.gh.writes + + +def test_status_is_read_only_and_hides_stale_process_data(tmp_path: Path) -> None: + workdir = tmp_path / "missing" + assert b._status(workdir)["running"] is False + assert not workdir.exists() + workdir.mkdir() + with contextlib.closing(b.Store(workdir / b._STATE_FILE)) as store: + store.put("process", {"pid": 123, "publish": True}) + store.put("settings", {"vendor": "toy", "actor": "operator"}) + progress = b.Progress(store) + progress.operation("publishing_promotion", 7) + progress.update(next_poll_at="future", last_completed_poll="previous") + progress.update(running=False, unexpected="do not expose unrelated cached fields") + before = (workdir / b._STATE_FILE).read_bytes() + with b._locked(workdir): + state = b._status(workdir) + assert state["running"] and state["pid"] == 123 + assert state["publish"] and state["vendor"] == "toy" + assert state["operation"]["pr"] == 7 + assert "unexpected" not in state + (workdir / "stop").touch() + assert b._status(workdir)["stop_requested"] + state = b._status(workdir) + assert state["running"] is False and state["pid"] is None + assert state["operation"] is None and state["next_poll_at"] is None + assert state["last_completed_poll"] == "previous" + assert (workdir / b._STATE_FILE).read_bytes() == before + + +@pytest.mark.parametrize("malformed", [False, True]) +def test_status_reports_unreadable_cache_without_losing_liveness( + tmp_path: Path, malformed: bool +) -> None: + database = tmp_path / b._STATE_FILE + if malformed: + with contextlib.closing(b.Store(database)) as store: + store.put("health", []) + else: + database.write_text("not SQLite") + before = database.read_bytes() + with b._locked(tmp_path): + result = b._status(tmp_path) + assert result["running"] and not result["state_available"] + assert result["state_error"] + assert database.read_bytes() == before + + +def test_log_and_status_errors_redact_secrets_and_yaml_excerpts( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + caplog.set_level(logging.INFO, logger="vendor-bot") + monkeypatch.setenv("GH_TOKEN", "private-environment-token") + detail = ( + "fetch failed https://user:password@example.invalid/repo " + "private-environment-token ghp_123456789 github_pat_abcdef\n" + "PR description text must not be logged" + ) + monitor.progress.error(detail, 7) + monitor._transition(7, "blocked", detail) + diagnostic = json.dumps(monitor.store.get("health")) + caplog.text + for secret in ( + "user:password", + "private-environment-token", + "ghp_123456789", + "github_pat_abcdef", + "PR description text", + ): + assert secret not in diagnostic + assert "fetch failed" in diagnostic + assert b._summary("Authorization: Bearer secret") == "Authorization: [REDACTED]" + + +def test_serve_reports_backoff_and_clears_process_on_exit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + caplog.set_level(logging.INFO, logger="vendor-bot") + b._STOP.clear() + args = argparse.Namespace( + workdir=tmp_path, + publish=False, + consumer_repo="org/consumer", + vendor="toy", + upstream_repo="org/library", + base_branch="main", + upstream_branch="main", + interval=30, + once=False, + ) + + class BrokenMonitor: + actor = "operator" + + def __init__(self, args, gh, store, progress): + self.progress = progress + + def poll(self): + self.progress.error("HTTP 503") + raise RuntimeError("HTTP 503") + + def finish_wait(seconds: float) -> bool: + state = b._status(tmp_path) + assert state["running"] + assert state["operation"]["phase"] == "waiting" + assert state["next_poll_at"] + assert state["last_error"]["message"] == "HTTP 503" + b._STOP.set() + return True + + monkeypatch.setattr(b, "Monitor", BrokenMonitor) + monkeypatch.setattr(b._STOP, "wait", finish_wait) + try: + b._serve(args) + finally: + b._STOP.clear() + assert "in 60s; consecutive_poll_failures=1" in caplog.text + assert b._status(tmp_path)["pid"] is None + assert b._status(tmp_path)["next_poll_at"] is None + + +def test_startup_error_is_visible_after_process_exits( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + args = argparse.Namespace( + workdir=tmp_path, + publish=False, + consumer_repo="org/consumer", + vendor="toy", + upstream_repo="org/library", + base_branch="main", + upstream_branch="main", + interval=30, + once=True, + ) + + def unauthorized(self, endpoint: str) -> dict: + raise RuntimeError("HTTP 401: authentication required") + + monkeypatch.setattr(p.GitHub, "get", unauthorized) + with pytest.raises(RuntimeError, match="HTTP 401"): + b._serve(args) + state = b._status(tmp_path) + assert not state["running"] and state["pid"] is None + assert state["last_error"]["message"] == "HTTP 401: authentication required" + assert state["last_completed_poll"] is None + assert state["next_poll_at"] is None + assert not any(t.name == "vendor-bot-heartbeat" for t in threading.enumerate()) From 30c9f1596aff409580314344825bfcb7f4db69a6 Mon Sep 17 00:00:00 2001 From: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> Date: Mon, 12 Oct 2026 10:24:08 +0000 Subject: [PATCH 06/11] [None][feat] Support vendor bot branch migration and actionable feedback Signed-off-by: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> --- AGENTS.md | 6 + scripts/vendor/README.md | 73 +- scripts/vendor/bot.py | 236 ++++++- scripts/vendor/diagnostics.py | 61 ++ scripts/vendor/metadata.py | 421 ++++++++++-- scripts/vendor/promote.py | 32 +- .../unittest/others/test_maintain_prims_ts.py | 32 +- tests/unittest/others/test_vendor_bot.py | 639 +++++++++++++++++- 8 files changed, 1374 insertions(+), 126 deletions(-) create mode 100644 scripts/vendor/diagnostics.py diff --git a/AGENTS.md b/AGENTS.md index 88ae8e124655..b9fd33e230f3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -211,6 +211,12 @@ only to verified lock-only promotions, not changes to these tools. Git/GitHub promotion commits are authoritative; SQLite is a disposable cache. After cache loss, stop the old process and relaunch with the same operator/config. Cold discovery backfills merged PRs; `--since` explicitly limits that history. +After a canonical-branch refresh merges, stop the bot and reuse its workdir with +the new `--canonical-branch` plus `--migrate-canonical-branch-from OLD_BRANCH`. +Start with `--once` without `--publish`; migration verifies the merged lock, +blocks unresolved old promotions, and atomically archives branch-dependent cache +results before rediscovery. Other identity changes still require another workdir. +After cache loss, use the new configuration without the migration flag. For monitoring, use INFO poll summaries/30-second active-operation heartbeats and `bot.py status --workdir ...`. Check the last poll outcome and next deadline; the process lock and heartbeat alone do not establish successful reconciliation. diff --git a/scripts/vendor/README.md b/scripts/vendor/README.md index e4021d8cb5d2..d070fefc6444 100644 --- a/scripts/vendor/README.md +++ b/scripts/vendor/README.md @@ -41,6 +41,9 @@ flowchart TD recover -->|pending recorded operation| verify recover -->|no record and pin still pending| metadata recover -->|completed promotion| merged + refresh[Separate upstream refresh PR and normal CI] -->|merged| migrate[Stop bot and verify explicit branch migration] + migrate --> rediscover[Reuse workdir and revalidate GitHub state] + rediscover --> rebase ``` Promotion preserves the source SHA, selected files, compatibility patch, and @@ -198,6 +201,43 @@ An existing workdir is bound to its operator and vendor configuration; use a different directory for another operator/vendor. Authentication is inherited from the launching environment and is not persisted in that state. +### Switching canonical branches after a refresh + +Keep the same workdir when a periodic source refresh introduces a new canonical +branch. Stop the publishing process for the cutover, and wait for the consumer +refresh PR to merge. Relaunch with the same configuration except for: + +```bash +--canonical-branch consumer-dev-YYYYMMDD \ +--migrate-canonical-branch-from consumer-dev \ +--workdir /path/to/private/vendor-bot-state \ +--once +``` + +First omit `--publish`: migration only changes local configuration/cache state; +this pass makes no remote writes. The bot verifies the merged base lock names +the new canonical repository/branch and its pin is reachable from that branch. +It refuses outstanding old-branch promotion PRs or unpublished promotion branches. +Finish those operations, or explicitly close superseded follow-up PRs after +maintainer review; migration never deletes branches or redirects old promotions. +Other operator/vendor identity changes still require a separate workdir. + +The configuration update and invalidation are one SQLite transaction. Old +branch-dependent results are archived, while logs, source caches, promotion +worktrees, tracked candidates, and the historical `--since` cutoff are retained. +Discovery restarts and rechecks candidates against Git/GitHub. Old successful +polls are cleared from active health status; `status` reports the canonical +repository/branch and the verified migration revision. Existing Git promotion +provenance is never rewritten. Repeating the same migration is a no-op and does +not discard subsequent discovery progress. + +After inspecting the dry-run results, restart with the new branch and +`--daemon --publish`, omitting `--migrate-canonical-branch-from`. Verify a completed +poll under the new configuration, then notify outstanding source-update PRs to +rebase their temporary source branches, update their consumer PRs and pins, and +regenerate vendor snapshots. If SQLite is lost, launch normally using the new +canonical configuration and the same cutoff; omit the migration flag. + The default interval is 120 seconds (`--interval`, minimum 30). A cold start discovers existing open PRs and backfills merged PRs from all history. This scan can be expensive; subsequent polls use a cached cursor and tracked candidates. @@ -296,10 +336,35 @@ local consumer checkout; no code or hooks from source forks are executed. ## Feedback and review safety -With `--publish`, invalid metadata produces an updatable comment containing the -specific error and a copyable marked block. Eligible open PRs also receive one -metadata-only `REQUEST_CHANGES` review. The bot deduplicates its feedback and -never submits `APPROVE` just because metadata is valid. +With `--publish`, recognized author-actionable problems produce an updatable +comment containing the specific problem, its location, and a **Fix** section. +Examples are included only when relevant, using the configured vendor and +upstream repository. Missing metadata gets a complete copyable marked block; +field errors get focused instructions; attribution failures get a correctly +nested resolution example. Existing entries for other vendors must be preserved. +Edit the **PR description**, not a comment, when correcting metadata. The bot +rechecks automatically on later polls; no manual recheck comment is required. + +| Problem | What the bot tells the author to fix | +|---------|-------------------------------------| +| Missing markers | Names exactly which of `` and `` is missing, and where to insert it. | +| Duplicate or reversed markers | Reports marker counts or ordering; asks the author to consolidate metadata into one block without losing vendor entries. | +| Malformed YAML or schema | Reports YAML line/column or the affected field path and the expected structure, without copying raw parser excerpts. YAML line 1 is the first line inside the fence. | +| Invalid upstream PR or resolution | Identifies the list entry, duplicated PR, source SHA, missing reason, or unlisted PR, and gives the correction. | +| Unresolved attribution | Shows candidate PRs and similarity scores where applicable; explains how to correct the PR list/source pin or supply a targeted author assertion. | +| Source-history or lock-selection problem | Explains the rebase or lock-file change and vendor regeneration needed. Rebase guidance identifies the canonical pin. Metadata edits alone cannot fix these failures. | + +`unpaired_reason` is appropriate only when there genuinely is no paired upstream +PR, not as a workaround for failed matching. The similarity threshold, matching +rules, and retained provenance are unchanged by diagnostic formatting. + +Eligible open PRs receive one metadata-only `REQUEST_CHANGES` review for an +author-actionable problem. Waiting for a preceding promotion is identified as +**maintainer action**, with a comment but no new change-request review. Unexpected +API, authentication, network, and local-cache failures stay in operator logs and +`status`; they do not post author-fix instructions or request changes. The bot +deduplicates feedback, including after SQLite loss, and never submits `APPROVE` +just because metadata is valid. After correction it dismisses only its own marked metadata reviews, when authorized. If dismissal is forbidden, an authorized human must dismiss the diff --git a/scripts/vendor/bot.py b/scripts/vendor/bot.py index f66da0682902..f3f858e4c29e 100644 --- a/scripts/vendor/bot.py +++ b/scripts/vendor/bot.py @@ -24,10 +24,12 @@ if __package__: from . import manage, metadata, promote + from .diagnostics import Diagnostic, field else: import manage import metadata import promote + from diagnostics import Diagnostic, field _LOG = logging.getLogger("vendor-bot") _STATE_FILE = "state.sqlite" @@ -75,6 +77,32 @@ def close(self) -> None: """Close SQLite before releasing the process lock.""" self.connection.close() + def migrate_branch(self, settings: dict, record: dict) -> None: + """Atomically archive derived results and start reconciliation for a new branch.""" + keys = ("cursor", "bootstrapped", "health", "last_poll") + prefixes = ("complete:", "promotion:", "observation:", "problem:") + rows = self.connection.execute("SELECT key, value FROM state").fetchall() + archived = { + key: json.loads(value) for key, value in rows if key in keys or key.startswith(prefixes) + } + history = self.get("branch_migrations", []) + history.append({**record, "previous_results": archived}) + with self.connection: + self.connection.executemany( + "DELETE FROM state WHERE key = ?", [(key,) for key in archived] + ) + self.connection.executemany( + "INSERT INTO state VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", + [ + (key, json.dumps(value, sort_keys=True)) + for key, value in ( + ("settings", settings), + ("branch_migrations", history), + ("migration", record), + ) + ], + ) + class Progress: """Main-thread status checkpoints with an independent, log-only heartbeat.""" @@ -192,10 +220,51 @@ def __init__( "repo": str(args.repo), } saved = store.get("settings") + migration_from = args.migrate_canonical_branch_from if saved is not None and saved != settings: - raise ValueError( - "Workdir belongs to a different operator/vendor configuration; choose another." + changed = { + key for key in saved.keys() | settings.keys() if saved.get(key) != settings.get(key) + } + if changed != {"canonical_branch"}: + raise ValueError( + "Workdir belongs to a different operator/vendor configuration; choose another." + ) + if migration_from != saved["canonical_branch"]: + raise ValueError( + "Canonical branch changed; stop the old process and restart with " + f"--migrate-canonical-branch-from {saved['canonical_branch']} after the refresh merges." + ) + record = self._verify_branch_migration(migration_from) + store.migrate_branch(settings, record) + self.progress.update( + last_completed_poll=None, + last_successful_poll=None, + last_poll_summary=None, + last_error=None, + next_poll_at=None, + ) + _LOG.info( + "Canonical branch migrated: %s -> %s; rediscovery required", + migration_from, + args.canonical_branch, ) + elif migration_from: + record = store.get("migration", {}) + if ( + saved == settings + and record.get("from") == migration_from + and record.get("to") == args.canonical_branch + ): + _LOG.info( + "Canonical branch migration already applied: %s -> %s", + migration_from, + args.canonical_branch, + ) + else: + raise ValueError( + "No matching saved branch migration. For a fresh/cache-loss start, omit " + "--migrate-canonical-branch-from and use the current canonical configuration." + ) store.put("settings", settings) if store.get("since") is None: store.put( @@ -203,6 +272,95 @@ def __init__( args.since or "1970-01-01T00:00:00Z", ) + def _verify_branch_migration(self, previous_branch: str) -> dict: + """Read authoritative remote state before changing the local configuration.""" + self.progress.operation("verifying_branch_migration") + revision = promote._main_sha(self.gh) + current = promote._lock_at(self.gh, self.gh.consumer_repo, revision) + if ( + promote._repo_name(current.url).lower() != self.args.canonical_repo.lower() + or current.branch != self.args.canonical_branch + ): + raise ValueError( + "Branch migration requires the merged base-branch lock to point to the new " + "canonical repository/branch. Finish the refresh before restarting." + ) + ref = self.gh.get( + f"repos/{self.args.canonical_repo}/git/ref/heads/{self.args.canonical_branch}" + ) + tip = promote._sha(ref["object"]["sha"]) + if self.gh.compare(self.args.canonical_repo, current.commit, tip)["status"] not in ( + "identical", + "ahead", + ): + raise ValueError( + "New canonical branch does not contain the merged lock's pinned commit." + ) + self._check_pending_promotions(previous_branch) + latest = promote._lock_at(self.gh, self.gh.consumer_repo, promote._main_sha(self.gh)) + if latest.to_mapping() != current.to_mapping(): + raise ValueError("Base vendor lock changed during migration; inspect it and retry.") + return { + "from": previous_branch, + "to": self.args.canonical_branch, + "at": _now(), + "consumer_revision": revision, + "canonical_tip": tip, + "pin": current.commit, + } + + def _check_pending_promotions(self, previous_branch: str) -> None: + """Do not strand open follow-ups or unpublished promotion branches.""" + prefix = f"chore/{self.gh.vendor_name}-promote-" + branches = { + item["name"]: item["commit"]["sha"] + for item in _pages(self.gh, f"repos/{self.args.fork}/branches") + if item["name"].startswith(prefix) + } + local = promote._git( + self.args.repo, + "for-each-ref", + "--format=%(refname:short) %(objectname)", + f"refs/heads/{prefix}*", + ) + for line in local.splitlines(): + branch, sha = line.split() + if branch not in branches: + branches[branch] = sha + # Open follow-ups can survive deletion of their publishing branch. + for pr in _pages(self.gh, f"{self.prefix}/pulls?state=open"): + branch = pr["head"]["ref"] + if pr["base"]["ref"] == self.gh.base_branch and branch.startswith(prefix): + branches[branch] = pr["head"]["sha"] + for branch, sha in branches.items(): + remote = self.gh.optional(f"{self.prefix}/commits/{sha}") + if remote is None: + remote = self.gh.optional(f"repos/{self.args.fork}/commits/{sha}") + message = ( + remote["commit"]["message"] + if remote + else promote._git(self.args.repo, "log", "-1", "--format=%B", sha) + ) + record = promote._record_from_message(message) + if record.get("canonical_branch") != previous_branch: + continue + if record.get("vendor") != self.gh.vendor_name: + raise ValueError(f"Unexpected vendor provenance on promotion branch {branch}.") + prs = list( + _pages( + self.gh, + f"{self.prefix}/pulls?state=all&head={self.args.fork.split('/')[0]}:{branch}" + f"&base={self.gh.base_branch}", + ) + ) + if len(prs) != 1 or prs[0]["state"] != "closed": + location = prs[0]["html_url"] if len(prs) == 1 else branch + raise ValueError( + f"Unresolved old-branch promotion: {location}. Finish it, or explicitly " + "close its superseded PR before migrating. Unpublished branches need " + "maintainer inspection; migration never deletes or redirects them." + ) + def _discover(self) -> list[int]: # Advance only after a complete scan, with overlap for edits arriving # during pagination. Tracked operations are reconciled independently. @@ -266,19 +424,12 @@ def _transition(self, number: int, state: str, reason: str = "") -> None: self.store.put(f"problem:{number}", observation["reason"] if state == "blocked" else None) def _feedback( - self, pr: dict, problem: str | None, *, attribution: dict[str, dict] | None = None + self, pr: dict, problem: Diagnostic | None, *, attribution: dict[str, dict] | None = None ) -> None: number = pr["number"] message = ( - f"@{pr['user']['login']} — vendor promotion needs author action:\n\n{problem}\n\n" - "Add or correct this block in the **PR description** (replace the example PR URL):\n\n" - f"````markdown\n{metadata.template(self.gh.vendor_name, self.gh.upstream_repo)}\n````\n\n" - "List every upstream PR; a full commit_map is not required. For an unresolved commit, " - "add a missing PR, update the source pin, or add under this vendor:\n\n" - "```yaml\nresolutions:\n FULL_SOURCE_SHA:\n upstream_pr: https://github.com/OWNER/REPO/pull/123\n" - " reason: Explain the adaptation or why this PR owns the change.\n```\n\n" - "If there is no paired upstream PR, use only `unpaired_reason: ...` for this vendor. " - "Author assertions are retained during future refresh until separately verified." + f"@{pr['user']['login']} — vendor promotion needs {problem.owner} action:\n\n{problem}" + "\n\nThe bot will recheck automatically on a later poll." if problem else "Vendor promotion metadata and source attribution are valid. " "This is **not** a code-review approval. Promotion still requires the source PR to merge." @@ -296,7 +447,7 @@ def _feedback( self._transition( number, "blocked" if problem else "metadata_valid", - problem or "dry run; no feedback published", + str(problem) if problem else "dry run; no feedback published", ) return # Check immutable head and description immediately before feedback. A @@ -334,7 +485,7 @@ def _feedback( self._transition( number, "blocked" if problem else "metadata_valid", - problem or "feedback reconciled; not a code-review approval", + str(problem) if problem else "feedback reconciled; not a code-review approval", ) return reviews = [ @@ -353,7 +504,7 @@ def _feedback( if not item.get("body", "").startswith(self.marker) and item["state"] in ("APPROVED", "CHANGES_REQUESTED") ] - if problem and not automated and not manual: + if problem and problem.owner == "author" and not automated and not manual: self.gh.api( f"{self.prefix}/pulls/{number}/reviews", method="POST", @@ -361,8 +512,8 @@ def _feedback( "event": "REQUEST_CHANGES", "commit_id": pr["head"]["sha"], "body": ( - f"{self.marker}\n{problem}\n\nSee the vendor-bot comment for the required " - "description format and author resolution instructions." + f"{self.marker}\n{problem.problem}\n\nSee the vendor-bot comment for " + "the specific fix and the latest validation result." ), }, ) @@ -408,7 +559,7 @@ def _feedback( self._transition( number, "blocked" if problem else "metadata_valid", - problem or "feedback reconciled; not a code-review approval", + str(problem) if problem else "feedback reconciled; not a code-review approval", ) def _inputs(self, pr: dict) -> tuple[manage.Vendor, manage.Vendor] | None: @@ -421,13 +572,25 @@ def _inputs(self, pr: dict) -> tuple[manage.Vendor, manage.Vendor] | None: base = commit["parents"][0]["sha"] else: base, revision = promote._main_sha(self.gh), pr["head"]["sha"] - before = promote._lock_document_at(self.gh, self.gh.consumer_repo, base)["vendors"] + try: + before = promote._lock_document_at(self.gh, self.gh.consumer_repo, base)["vendors"] + except Diagnostic as error: + raise RuntimeError(f"Canonical/base vendor lock is invalid: {error}") from error after = promote._lock_document_at(self.gh, self.gh.consumer_repo, revision)["vendors"] name = self.gh.vendor_name if name not in before or name not in after: return None # Vendor creation/removal is not promotion. previous = manage._validate_vendor(name, before[name]) - reviewed = manage._validate_vendor(name, after[name]) + try: + reviewed = manage._validate_vendor(name, after[name]) + promote._repo_name(reviewed.url) + except (manage.VendorError, ValueError) as error: + raise Diagnostic( + f"Invalid lock entry at `vendors.{name}`: {field(_summary(error))}.", + f"Correct that entry in `{promote._LOCK}` with `scripts/vendor/manage.py`, " + "regenerate the vendored snapshot if its source changed, and push the PR. " + "Editing the PR description will not fix a lock-file error.", + ) from error if previous.commit == reviewed.commit or not reviewed.branch: return None # Includes lock-only promotions and tag-only updates. canonical = ( @@ -443,8 +606,16 @@ def _inputs(self, pr: dict) -> tuple[manage.Vendor, manage.Vendor] | None: if not canonical: if pr.get("merged"): return None # Historical update belonging to another canonical branch. - raise ValueError( - "Finish the preceding vendor promotion, then rebase this source-update PR onto the canonical pin." + raise Diagnostic( + "Waiting for the preceding vendor promotion: the base lock still points " + f"to repository {field(previous.url)}, branch {field(previous.branch)}.", + f"A maintainer must finish the preceding lock-only promotion back to " + f"{field(self.args.canonical_repo)} / {field(self.args.canonical_branch)}. " + "After it merges, the author should rebase this consumer PR onto the latest " + "base branch and rebase its temporary source branch onto the new canonical " + "pin, then update the lock/vendor snapshot. No metadata edit can unblock " + "the preceding promotion.", + owner="maintainer", ) return previous, reviewed @@ -509,8 +680,8 @@ def inspect(self, number: int) -> None: return previous, reviewed = inputs self.progress.count("relevant") - except (ValueError, manage.VendorError) as error: - self._feedback(pr, str(error)) + except Diagnostic as error: + self._feedback(pr, error) return arguments = self._arguments(number, reviewed) if pr.get("merged"): @@ -532,8 +703,8 @@ def inspect(self, number: int) -> None: # Until provenance is committed, decisions follow current remote # inputs, not whichever upstream heads a previous process cached. _, evidence = metadata.resolve(self.gh, previous, reviewed, entry, cache) - except (ValueError, manage.VendorError) as error: - self._feedback(pr, str(error)) + except Diagnostic as error: + self._feedback(pr, error) return self.progress.operation("reconciling_feedback", number) self._feedback(pr, None, attribution=evidence) @@ -787,7 +958,7 @@ def _status(workdir: Path) -> dict: sqlite3.connect(f"{database.as_uri()}?mode=ro", uri=True, timeout=1) ) as connection: rows = connection.execute( - "SELECT key, value FROM state WHERE key IN ('health', 'process', 'settings')" + "SELECT key, value FROM state WHERE key IN ('health', 'process', 'settings', 'migration')" ).fetchall() state = {key: json.loads(value) for key, value in rows} if any(not isinstance(value, dict) for value in state.values()): @@ -810,6 +981,9 @@ def _status(workdir: Path) -> dict: pid=process.get("pid") if active else None, publish=process.get("publish"), vendor=state.get("settings", {}).get("vendor"), + canonical_repo=state.get("settings", {}).get("canonical_repo"), + canonical_branch=state.get("settings", {}).get("canonical_branch"), + migration=state.get("migration"), ) if not active: result["operation"] = None @@ -828,6 +1002,10 @@ def _parser() -> argparse.ArgumentParser: run.add_argument("--upstream-repo", required=True) run.add_argument("--canonical-repo", required=True) run.add_argument("--canonical-branch", required=True) + run.add_argument( + "--migrate-canonical-branch-from", + help="Explicitly migrate this stopped workdir from OLD_BRANCH after a merged refresh.", + ) run.add_argument("--fork", required=True, help="Consumer publishing fork OWNER/REPO.") run.add_argument("--consumer-repo", default="NVIDIA/TensorRT-LLM") run.add_argument("--base-branch", default="main") @@ -897,6 +1075,10 @@ def main(argv: list[str] | None = None) -> int: promote._repo_name(f"https://github.com/{name}") for branch in (args.canonical_branch, args.base_branch, args.upstream_branch): promote._run(["git", "check-ref-format", f"refs/heads/{branch}"]) + if args.migrate_canonical_branch_from: + promote._run( + ["git", "check-ref-format", f"refs/heads/{args.migrate_canonical_branch_from}"] + ) if args.since: datetime.datetime.strptime(args.since, "%Y-%m-%dT%H:%M:%SZ") args.repo = args.repo.expanduser().resolve() diff --git a/scripts/vendor/diagnostics.py b/scripts/vendor/diagnostics.py new file mode 100644 index 000000000000..19dd7b1698bc --- /dev/null +++ b/scripts/vendor/diagnostics.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Actionable diagnostics safe to publish separately from operator failures.""" + +from __future__ import annotations + +import html +from typing import Literal + +import yaml + + +class Diagnostic(ValueError): + """A recognized problem with a corrective action and an optional Markdown example.""" + + def __init__( + self, + problem: str, + action: str, + *, + example: str = "", + owner: Literal["author", "maintainer"] = "author", + ) -> None: + self.problem, self.action, self.example, self.owner = problem, action, example, owner + super().__init__(problem) + + def __str__(self) -> str: + return f"{self.problem}\n\n**Fix:** {self.action}" + ( + f"\n\n{self.example}" if self.example else "" + ) + + +def field(value: object) -> str: + """Quote a bounded field identifier without interpreting author-supplied Markdown.""" + text = str(value).replace("\r", "\\r").replace("\n", "\\n")[:160] + return "" + html.escape(text) + "" + + +def yaml_problem(error: yaml.YAMLError) -> str: + """Describe YAML failures and locations without echoing source excerpts or values.""" + detail = "invalid YAML syntax or unsupported YAML value" + if isinstance(error, yaml.MarkedYAMLError): + problem = error.problem or "" + if problem.startswith("found duplicate key"): + detail = "duplicate mapping key; keep one value for each key" + elif "merge keys are not allowed" in problem: + detail = "YAML merge keys are not allowed; write fields explicitly" + elif problem == "found an unhashable mapping key": + detail = "mapping keys must be scalar field names, not lists or mappings" + elif "could not find expected ':'" in problem: + detail = "missing ':' after a mapping key" + elif problem.startswith("expected ',' or ']'"): + detail = "expected a comma between list items or a closing ']'" + elif problem.startswith("expected ',' or '}'"): + detail = "expected a comma between mapping entries or a closing '}'" + elif problem.startswith("expected "): + detail = "unexpected field or indentation; align sibling fields with spaces" + mark = error.problem_mark + if mark is not None: + detail += f" at YAML line {mark.line + 1}, column {mark.column + 1}" + return detail diff --git a/scripts/vendor/metadata.py b/scripts/vendor/metadata.py index 89babbe6e734..544cc3dc3ee6 100644 --- a/scripts/vendor/metadata.py +++ b/scripts/vendor/metadata.py @@ -18,9 +18,11 @@ if __package__: from . import manage, promote + from .diagnostics import Diagnostic, field, yaml_problem else: import manage import promote + from diagnostics import Diagnostic, field, yaml_problem BEGIN = "" END = "" @@ -37,46 +39,201 @@ def template(vendor_name: str, upstream_repo: str) -> str: ) +def _vendor_example(vendor_name: str, content: str) -> str: + return f"```yaml\nvendors:\n {vendor_name}:\n{content}\n```" + + +def _resolution_example(vendor_name: str, upstream_repo: str) -> str: + return _vendor_example( + vendor_name, + f" upstream_prs:\n - https://github.com/{upstream_repo}/pull/123\n" + " resolutions:\n FULL_40_CHARACTER_SOURCE_SHA:\n" + f" upstream_pr: https://github.com/{upstream_repo}/pull/123\n" + " reason: Explain the downstream adaptation or why this PR owns the change.", + ) + + +def _pr_reference( + reference: object, upstream_repo: str, path: str, *, require_url: bool = True +) -> int: + if isinstance(reference, str) and ( + not require_url or reference.startswith(f"https://github.com/{upstream_repo}/pull/") + ): + try: + return promote._pr_number(reference, upstream_repo) + except ValueError: + pass + raise Diagnostic( + f"Invalid upstream PR URL at {field(path)}.", + f"In the PR description, use a full URL such as " + f"`https://github.com/{upstream_repo}/pull/123` with the actual PR number " + "and no query string or fragment.", + ) + + def parse(body: str, vendor_name: str, upstream_repo: str) -> dict: """Read exactly one marked block and validate the selected vendor's inputs.""" if len(body) > 65536: - raise ValueError("PR description exceeds the supported metadata size.") + raise Diagnostic( + "PR description exceeds the supported metadata size (65,536 characters).", + "Shorten the PR description to at most 65,536 characters, keeping the complete " + "vendor-promotion block. Move long logs to an attachment or a comment.", + ) body = body.replace("\r\n", "\n").replace("\r", "\n") - if body.count(BEGIN) != 1 or body.count(END) != 1: - raise ValueError("PR description must contain exactly one vendor-promotion marked block.") + counts = {BEGIN: body.count(BEGIN), END: body.count(END)} + missing = [marker for marker, count in counts.items() if count == 0] + duplicates = [marker for marker, count in counts.items() if count > 1] + full_example = f"````markdown\n{template(vendor_name, upstream_repo)}\n````" + if missing or duplicates: + problems, actions = [], [] + if missing: + problems.append( + "Missing vendor-promotion markers in the PR description: " + + ", ".join(f"`{marker}`" for marker in missing) + + "." + ) + if len(missing) == 2: + actions.append( + "Edit this PR's description—not a comment—and insert the complete block " + "below. Replace the example upstream PR URL with your actual PR URL. " + "If there is genuinely no paired upstream PR, replace `upstream_prs` " + "with a nonempty `unpaired_reason` explaining why." + ) + else: + position = "before" if missing[0] == BEGIN else "after" + actions.append( + f"Edit this PR's description—not a comment—and insert `{missing[0]}` " + f"on its own line {position} the existing fenced YAML block. " + "Keep the existing metadata and the other marker." + ) + if duplicates: + problems.append( + "Duplicate vendor-promotion markers: " + + ", ".join(f"`{marker}` appears {counts[marker]} times" for marker in duplicates) + + "." + ) + actions.append( + "Consolidate all vendor entries into one `vendors` mapping in one marked " + "block in the PR description. Remove extra markers/blocks, preserving the " + "other vendors' metadata; keep one start marker and one end marker." + ) + raise Diagnostic( + " ".join(problems), + " ".join(actions), + example=full_example if len(missing) == 2 else "", + ) start, end = body.index(BEGIN) + len(BEGIN), body.index(END) + if start >= end: + raise Diagnostic( + "The vendor-promotion end marker appears before the start marker.", + f"Move `{BEGIN}` before the fenced YAML block and `{END}` after it " + "in the PR description. Preserve the metadata between them.", + ) block = body[start:end].strip() - if start >= end or not block.startswith("```yaml\n") or not block.endswith("\n```"): - raise ValueError("The marked block must contain one fenced yaml mapping.") + if not block.startswith("```yaml\n") or not block.endswith("\n```"): + raise Diagnostic( + "The marked block must contain one fenced yaml mapping.", + "In the PR description, put the metadata between an opening line of three " + "backticks followed by `yaml` and a closing line of three backticks. " + "Keep both markers outside that fence, as shown below; retain your real PR URLs.", + example=full_example, + ) try: content = block[len("```yaml\n") : -len("\n```")] if any( isinstance(token, (yaml.tokens.AliasToken, yaml.tokens.AnchorToken)) for token in yaml.scan(content) ): - raise ValueError("Promotion YAML aliases and anchors are not allowed.") + raise Diagnostic( + "Promotion YAML aliases and anchors are not allowed.", + "In the PR description's metadata block, replace YAML anchors (`&name`) " + "and aliases (`*name`) with explicit values.", + ) data = yaml.load(content, Loader=manage._UniqueKeyLoader) except yaml.YAMLError as error: - raise ValueError(f"Invalid promotion YAML: {error}") from error - if not isinstance(data, dict) or set(data) != {"schema_version", "vendors"}: - raise ValueError("Promotion YAML requires only schema_version and vendors.") + raise Diagnostic( + f"Invalid promotion YAML: {yaml_problem(error)}.", + "Correct the metadata block in the PR description at the location above " + "(line 1 is the first line inside the YAML fence). Use space indentation, " + "balanced brackets/quotes, and unique mapping keys.", + ) from error + if not isinstance(data, dict): + raise Diagnostic( + "Promotion YAML must be a mapping, not a list or scalar.", + "Use top-level `schema_version: 1` and `vendors:` fields in the PR description, " + "following this structure. Replace the example URL.", + example=full_example, + ) + missing_keys = {"schema_version", "vendors"} - set(data) + extra_keys = set(data) - {"schema_version", "vendors"} + if missing_keys or extra_keys: + raise Diagnostic( + "Invalid top-level promotion YAML fields. " + + ( + "Missing: " + ", ".join(field(key) for key in sorted(missing_keys)) + ". " + if missing_keys + else "" + ) + + ( + "Unsupported: " + ", ".join(field(key) for key in sorted(extra_keys, key=str)) + "." + if extra_keys + else "" + ), + "In the PR description, keep only top-level `schema_version: 1` and `vendors:`; " + "put vendor-specific fields under `vendors.`.", + ) if type(data["schema_version"]) is not int or data["schema_version"] != 1: - raise ValueError("Unsupported promotion schema_version; expected 1.") + raise Diagnostic( + "Unsupported promotion schema_version; expected integer 1.", + "Set `schema_version: 1` (without quotes) in the PR description's metadata block.", + ) entries = data["vendors"] if not isinstance(entries, dict) or vendor_name not in entries: - raise ValueError(f"Promotion YAML must include vendors.{vendor_name}.") - if any( - not isinstance(name, str) or not manage._NAME_PATTERN.fullmatch(name) for name in entries - ): - raise ValueError("Invalid vendor key in promotion YAML.") + raise Diagnostic( + f"Promotion YAML must include vendors.{vendor_name} as a mapping entry.", + f"Add `{vendor_name}:` under `vendors:` in the existing PR-description block. " + "Preserve any other vendor entries and replace the example URL.", + example=_vendor_example( + vendor_name, + f" upstream_prs:\n - https://github.com/{upstream_repo}/pull/123", + ), + ) + invalid_names = [ + name + for name in entries + if not isinstance(name, str) or not manage._NAME_PATTERN.fullmatch(name) + ] + if invalid_names: + raise Diagnostic( + "Invalid vendor keys under `vendors`: " + + ", ".join(field(name) for name in invalid_names) + + ".", + "Use the exact vendor names from `3rdparty/vendor_sources.lock.yaml` as " + "string keys under `vendors` in the PR description.", + ) entry = entries[vendor_name] + path = f"vendors.{vendor_name}" if not isinstance(entry, dict) or set(entry) - { "upstream_prs", "unpaired_reason", "resolutions", }: - raise ValueError( - "Vendor metadata allows upstream_prs, unpaired_reason, and resolutions only." + extra = ( + set(entry) - {"upstream_prs", "unpaired_reason", "resolutions"} + if isinstance(entry, dict) + else set() + ) + raise Diagnostic( + f"Invalid vendor metadata at {field(path)}. " + + ( + "Unsupported fields: " + + ", ".join(field(key) for key in sorted(extra, key=str)) + + "." + if extra + else "Expected a mapping." + ), + "Use only `upstream_prs`, `unpaired_reason`, and `resolutions` under this vendor " + "in the PR description. Repository/operator settings do not belong here.", ) if "unpaired_reason" in entry: if ( @@ -84,7 +241,13 @@ def parse(body: str, vendor_name: str, upstream_repo: str) -> dict: or not isinstance(entry["unpaired_reason"], str) or not entry["unpaired_reason"].strip() ): - raise ValueError("unpaired_reason must be nonempty and cannot accompany upstream PRs.") + raise Diagnostic( + f"Invalid {field(path + '.unpaired_reason')}: it must be nonempty and " + "cannot accompany `upstream_prs` or `resolutions`.", + "If there genuinely is no paired upstream PR, keep only `unpaired_reason` " + "with an explanation under this vendor. Otherwise remove `unpaired_reason` " + "and list the actual upstream PRs. Do not use it to bypass failed matching.", + ) return { "upstream_prs": [], "resolutions": {}, @@ -92,30 +255,84 @@ def parse(body: str, vendor_name: str, upstream_repo: str) -> dict: } references = entry.get("upstream_prs") if not isinstance(references, list) or not references: - raise ValueError("upstream_prs must be a nonempty list of upstream PR URLs.") + raise Diagnostic( + f"{field(path + '.upstream_prs')} must be a nonempty list of upstream PR URLs.", + "In the PR description, list every related upstream PR using YAML list items " + "(`- URL`); replace the example URL. If there genuinely is no paired PR, use " + "only a nonempty `unpaired_reason` instead.", + example=_vendor_example( + vendor_name, + f" upstream_prs:\n - https://github.com/{upstream_repo}/pull/123", + ), + ) numbers = [] - for reference in references: - if not isinstance(reference, str) or not reference.startswith( - f"https://github.com/{upstream_repo}/pull/" - ): - raise ValueError(f"Each upstream PR must be a URL in {upstream_repo}.") - numbers.append(promote._pr_number(reference, upstream_repo)) - if len(set(numbers)) != len(numbers): - raise ValueError("upstream_prs contains duplicate PRs.") + for index, reference in enumerate(references): + number = _pr_reference(reference, upstream_repo, f"{path}.upstream_prs[{index}]") + if number in numbers: + raise Diagnostic( + f"{field(path + '.upstream_prs')} contains duplicate upstream PR #{number}.", + f"Remove the repeated `https://github.com/{upstream_repo}/pull/{number}` " + "entry from the PR description; keep each upstream PR once.", + ) + numbers.append(number) resolutions = entry.get("resolutions", {}) if not isinstance(resolutions, dict): - raise ValueError("resolutions must be a mapping keyed by full source commit SHA.") + raise Diagnostic( + f"{field(path + '.resolutions')} must be a mapping keyed by full source commit SHA.", + "Replace it with the structure below inside the existing vendor entry. Use a " + "full 40-character source SHA, a listed upstream PR, and a specific explanation.", + example=_resolution_example(vendor_name, upstream_repo), + ) normalized = {} for sha, resolution in resolutions.items(): - promote._sha(sha) + resolution_path = f"{path}.resolutions.{sha}" + try: + promote._sha(sha) + except ValueError as error: + raise Diagnostic( + f"Invalid source commit key at {field(resolution_path)}.", + "Use the full lowercase 40-character SHA from the bot's unresolved-commit " + "list, not a short SHA, branch name, or upstream commit SHA.", + ) from error if not isinstance(resolution, dict) or set(resolution) != {"upstream_pr", "reason"}: - raise ValueError("Each resolution requires upstream_pr and reason.") + details = "Expected a mapping." + if isinstance(resolution, dict): + absent = {"upstream_pr", "reason"} - set(resolution) + unknown = set(resolution) - {"upstream_pr", "reason"} + details = ( + "Missing: " + ", ".join(field(key) for key in sorted(absent)) + ". " + if absent + else "" + ) + ( + "Unsupported: " + + ", ".join(field(key) for key in sorted(unknown, key=str)) + + "." + if unknown + else "" + ) + raise Diagnostic( + f"{field(resolution_path)} requires exactly `upstream_pr` and `reason`. {details}", + "Add the missing fields or remove extra fields in this resolution, " + "following the structure below. Preserve other vendor metadata.", + example=_resolution_example(vendor_name, upstream_repo), + ) reference, reason = resolution["upstream_pr"], resolution["reason"] - if not isinstance(reference, str): - raise ValueError("Resolution upstream_pr must be a PR URL.") - number = promote._pr_number(reference, upstream_repo) - if number not in numbers or not isinstance(reason, str) or not reason.strip(): - raise ValueError("Resolution must name a listed PR and explain the unmatched patch.") + number = _pr_reference( + reference, upstream_repo, f"{resolution_path}.upstream_pr", require_url=False + ) + if number not in numbers: + raise Diagnostic( + f"{field(resolution_path)} names upstream PR #{number}, which is not listed.", + f"Add `https://github.com/{upstream_repo}/pull/{number}` to " + f"`{path}.upstream_prs` if it owns this change, or correct the resolution " + "to reference a listed PR.", + ) + if not isinstance(reason, str) or not reason.strip(): + raise Diagnostic( + f"{field(resolution_path + '.reason')} must be a nonempty string.", + "Explain the downstream adaptation or why this upstream PR owns the " + "unmatched change. Author assertions are retained until separately verified.", + ) normalized[sha] = {"upstream_pr": number, "reason": reason.strip()} return {"upstream_prs": sorted(numbers), "resolutions": normalized, "unpaired_reason": None} @@ -242,6 +459,7 @@ def match( source: str = ".", include: tuple[str, ...] = ("**/*",), upstream_repo: str | None = None, + vendor_name: str = "VENDOR_NAME", ) -> tuple[dict[str, int | None], dict[str, dict]]: """Match a linear source range against fetched immutable upstream PR snapshots. @@ -251,20 +469,41 @@ def match( """ commits = _commits(repo, previous, reviewed) if not commits: - raise ValueError("Source update has no new commits to attribute.") + raise Diagnostic( + "Source update has no new commits to attribute.", + "Check the source SHA in `3rdparty/vendor_sources.lock.yaml`. Pin the intended " + "new source commit, or remove the unnecessary source update from this PR.", + ) source_parents = {} for sha in commits: parents = promote._git(repo, "show", "-s", "--format=%P", sha).split() if len(parents) != 1: - raise ValueError("Source history must be linear; rebase merge commits first.") + raise Diagnostic( + f"Source history must be linear; commit `{sha}` has {len(parents)} parents.", + f"Rebase the temporary source branch onto canonical pin `{previous}` without " + "merge commits. Then use `scripts/vendor/manage.py` to update the lock and " + "vendored snapshot to the new source SHA, and push the consumer PR.", + ) source_parents[sha] = parents[0] resolutions = resolutions or {} if set(resolutions) - set(commits): - raise ValueError("Resolutions include commits outside the reviewed source delta.") + raise Diagnostic( + "Resolutions include commits outside the reviewed source delta: " + + ", ".join(field(sha) for sha in sorted(set(resolutions) - set(commits))) + + ".", + f"Remove those stale entries from `vendors.{vendor_name}.resolutions` in the PR " + "description. After rebasing, use the new source SHAs reported by the bot.", + ) candidates: dict[str, set[int]] = {sha: set() for sha in commits} patches = {sha: _patch(repo, f"{sha}^", sha, source=source, include=include) for sha in commits} if any(not patches[sha] for sha in resolutions): - raise ValueError("Remove resolutions for commits with no lock-selected file changes.") + raise Diagnostic( + "Remove resolutions for commits with no lock-selected file changes: " + + ", ".join(f"`{sha}`" for sha in resolutions if not patches[sha]) + + ".", + f"Delete only these entries from `vendors.{vendor_name}.resolutions`. " + "Excluded-only changes do not need upstream attribution.", + ) series = _patch(repo, previous, reviewed, source=source, include=include) methods: dict[tuple[str, int], str] = {} comparisons: dict[int, list[tuple[str, str, str, str]]] = {} @@ -338,7 +577,11 @@ def match( resolution = resolutions[sha] number = resolution["upstream_pr"] if number not in upstream_prs: - raise ValueError("Resolution names an unlisted upstream PR.") + raise Diagnostic( + f"Resolution for `{sha}` names unlisted upstream PR #{number}.", + f"Add that PR to `vendors.{vendor_name}.upstream_prs` if it owns this " + "change, or correct the resolution to reference a listed PR.", + ) assignments[sha] = number evidence[sha] = { "method": "author-assertion", @@ -364,7 +607,10 @@ def match( evidence[sha]["method"] = methods[sha, number] else: options = ( - ", ".join(f"#{number}" for number in sorted(candidates[sha])) or "no exact match" + ", ".join( + _upstream_reference(number, upstream_repo) for number in sorted(candidates[sha]) + ) + or "no exact match" ) detail = f"- `{sha}`: {options}" if sha in similarities: @@ -386,17 +632,28 @@ def match( unresolved.append(detail) accepted = similarity_feedback(evidence, upstream_repo) if unresolved: - raise ValueError( - "Author action required: upstream attribution is unresolved. Add missing upstream PRs, " - "update/rebase the source pin, or add a targeted resolution with an upstream_pr and " - "reason for each commit below. No complete commit_map is required.\n" + raise Diagnostic( + "Upstream attribution is unresolved for these source commits:\n\n" + "\n".join(unresolved) - + ("\n\n" + accepted if accepted else "") + + ("\n\n" + accepted if accepted else ""), + f"In the PR description, add missing PRs to `vendors.{vendor_name}.upstream_prs`, " + "remove incorrectly listed PRs, or correct/rebase the source pin and update the " + "vendored snapshot. For an intentional adaptation or ambiguous ownership, merge " + "a targeted `resolutions` entry into the existing vendor metadata as shown below. " + "Replace the SHA and URL and give a specific reason; preserve other entries. " + "No complete commit_map is required. A resolution records an author assertion, " + "not verified equivalence; it is retained during refresh until separately verified.", + example=_resolution_example(vendor_name, upstream_repo or "OWNER/REPO"), ) if set(assignments.values()) - {None} != set(upstream_prs): - raise ValueError( - "Some listed upstream PRs cover no new lock-selected changes; remove unrelated PRs." - + ("\n\n" + accepted if accepted else "") + unused = set(upstream_prs) - set(assignments.values()) + raise Diagnostic( + "Some listed upstream PRs cover no new lock-selected changes: " + + ", ".join(_upstream_reference(number, upstream_repo) for number in sorted(unused)) + + "." + + ("\n\n" + accepted if accepted else ""), + f"Remove these unrelated PRs from `vendors.{vendor_name}.upstream_prs` in the " + "PR description. PRs covering only excluded tests/docs do not need to be listed.", ) return assignments, evidence @@ -410,35 +667,58 @@ def resolve( ) -> tuple[dict[str, int | None], dict[str, dict]]: """Fetch only Git data into a private bare cache and resolve declared PRs.""" if previous.source != reviewed.source or set(previous.include) != set(reviewed.include): - raise ValueError( - "Automatic attribution requires unchanged lock source/include selection; " - "separate the selection change from the source update." + changed = (["source"] if previous.source != reviewed.source else []) + ( + ["include"] if set(previous.include) != set(reviewed.include) else [] + ) + raise Diagnostic( + "Automatic attribution requires unchanged lock source/include selection. " + "Changed fields: " + ", ".join(f"`{name}`" for name in changed) + ".", + "Separate the selection change into a different PR. In this source-update PR, " + "restore the canonical `source`/`include` selection in " + "`3rdparty/vendor_sources.lock.yaml` and regenerate the vendor snapshot.", ) cache.mkdir(parents=True, exist_ok=True) if not (cache / "HEAD").exists(): promote._git(cache, "init", "--bare", "--quiet") if promote._git(cache, "rev-parse", "--is-bare-repository") != "true": raise ValueError("Matching cache must be a dedicated bare repository.") - promote._check_source_repo(gh, promote._repo_name(reviewed.url)) - for sha in (previous.commit, reviewed.commit): - promote._git(cache, "fetch", "--quiet", "--no-tags", reviewed.url, sha) + source_repo = promote._repo_name(reviewed.url) try: - common = promote._git(cache, "merge-base", previous.commit, reviewed.commit) - except RuntimeError as error: - raise ValueError( - "Source revisions have no merge base; rebase the temporary source branch onto the canonical pin." + promote._check_source_repo(gh, source_repo) + except Diagnostic as error: + raise Diagnostic( + f"The source repository {field(source_repo)} is not a fork of {field(gh.upstream_repo)}.", + "Push the temporary source branch to a fork of the configured upstream and " + "update the lock URL, branch, SHA, and vendored snapshot with " + "`scripts/vendor/manage.py`.", ) from error + for sha in (previous.commit, reviewed.commit): + promote._git(cache, "fetch", "--quiet", "--no-tags", reviewed.url, sha) + # An empty successful query means unrelated histories; Git failures remain + # operator errors rather than being relabeled as author mistakes. + common = promote._git( + cache, "merge-base", previous.commit, reviewed.commit, allowed_returncodes=(0, 1) + ) + rebase_action = ( + f"Rebase the temporary source branch {field(reviewed.branch)} onto canonical " + f"repository {field(previous.url)}, branch {field(previous.branch)}, SHA " + f"`{previous.commit}`. Then use `scripts/vendor/manage.py` to update " + "`3rdparty/vendor_sources.lock.yaml` and the vendored snapshot to the rebased SHA, " + "and push the source branch and consumer PR. Editing metadata alone will not fix this." + ) + if not common: + raise Diagnostic("Source revisions have no merge base.", rebase_action) if common != previous.commit: - raise ValueError( - "Source update is not a fast-forward; rebase the temporary source branch onto the canonical pin." - ) + raise Diagnostic("Source update is not a fast-forward.", rebase_action) if entry["unpaired_reason"]: commits = _commits(cache, previous.commit, reviewed.commit) if not commits or any( len(promote._git(cache, "show", "-s", "--format=%P", sha).split()) != 1 for sha in commits ): - raise ValueError("Unpaired source updates still require a nonempty linear history.") + raise Diagnostic( + "Unpaired source updates still require a nonempty linear history.", rebase_action + ) return {}, {} prs = {} for number in entry["upstream_prs"]: @@ -446,8 +726,18 @@ def resolve( if pr["base"]["ref"] != gh.upstream_branch or ( pr["state"] == "closed" and not pr.get("merged") ): - raise ValueError( - f"Upstream PR #{number} must target {gh.upstream_branch} and be open or merged." + raise Diagnostic( + f"{_upstream_reference(number, gh.upstream_repo)} must target " + f"`{gh.upstream_branch}` and be open or merged; currently targets " + f"{field(pr['base']['ref'])} and is " + + ( + "closed without merging." + if pr["state"] == "closed" and not pr.get("merged") + else f"{field(pr['state'])}." + ), + "Correct the upstream PR target or reopen the upstream PR if appropriate; " + "otherwise replace its URL in the consumer PR description with the correct " + "open/merged upstream PR. Do not remove attribution for changes still included.", ) for sha in (pr["base"]["sha"], pr["head"]["sha"]): promote._git( @@ -468,4 +758,5 @@ def resolve( source=reviewed.source, include=reviewed.include, upstream_repo=gh.upstream_repo, + vendor_name=gh.vendor_name, ) diff --git a/scripts/vendor/promote.py b/scripts/vendor/promote.py index cee4f4a7f02a..d06ed02200c4 100644 --- a/scripts/vendor/promote.py +++ b/scripts/vendor/promote.py @@ -29,8 +29,10 @@ if __package__: from . import manage as vendor + from .diagnostics import Diagnostic, field, yaml_problem else: import manage as vendor + from diagnostics import Diagnostic, field, yaml_problem _LOCK = "3rdparty/vendor_sources.lock.yaml" _BEGIN = "----- BEGIN VENDOR PROMOTION V1 -----" @@ -48,6 +50,7 @@ def _run( stream: bool = False, strip: bool = True, binary_output: bool = False, + allowed_returncodes: tuple[int, ...] = (0,), ) -> str: # Do not use manage._run_git here: its synthetic identity is for # patch generation. Real promotion commits must use the maintainer's identity. @@ -68,7 +71,7 @@ def _run( capture_output=not stream, timeout=timeout, ) - if result.returncode: + if result.returncode not in allowed_returncodes: detail = (result.stderr or result.stdout or "See streamed output above.").strip() raise RuntimeError(f"{command[0]} failed: {detail}") output = result.stdout or "" @@ -77,12 +80,15 @@ def _run( return output.strip() if strip else output -def _git(repo: Path, *args: str, text: str | None = None) -> str: +def _git( + repo: Path, *args: str, text: str | None = None, allowed_returncodes: tuple[int, ...] = (0,) +) -> str: return _run( ["git", "-C", str(repo), *args], text=text, timeout=1800 if args[:2] == ("worktree", "add") else 300, stream=args[:2] == ("worktree", "add") or args[0] in ("fetch", "commit", "push"), + allowed_returncodes=allowed_returncodes, ) @@ -175,10 +181,17 @@ def _lock_document_at(gh: GitHub, repo: str, commit: str) -> dict: isinstance(token, (yaml.tokens.AliasToken, yaml.tokens.AnchorToken)) for token in yaml.scan(content) ): - raise ValueError("Vendor lock YAML aliases and anchors are not allowed.") + raise Diagnostic( + "Vendor lock YAML aliases and anchors are not allowed.", + f"Replace anchors and aliases with explicit values in `{_LOCK}` and push the PR.", + ) data = yaml.load(content, Loader=vendor._UniqueKeyLoader) except yaml.YAMLError as error: - raise ValueError(f"Invalid vendor lock YAML: {error}") from error + raise Diagnostic( + f"Invalid vendor lock YAML: {yaml_problem(error)}.", + f"Correct the YAML at that location in `{_LOCK}`, keeping keys unique and " + "using spaces for indentation; then push the PR.", + ) from error if ( not isinstance(data, dict) or set(data) != {"schema_version", "vendors"} @@ -186,7 +199,11 @@ def _lock_document_at(gh: GitHub, repo: str, commit: str) -> dict: or data["schema_version"] != 1 or not isinstance(data["vendors"], dict) ): - raise ValueError("Unsupported vendor lock schema.") + raise Diagnostic( + "Unsupported vendor lock schema.", + f"In `{_LOCK}`, use only top-level `schema_version: 1` and a `vendors` mapping. " + "Preserve other vendors' entries and push the corrected file.", + ) return data @@ -205,7 +222,10 @@ def _check_fork(gh: GitHub, repo: str, upstream: str) -> None: repo.lower() == upstream.lower() or info.get("source", {}).get("full_name", "").lower() != upstream.lower() ): - raise ValueError(f"{repo} must be a personal fork of {upstream}, not upstream itself.") + raise Diagnostic( + f"{field(repo)} must be a personal fork of {field(upstream)}, not upstream itself.", + "Use a personal fork of the configured upstream repository for publication.", + ) def _check_source_repo(gh: GitHub, repo: str) -> None: diff --git a/tests/unittest/others/test_maintain_prims_ts.py b/tests/unittest/others/test_maintain_prims_ts.py index 09699b2dea65..bbd6c96f3c1e 100644 --- a/tests/unittest/others/test_maintain_prims_ts.py +++ b/tests/unittest/others/test_maintain_prims_ts.py @@ -39,11 +39,13 @@ pytestmark = pytest.mark.cpu_only -def _git(repo: Path, *args: str, text: str | None = None) -> str: +def _git( + repo: Path, *args: str, text: str | None = None, allowed_returncodes: tuple[int, ...] = (0,) +) -> str: result = subprocess.run( ["git", "-C", str(repo), *args], input=text, text=True, capture_output=True, timeout=30 ) - assert result.returncode == 0, result.stderr + assert result.returncode in allowed_returncodes, result.stderr return result.stdout.strip() @@ -197,7 +199,13 @@ def compare(self, repo: str, base: str, head: str, *, all_commits: bool = False) "total_commits": len(commits), } - def routed_git(self, repo: Path, *args: str, text: str | None = None) -> str: + def routed_git( + self, + repo: Path, + *args: str, + text: str | None = None, + allowed_returncodes: tuple[int, ...] = (0,), + ) -> str: self.git_calls.append(args) replacements = { "https://github.com/NVIDIA/TensorRT-LLM.git": str(self.consumer), @@ -208,7 +216,12 @@ def routed_git(self, repo: Path, *args: str, text: str | None = None) -> str: } if args[:3] == ("remote", "get-url", "fork"): return "https://github.com/maintainer/TensorRT-LLM.git" - return _git(repo, *[replacements.get(arg, arg) for arg in args], text=text) + return _git( + repo, + *[replacements.get(arg, arg) for arg in args], + text=text, + allowed_returncodes=allowed_returncodes, + ) def api( self, endpoint: str, *, method: str = "GET", payload: dict | None = None @@ -1286,6 +1299,7 @@ def test_bot_recovers_publication_after_cache_loss( workdir = world.root / "bot-state" b._initialize_workdir(workdir) args = argparse.Namespace( + migrate_canonical_branch_from=None, canonical_repo="maintainer/flashinfer", canonical_branch="trtllm-prims-ts-dev", fork="maintainer/TensorRT-LLM", @@ -1301,8 +1315,13 @@ def test_bot_recovers_publication_after_cache_loss( world.state = "CLEAN" git, api = world.routed_git, world.api - def crash_git(repo: Path, *arguments: str, text: str | None = None) -> str: - result = git(repo, *arguments, text=text) + def crash_git( + repo: Path, + *arguments: str, + text: str | None = None, + allowed_returncodes: tuple[int, ...] = (0,), + ) -> str: + result = git(repo, *arguments, text=text, allowed_returncodes=allowed_returncodes) boundary = ( "worktree" if arguments[:2] == ("worktree", "add") @@ -1471,6 +1490,7 @@ def test_recovery_rejects_changed_artifacts_without_writes(world: World, tamper: def test_bot_cold_recovery_dry_run_and_superseded_history(world: World) -> None: args = argparse.Namespace( + migrate_canonical_branch_from=None, canonical_repo="maintainer/flashinfer", canonical_branch="trtllm-prims-ts-dev", fork="maintainer/TensorRT-LLM", diff --git a/tests/unittest/others/test_vendor_bot.py b/tests/unittest/others/test_vendor_bot.py index bff2d885370a..e69166fb3050 100644 --- a/tests/unittest/others/test_vendor_bot.py +++ b/tests/unittest/others/test_vendor_bot.py @@ -42,6 +42,10 @@ def _body(content: str) -> str: return f"{m.BEGIN}\n```yaml\nschema_version: 1\nvendors:\n toy:\n{content}\n```\n{m.END}" +def _issue(message: str = "Missing marked block") -> m.Diagnostic: + return m.Diagnostic(message, "Edit the PR description to supply the missing metadata.") + + @pytest.mark.parametrize("line_ending", ["\n", "\r\n", "\r"], ids=["lf", "crlf", "cr"]) def test_metadata_only_needs_upstream_urls(line_ending: str) -> None: entry = m.parse( @@ -87,8 +91,10 @@ def test_metadata_size_limit_precedes_line_ending_normalization() -> None: ], ) def test_invalid_metadata_rejected(content: str) -> None: - with pytest.raises(ValueError): + with pytest.raises(m.Diagnostic) as failure: m.parse(_body(content), "toy", "org/library") + assert failure.value.action + assert "**Fix:**" in str(failure.value) @pytest.mark.parametrize( @@ -111,6 +117,183 @@ def test_author_resolution_requires_explanation() -> None: m.parse(_body(content.replace("adapted ABI", "''")), "toy", "org/library") +@pytest.mark.parametrize("line_ending", ["\n", "\r\n", "\r"]) +@pytest.mark.parametrize( + "start_count,end_count", [(0, 0), (0, 1), (1, 0), (2, 1), (1, 2), (2, 2), (2, 0)] +) +def test_marker_diagnostics_name_missing_and_duplicate_markers( + start_count: int, end_count: int, line_ending: str +) -> None: + body = line_ending.join([m.BEGIN] * start_count + [m.END] * end_count) + with pytest.raises(m.Diagnostic) as failure: + m.parse(body, "toy", "org/library") + issue = failure.value + for marker, count in ((m.BEGIN, start_count), (m.END, end_count)): + if count == 0: + assert f"`{marker}`" in issue.problem and "Missing" in issue.problem + elif count > 1: + assert f"`{marker}` appears {count} times" in issue.problem + else: + assert f"`{marker}`" not in issue.problem + assert "PR" in issue.action + if start_count == end_count == 0: + example = issue.example.removeprefix("````markdown\n").removesuffix("\n````") + assert m.parse(example, "toy", "org/library")["upstream_prs"] == [123] + assert "not a comment" in issue.action.replace("—", " ") + elif 0 in (start_count, end_count): + assert "before" in issue.action if start_count == 0 else "after" in issue.action + if max(start_count, end_count) > 1: + assert "preserving" in issue.action and "Consolidate" in issue.action + + +def test_reversed_markers_and_bad_fence_have_different_fixes() -> None: + with pytest.raises(m.Diagnostic) as reversed_markers: + m.parse(m.END + m.BEGIN, "toy", "org/library") + assert "end marker appears before" in reversed_markers.value.problem + assert f"`{m.BEGIN}` before" in reversed_markers.value.action + with pytest.raises(m.Diagnostic) as bad_fence: + m.parse(m.BEGIN + "\nschema_version: 1\n" + m.END, "toy", "org/library") + assert "fenced yaml" in bad_fence.value.problem + assert "three backticks" in bad_fence.value.action + assert m.BEGIN in bad_fence.value.example + + +@pytest.mark.parametrize( + "content,problem,fix", + [ + (" upstream_prs: []", "vendors.toy.upstream_prs", "list every related upstream PR"), + ( + " upstream_prs: [https://github.com/wrong/repo/pull/42]", + "upstream_prs[0]", + "org/library/pull/123", + ), + (" upstream_prs: [42]", "upstream_prs[0]", "full URL"), + ( + " upstream_prs: [https://github.com/org/library/pull/42#discussion]", + "upstream_prs[0]", + "no query string or fragment", + ), + ( + " upstream_prs: [https://github.com/org/library/pull/42, https://github.com/org/library/pull/42]", + "duplicate upstream PR #42", + "Remove the repeated", + ), + (" canonical_repo: wrong", "canonical_repo", "Repository/operator settings"), + (" unpaired_reason: ''", "vendors.toy.unpaired_reason", "Do not use it to bypass"), + (" unpaired_reason: downstream\n resolutions: {}", "cannot accompany", "keep only"), + ( + " upstream_prs: [https://github.com/org/library/pull/42]\n resolutions: []", + "vendors.toy.resolutions", + "full 40-character", + ), + ( + " upstream_prs: &prs [https://github.com/org/library/pull/42]", + "aliases and anchors", + "explicit values", + ), + ], +) +def test_metadata_diagnostics_identify_field_and_fix(content: str, problem: str, fix: str) -> None: + with pytest.raises(m.Diagnostic) as failure: + m.parse(_body(content), "toy", "org/library") + assert problem in failure.value.problem + assert fix in failure.value.action + + +@pytest.mark.parametrize( + "content,problem", + [ + ("schema_version: '1'\nvendors: {}", "schema_version"), + ("schema_version: 1", "vendors"), + ("schema_version: 1\nvendors: {}\nextra: false", "extra"), + ("schema_version: 1\nvendors: {}", "vendors.toy"), + ("[]", "mapping, not a list"), + ], +) +def test_metadata_structure_has_targeted_guidance(content: str, problem: str) -> None: + with pytest.raises(m.Diagnostic) as failure: + m.parse(f"{m.BEGIN}\n```yaml\n{content}\n```\n{m.END}", "toy", "org/library") + assert problem in failure.value.problem + assert "PR description" in failure.value.action or "PR-description" in failure.value.action + + +@pytest.mark.parametrize( + "content,problem", + [ + (" upstream_prs: [secret-unclosed", "YAML line"), + (" private-key: secret\n private-key: secret", "duplicate mapping key"), + (" <<: {private-key: secret}", "merge keys are not allowed"), + ], +) +def test_yaml_errors_show_location_without_echoing_values(content: str, problem: str) -> None: + with pytest.raises(m.Diagnostic) as failure: + m.parse(_body(content), "toy", "org/library") + assert problem in failure.value.problem + assert "YAML line" in failure.value.problem and "column" in failure.value.problem + assert "secret" not in str(failure.value) + assert "line 1 is the first line inside the YAML fence" in failure.value.action + + +@pytest.mark.parametrize( + "sha,resolution,problem,fix", + [ + ( + "abc123", + "upstream_pr: https://github.com/org/library/pull/42\n reason: adapted", + "Invalid source commit key", + "40-character", + ), + ( + "a" * 40, + "upstream_pr: https://github.com/org/library/pull/42", + "exactly", + "missing fields", + ), + ( + "a" * 40, + "upstream_pr: https://github.com/org/library/pull/43\n reason: adapted", + "not listed", + "upstream_prs", + ), + ( + "a" * 40, + "upstream_pr: https://github.com/org/library/pull/42\n reason: ''", + ".reason", + "Explain the downstream adaptation", + ), + ], +) +def test_resolution_errors_identify_commit_and_fix( + sha: str, resolution: str, problem: str, fix: str +) -> None: + content = ( + " upstream_prs: [https://github.com/org/library/pull/42]\n" + f" resolutions:\n {sha}:\n {resolution}" + ) + with pytest.raises(m.Diagnostic) as failure: + m.parse(_body(content), "toy", "org/library") + assert sha in failure.value.problem and problem in failure.value.problem + assert fix in failure.value.action + + +def test_resolution_example_is_nested_under_real_vendor_and_parseable() -> None: + example = m._resolution_example("toy", "org/library") + yaml_text = example.removeprefix("```yaml\n").removesuffix("\n```") + body = (f"{m.BEGIN}\n```yaml\nschema_version: 1\n{yaml_text}\n```\n{m.END}").replace( + "FULL_40_CHARACTER_SOURCE_SHA", "a" * 40 + ) + entry = m.parse(body, "toy", "org/library") + assert entry["resolutions"]["a" * 40]["upstream_pr"] == 123 + + +def test_untrusted_field_names_cannot_inject_markdown() -> None: + body = _body(" '': value") + with pytest.raises(m.Diagnostic) as failure: + m.parse(body, "toy", "org/library") + assert " None: entry = m.parse( _body(" unpaired_reason: downstream-only compatibility"), "toy", "org/library" @@ -341,7 +524,7 @@ def test_matching_preserves_whitespace(source: Path, different: str) -> None: downstream = _commit(source, "VALUE = 1\n") _git(source, "checkout", "-q", "-b", "upstream", base) upstream = _commit(source, different) - with pytest.raises(ValueError, match="Author action required"): + with pytest.raises(ValueError, match="Upstream attribution is unresolved"): m.match(source, base, downstream, {42: _pr(base, upstream)}) @@ -372,7 +555,7 @@ def test_hunk_labels_are_not_part_of_edit_identity(source: Path) -> None: assignments, evidence = m.match(source, base, downstream, {42: _pr(base, upstream)}) assert assignments == {downstream: 42} assert evidence[downstream]["method"] == "same-edits" - with pytest.raises(ValueError, match="#41, #42"): + with pytest.raises(ValueError, match="#41.*#42"): m.match(source, base, downstream, {41: _pr(base, downstream), 42: _pr(base, upstream)}) @@ -405,7 +588,7 @@ def test_matching_squashed_series_and_ambiguous_duplicates(source: Path) -> None assignments, evidence = m.match(source, base, second, {42: _pr(base, squash)}) assert assignments == {first: 42, second: 42} assert evidence[first]["method"] == "source-series-aggregate" - with pytest.raises(ValueError, match="#42, #43"): + with pytest.raises(ValueError, match="#42.*#43"): m.match(source, base, second, {42: _pr(base, squash), 43: _pr(base, squash)}) @@ -480,7 +663,7 @@ def test_multiple_similar_prs_remain_ambiguous(source: Path) -> None: downstream = _commit(source, "".join(lines)) _git(source, "checkout", "-q", "-b", "upstream", base) upstream = _commit(source, "".join(lines[:-1]) + "adapted\n") - with pytest.raises(ValueError, match="#41, #42") as failure: + with pytest.raises(ValueError, match="#41.*#42") as failure: m.match(source, base, downstream, {41: _pr(base, upstream), 42: _pr(base, upstream)}) assert "#41: 95.00%" in str(failure.value) assert "#42: 95.00%" in str(failure.value) @@ -603,6 +786,7 @@ def monitor(tmp_path: Path): args = argparse.Namespace( canonical_repo="operator/library", canonical_branch="dev", + migrate_canonical_branch_from=None, fork="operator/consumer", repo=tmp_path, workdir=tmp_path, @@ -617,13 +801,236 @@ def monitor(tmp_path: Path): b._STOP.clear() +@pytest.fixture +def migration(monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch): + args = copy.copy(monitor.args) + args.canonical_branch = "dev-new" + args.migrate_canonical_branch_from = "dev" + pin = _vendor("b" * 40, "https://github.com/operator/library.git", "dev-new") + monkeypatch.setattr(p, "_main_sha", lambda gh: "c" * 40) + monkeypatch.setattr(p, "_lock_at", lambda *args: pin) + original_get = monitor.gh.get + monkeypatch.setattr( + monitor.gh, + "get", + lambda endpoint: {"object": {"sha": "b" * 40}} + if "/git/ref/" in endpoint + else original_get(endpoint), + ) + monkeypatch.setattr(monitor.gh, "compare", lambda *args: {"status": "identical"}, raising=False) + monkeypatch.setattr(b, "_pages", lambda *args: iter([])) + monkeypatch.setattr(p, "_git", lambda *args: "") + return args, pin + + +def test_branch_migration_preserves_workdir_and_reconciles_again( + monitor: b.Monitor, + migration, + caplog: pytest.LogCaptureFixture, +) -> None: + caplog.set_level(logging.INFO, logger="vendor-bot") + args, _ = migration + store = monitor.store + for key, value in { + "complete:7": "old-pr", + "promotion:8": 88, + "observation:7": {"state": "completed"}, + "problem:8": "old problem", + "bootstrapped": True, + "cursor": "old cursor", + "last_poll": "old", + "health": {"last_successful_poll": "old", "last_completed_poll": "old"}, + "tracked": [7, 8], + "candidate:7": {"head": "a" * 40, "relevant": True}, + }.items(): + store.put(key, value) + before_since = store.get("since") + (args.workdir / "bot.log").write_text("previous log\n") + restarted = b.Monitor(args, monitor.gh, store) + assert store.get("settings")["canonical_branch"] == "dev-new" + assert store.get("complete:7") is None + assert store.get("promotion:8") is None + assert store.get("problem:8") is None + assert store.get("cursor") is None and store.get("bootstrapped") is None + assert store.get("tracked") == [7, 8] + assert store.get("candidate:7")["relevant"] + assert store.get("since") == before_since + assert store.get("branch_migrations")[0]["previous_results"]["complete:7"] == "old-pr" + assert store.get("migration")["consumer_revision"] == "c" * 40 + assert restarted.progress.state["last_successful_poll"] is None + assert restarted.progress.state["last_completed_poll"] is None + assert (args.workdir / "bot.log").read_text() == "previous log\n" + assert not monitor.gh.writes + assert "dev -> dev-new; rediscovery required" in caplog.text + status = b._status(args.workdir) + assert status["canonical_branch"] == "dev-new" + assert status["migration"]["from"] == "dev" + + +def test_branch_migration_retry_does_not_reset_new_progress(monitor: b.Monitor, migration) -> None: + args, _ = migration + b.Monitor(args, monitor.gh, monitor.store) + monitor.store.put("cursor", "new progress") + b.Monitor(args, monitor.gh, monitor.store) + assert monitor.store.get("cursor") == "new progress" + assert len(monitor.store.get("branch_migrations")) == 1 + args.migrate_canonical_branch_from = None + b.Monitor(args, monitor.gh, monitor.store) + assert monitor.store.get("cursor") == "new progress" + + +@pytest.mark.parametrize("old", [None, "wrong"]) +def test_branch_migration_requires_explicit_old_branch(monitor: b.Monitor, migration, old) -> None: + args, _ = migration + args.migrate_canonical_branch_from = old + with pytest.raises(ValueError, match="--migrate-canonical-branch-from dev"): + b.Monitor(args, monitor.gh, monitor.store) + assert monitor.store.get("settings")["canonical_branch"] == "dev" + + +@pytest.mark.parametrize("key", ["canonical_repo", "fork", "repo"]) +def test_branch_migration_cannot_change_identity(monitor: b.Monitor, migration, key: str) -> None: + args, _ = migration + setattr(args, key, "other/value") + with pytest.raises(ValueError, match="different operator/vendor"): + b.Monitor(args, monitor.gh, monitor.store) + assert monitor.store.get("settings")["canonical_branch"] == "dev" + + +@pytest.mark.parametrize( + "branch,url", + [ + ("dev", "https://github.com/operator/library.git"), + ("dev-new", "https://github.com/other/library.git"), + ], +) +def test_branch_migration_requires_merged_refresh( + monitor: b.Monitor, + migration, + monkeypatch: pytest.MonkeyPatch, + branch: str, + url: str, +) -> None: + args, _ = migration + monkeypatch.setattr(p, "_lock_at", lambda *args: _vendor("b" * 40, url, branch)) + with pytest.raises(ValueError, match="merged base-branch lock"): + b.Monitor(args, monitor.gh, monitor.store) + assert monitor.store.get("settings")["canonical_branch"] == "dev" + + +@pytest.mark.parametrize("status", ["behind", "diverged"]) +def test_branch_migration_rejects_unreachable_pin( + monitor: b.Monitor, + migration, + monkeypatch: pytest.MonkeyPatch, + status: str, +) -> None: + args, _ = migration + monkeypatch.setattr(monitor.gh, "compare", lambda *args: {"status": status}) + with pytest.raises(ValueError, match="does not contain"): + b.Monitor(args, monitor.gh, monitor.store) + + +def test_branch_migration_rejects_lock_race( + monitor: b.Monitor, + migration, + monkeypatch: pytest.MonkeyPatch, +) -> None: + args, pin = migration + locks = iter([pin, dataclasses.replace(pin, commit="d" * 40)]) + monkeypatch.setattr(p, "_lock_at", lambda *args: next(locks)) + with pytest.raises(ValueError, match="changed during migration"): + b.Monitor(args, monitor.gh, monitor.store) + assert monitor.store.get("settings")["canonical_branch"] == "dev" + + +@pytest.mark.parametrize("state", ["open", "absent", "closed"]) +@pytest.mark.parametrize("origin", ["remote", "local", "pr"]) +def test_branch_migration_checks_pending_promotions( + monitor: b.Monitor, + migration, + monkeypatch: pytest.MonkeyPatch, + state: str, + origin: str, +) -> None: + args, _ = migration + branch = "chore/toy-promote-7-aaaaaaaaaaaa" + pr = { + "head": {"ref": branch, "sha": "a" * 40}, + "base": {"ref": "main"}, + "state": state, + "html_url": "https://github.com/org/consumer/pull/88", + } + + def pages(gh, endpoint): + if endpoint.endswith("/branches"): + return iter( + [{"name": branch, "commit": {"sha": "a" * 40}}] if origin == "remote" else [] + ) + if "state=open" in endpoint: + return iter([pr] if origin == "pr" else []) + return iter([pr] if state != "absent" else []) + + monkeypatch.setattr(b, "_pages", pages) + monkeypatch.setattr( + p, "_git", lambda *args: f"{branch} {'a' * 40}" if origin == "local" else "" + ) + monkeypatch.setattr( + monitor.gh, "optional", lambda endpoint: {"commit": {"message": "record"}}, raising=False + ) + monkeypatch.setattr( + p, "_record_from_message", lambda message: {"vendor": "toy", "canonical_branch": "dev"} + ) + if state == "closed": + b.Monitor(args, monitor.gh, monitor.store) + else: + with pytest.raises(ValueError, match="Unresolved old-branch promotion"): + b.Monitor(args, monitor.gh, monitor.store) + assert monitor.store.get("settings")["canonical_branch"] == "dev" + assert not monitor.gh.writes + + +def test_branch_migration_sqlite_transaction_rolls_back(monitor: b.Monitor, migration) -> None: + import sqlite3 + + args, _ = migration + monitor.store.put("cursor", "preserved") + monitor.store.put("complete:7", "preserved") + monitor.store.connection.execute( + "CREATE TRIGGER fail_migration BEFORE INSERT ON state WHEN NEW.key = 'migration' " + "BEGIN SELECT RAISE(ABORT, 'injected failure'); END" + ) + with pytest.raises(sqlite3.IntegrityError, match="injected failure"): + b.Monitor(args, monitor.gh, monitor.store) + assert monitor.store.get("settings")["canonical_branch"] == "dev" + assert monitor.store.get("cursor") == "preserved" + assert monitor.store.get("complete:7") == "preserved" + assert monitor.store.get("branch_migrations") is None + monitor.store.connection.execute("DROP TRIGGER fail_migration") + b.Monitor(args, monitor.gh, monitor.store) + + +def test_branch_migration_cache_loss_uses_normal_start( + monitor: b.Monitor, migration, tmp_path: Path +) -> None: + args, _ = migration + with contextlib.closing(b.Store(tmp_path / "fresh.sqlite")) as store: + with pytest.raises(ValueError, match="fresh/cache-loss start"): + b.Monitor(args, monitor.gh, store) + args.migrate_canonical_branch_from = None + b.Monitor(args, monitor.gh, store) + assert store.get("settings")["canonical_branch"] == "dev-new" + assert store.get("bootstrapped") is None + + def test_feedback_requests_changes_once_and_clears_only_metadata_review(monitor: b.Monitor) -> None: gh = monitor.gh - monitor._feedback(gh.pr, "Missing marked block") - monitor._feedback(gh.pr, "Missing marked block") + monitor._feedback(gh.pr, _issue()) + monitor._feedback(gh.pr, _issue()) assert len(gh.comments) == len(gh.reviews) == 1 assert "PR description" in gh.comments[0]["body"] - assert "resolutions:" in gh.comments[0]["body"] + assert "resolutions:" not in gh.comments[0]["body"] + assert "**Fix:**" in gh.comments[0]["body"] assert gh.writes[1][2]["event"] == "REQUEST_CHANGES" monitor._feedback(gh.pr, None) assert gh.reviews[0]["state"] == "DISMISSED" @@ -639,7 +1046,7 @@ def test_bot_does_not_override_operator_manual_review(monitor: b.Monitor) -> Non "state": "CHANGES_REQUESTED", } ) - monitor._feedback(monitor.gh.pr, "Missing metadata") + monitor._feedback(monitor.gh.pr, _issue("Missing metadata")) monitor._feedback(monitor.gh.pr, None) assert monitor.gh.reviews[0]["state"] == "CHANGES_REQUESTED" assert len(monitor.gh.reviews) == 1 @@ -647,11 +1054,11 @@ def test_bot_does_not_override_operator_manual_review(monitor: b.Monitor) -> Non def test_dry_run_and_self_review(monitor: b.Monitor) -> None: monitor.args.publish = False - monitor._feedback(monitor.gh.pr, "Missing metadata") + monitor._feedback(monitor.gh.pr, _issue("Missing metadata")) assert not monitor.gh.writes monitor.args.publish = True monitor.gh.pr["user"]["login"] = "operator" - monitor._feedback(monitor.gh.pr, "Missing metadata") + monitor._feedback(monitor.gh.pr, _issue("Missing metadata")) assert monitor.gh.comments assert not monitor.gh.reviews @@ -659,8 +1066,204 @@ def test_dry_run_and_self_review(monitor: b.Monitor) -> None: def test_feedback_rechecks_mutable_description(monitor: b.Monitor) -> None: stale = copy.deepcopy(monitor.gh.pr) monitor.gh.pr["body"] = "updated while checking" - monitor._feedback(stale, "Missing metadata") + monitor._feedback(stale, _issue("Missing metadata")) + assert not monitor.gh.writes + + +@pytest.mark.parametrize("missing", [(m.BEGIN, m.END), (m.BEGIN,), (m.END,)]) +def test_posted_marker_feedback_is_specific_and_recovers_without_cache( + monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch, missing: tuple[str, ...] +) -> None: + previous = _vendor("a" * 40, "https://github.com/operator/library.git", "dev") + reviewed = _vendor("b" * 40, "https://github.com/author/library.git", "temporary") + monkeypatch.setattr(monitor, "_inputs", lambda pr: (previous, reviewed)) + valid = m.template("toy", "org/library") + monitor.gh.pr["body"] = valid + for marker in missing: + monitor.gh.pr["body"] = monitor.gh.pr["body"].replace(marker, "") + monitor.inspect(7) + body = monitor.gh.comments[0]["body"] + for marker in missing: + assert f"`{marker}`" in body + assert "**Fix:**" in body and "PR's description—not a comment" in body + assert "resolutions:" not in body and "exactly one" not in body + assert monitor.gh.reviews[0]["state"] == "CHANGES_REQUESTED" + writes = copy.deepcopy(monitor.gh.writes) + with contextlib.closing(b.Store(":memory:")) as store: + restarted = b.Monitor(monitor.args, monitor.gh, store) + monkeypatch.setattr(restarted, "_inputs", lambda pr: (previous, reviewed)) + restarted.inspect(7) + assert monitor.gh.writes == writes + monitor.gh.pr["body"] = valid + monkeypatch.setattr(m, "resolve", lambda *args: ({reviewed.commit: 123}, {})) + restarted.inspect(7) + assert len(monitor.gh.comments) == 1 + assert monitor.gh.reviews[0]["state"] == "DISMISSED" + assert not any(payload.get("event") == "APPROVE" for _, _, payload in monitor.gh.writes) + + +@pytest.mark.parametrize("stage", ["inputs", "resolve"]) +@pytest.mark.parametrize("error_type", [ValueError, RuntimeError, b.manage.VendorError]) +def test_unrecognized_failures_are_operator_errors_not_author_feedback( + monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch, stage: str, error_type: type[Exception] +) -> None: + previous = _vendor("a" * 40, "https://github.com/operator/library.git", "dev") + reviewed = _vendor("b" * 40, "https://github.com/author/library.git", "temporary") + monitor.gh.pr["body"] = m.template("toy", "org/library") + monkeypatch.setattr(monitor, "_inputs", lambda pr: (previous, reviewed)) + + def fail(*args): + raise error_type("Operator failure; do not request author changes") + + if stage == "inputs": + monkeypatch.setattr(monitor, "_inputs", fail) + else: + monkeypatch.setattr(m, "resolve", fail) + monitor.poll() assert not monitor.gh.writes + health = monitor.store.get("health") + assert health["counts"]["errors"] == 1 + assert "Operator failure" in health["last_error"]["message"] + + +def test_waiting_for_prior_promotion_is_maintainer_action_without_review( + monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch +) -> None: + previous = _vendor("a" * 40, "https://github.com/previous/library.git", "prior-update") + reviewed = _vendor("b" * 40, "https://github.com/author/library.git", "temporary") + monkeypatch.setattr(p, "_main_sha", lambda gh: "c" * 40) + monkeypatch.setattr( + p, + "_lock_document_at", + lambda gh, repo, sha: { + "vendors": {"toy": (previous if sha == "c" * 40 else reviewed).to_mapping()} + }, + ) + monitor.inspect(7) + body = monitor.gh.comments[0]["body"] + assert "needs maintainer action" in body and "Waiting for the preceding" in body + assert "operator/library" in body and "dev" in body + assert "After it merges" in body and "No metadata edit" in body + assert "```yaml" not in body and "resolutions:" not in body + assert not monitor.gh.reviews + writes = copy.deepcopy(monitor.gh.writes) + monitor.inspect(7) + assert monitor.gh.writes == writes + + +def test_invalid_pr_lock_has_file_fix_not_metadata_template( + monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch +) -> None: + previous = _vendor("a" * 40, "https://github.com/operator/library.git", "dev") + invalid = previous.to_mapping() | {"commit": "short-sha"} + monkeypatch.setattr(p, "_main_sha", lambda gh: "c" * 40) + monkeypatch.setattr( + p, + "_lock_document_at", + lambda gh, repo, sha: { + "vendors": {"toy": previous.to_mapping() if sha == "c" * 40 else invalid} + }, + ) + monitor.inspect(7) + body = monitor.gh.comments[0]["body"] + assert "vendors.toy" in body and "40-hex SHA" in body + assert p._LOCK in body and "Editing the PR description will not fix" in body + assert "```yaml" not in body + assert monitor.gh.reviews[0]["state"] == "CHANGES_REQUESTED" + + +@pytest.mark.parametrize("history", ["diverged", "unrelated", "merge"]) +def test_source_history_diagnostic_has_canonical_pin_and_regeneration_steps( + source: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, history: str +) -> None: + base = _commit(source, "base\n", "src/code.py") + canonical = _commit(source, "canonical\n", "canonical.py") + _git(source, "checkout", "-q", "-b", "temporary", base) + if history == "unrelated": + _git(source, "checkout", "-q", "--orphan", "unrelated") + reviewed_sha = _commit(source, "change\n", "src/code.py") + if history == "merge": + _git(source, "merge", "--no-ff", "-m", "merge canonical", canonical) + reviewed_sha = _git(source, "rev-parse", "HEAD") + previous = _vendor(canonical, "https://github.com/operator/library.git", "dev") + reviewed = _vendor(reviewed_sha, "https://github.com/author/library.git", "temporary") + original_git = p._git + + def local_git(repo, *args, **kwargs): + if args and args[0] == "fetch": + args = tuple(str(source) if arg == reviewed.url else arg for arg in args) + return original_git(repo, *args, **kwargs) + + monkeypatch.setattr(p, "_git", local_git) + monkeypatch.setattr(p, "_check_source_repo", lambda *args: None) + with pytest.raises(m.Diagnostic) as failure: + m.resolve( + FakeGitHub(), + previous, + reviewed, + {"unpaired_reason": "downstream-only", "upstream_prs": [], "resolutions": {}}, + tmp_path / "cache.git", + ) + issue = failure.value + assert { + "diverged": "not a fast-forward", + "unrelated": "no merge base", + "merge": "linear history", + }[history] in issue.problem + assert ( + canonical in issue.action and "operator/library" in issue.action and "dev" in issue.action + ) + assert "temporary" in issue.action and "scripts/vendor/manage.py" in issue.action + assert "vendored snapshot" in issue.action and "push" in issue.action + assert not issue.example + + +@pytest.mark.parametrize("status", ["closed", "wrong-target"]) +def test_upstream_pr_diagnostic_links_affected_pr_and_fix( + source: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, status: str +) -> None: + base = _commit(source, "base\n", "src/code.py") + head = _commit(source, "changed\n", "src/code.py") + previous = _vendor(base, "https://github.com/operator/library.git", "dev") + reviewed = _vendor(head, "https://github.com/author/library.git", "temporary") + gh = FakeGitHub() + pr = _pr(base, head) + if status == "closed": + pr["state"] = "closed" + else: + pr["base"]["ref"] = "release" + monkeypatch.setattr(gh, "get", lambda endpoint: pr) + monkeypatch.setattr(p, "_check_source_repo", lambda *args: None) + original_git = p._git + + def local_git(repo, *args, **kwargs): + if args and args[0] == "fetch": + args = tuple(str(source) if arg == reviewed.url else arg for arg in args) + return original_git(repo, *args, **kwargs) + + monkeypatch.setattr(p, "_git", local_git) + with pytest.raises(m.Diagnostic) as failure: + m.resolve( + gh, + previous, + reviewed, + {"unpaired_reason": None, "upstream_prs": [42], "resolutions": {}}, + tmp_path / "cache.git", + ) + assert "https://github.com/org/library/pull/42" in failure.value.problem + assert ( + "closed without merging" in failure.value.problem + if status == "closed" + else "release" in failure.value.problem + ) + assert "reopen" in failure.value.action and "replace its URL" in failure.value.action + assert not failure.value.example + + +def test_git_expected_absence_does_not_hide_unexpected_failures(source: Path) -> None: + base = _commit(source, "base\n") + with pytest.raises(RuntimeError, match="git failed"): + p._git(source, "merge-base", base, "f" * 40, allowed_returncodes=(0, 1)) def test_discovery_and_persistent_operator_scope(monitor: b.Monitor) -> None: @@ -751,11 +1354,11 @@ def test_monitor_posts_similarity_and_deduplicates_feedback( def test_feedback_deduplicates_with_empty_sqlite(monitor: b.Monitor) -> None: - monitor._feedback(monitor.gh.pr, "Missing block") + monitor._feedback(monitor.gh.pr, _issue("Missing block")) writes = copy.deepcopy(monitor.gh.writes) with contextlib.closing(b.Store(":memory:")) as store: restarted = b.Monitor(monitor.args, monitor.gh, store) - restarted._feedback(monitor.gh.pr, "Missing block") + restarted._feedback(monitor.gh.pr, _issue("Missing block")) assert monitor.gh.writes == writes restarted._feedback(monitor.gh.pr, None) assert len(monitor.gh.comments) == len(monitor.gh.reviews) == 1 @@ -855,7 +1458,7 @@ def test_monitor_reports_attribution_to_author_and_retries_after_correction( monitor.gh.pr["body"] = _body(" upstream_prs: [https://github.com/org/library/pull/42]") def unresolved(*args): - raise ValueError("Author action required: no exact match for " + reviewed.commit) + raise _issue("Author action required: no exact match for " + reviewed.commit) monkeypatch.setattr(m, "resolve", unresolved) monitor.inspect(7) @@ -871,7 +1474,7 @@ def test_forbidden_dismissal_informs_author_without_approving( monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch, ) -> None: - monitor._feedback(monitor.gh.pr, "Missing block") + monitor._feedback(monitor.gh.pr, _issue("Missing block")) original = monitor.gh.api def forbidden(endpoint, **kwargs): @@ -1062,8 +1665,8 @@ def test_metadata_transitions_are_logged_once_and_clear_problem( ) -> None: caplog.set_level(logging.INFO, logger="vendor-bot") monitor.args.publish = publish - monitor._feedback(monitor.gh.pr, "Missing marked block") - monitor._feedback(monitor.gh.pr, "Missing marked block") + monitor._feedback(monitor.gh.pr, _issue()) + monitor._feedback(monitor.gh.pr, _issue()) assert caplog.text.count("PR #7: blocked: Missing marked block") == 1 assert monitor.store.get("problem:7") == "Missing marked block" monitor._feedback(monitor.gh.pr, None) From 718ac7e37484d4a7769d25ba65b8c2147ddacbf0 Mon Sep 17 00:00:00 2001 From: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> Date: Tue, 13 Oct 2026 08:21:38 +0000 Subject: [PATCH 07/11] [None][fix] Classify vendor updates against the PR merge base Signed-off-by: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> --- scripts/vendor/README.md | 7 ++++ scripts/vendor/bot.py | 19 ++++++++++ tests/unittest/others/test_vendor_bot.py | 45 ++++++++++++++++++++++++ 3 files changed, 71 insertions(+) diff --git a/scripts/vendor/README.md b/scripts/vendor/README.md index d070fefc6444..0f59ee071bfd 100644 --- a/scripts/vendor/README.md +++ b/scripts/vendor/README.md @@ -246,6 +246,13 @@ same cutoff on every launch, including recovery. Historical updates belonging to another canonical branch, or superseded pins without a recoverable promotion, are not promoted or sent author feedback. +Open PRs qualify only when they change the selected vendor's source commit +relative to their Git merge base. A stale PR head after a canonical refresh is +not itself a source update: changes only to other vendors or patch/digest fields +do not require promotion metadata. Eligible updates are still checked against +the latest canonical lock, so their source branches must include the refreshed +canonical history before promotion. + ### Monitoring health `--log-level INFO` is the default, in both foreground and daemon mode. Logs use diff --git a/scripts/vendor/bot.py b/scripts/vendor/bot.py index f3f858e4c29e..5831ee288b0a 100644 --- a/scripts/vendor/bot.py +++ b/scripts/vendor/bot.py @@ -593,6 +593,25 @@ def _inputs(self, pr: dict) -> tuple[manage.Vendor, manage.Vendor] | None: ) from error if previous.commit == reviewed.commit or not reviewed.branch: return None # Includes lock-only promotions and tag-only updates. + if not pr.get("merged"): + comparison = self.gh.compare(self.gh.consumer_repo, base, revision) + merge_base = promote._sha(comparison["merge_base_commit"]["sha"]) + try: + original = ( + before + if merge_base == base + else promote._lock_document_at(self.gh, self.gh.consumer_repo, merge_base)[ + "vendors" + ] + ) + except Diagnostic as error: + raise RuntimeError(f"PR merge-base vendor lock is invalid: {error}") from error + # Classify the PR's own changes, but retain the current canonical + # pin as the baseline for source-history checks and promotion. + if name not in original or ( + manage._validate_vendor(name, original[name]).commit == reviewed.commit + ): + return None canonical = ( promote._repo_name(previous.url).lower() == self.args.canonical_repo.lower() and previous.branch == self.args.canonical_branch diff --git a/tests/unittest/others/test_vendor_bot.py b/tests/unittest/others/test_vendor_bot.py index e69166fb3050..6dff26cf8feb 100644 --- a/tests/unittest/others/test_vendor_bot.py +++ b/tests/unittest/others/test_vendor_bot.py @@ -732,6 +732,7 @@ def __init__(self) -> None: self.comments: list[dict] = [] self.reviews: list[dict] = [] self.writes: list[tuple[str, str, dict]] = [] + self.merge_base_sha = "c" * 40 def api( self, endpoint: str, *, method: str = "GET", payload: dict | None = None @@ -740,6 +741,8 @@ def api( self.writes.append((endpoint, method, payload)) if endpoint == "user": return {"login": "operator"} + if endpoint.startswith("repos/org/consumer/compare/"): + return {"merge_base_commit": {"sha": self.merge_base_sha}} if endpoint.endswith("/pulls/7"): return copy.deepcopy(self.pr) if "/pulls?" in endpoint: @@ -1126,6 +1129,48 @@ def fail(*args): assert "Operator failure" in health["last_error"]["message"] +@pytest.mark.parametrize("change", ["other-vendor", "patch-only", "vendor-created"]) +def test_refresh_ignores_prs_without_selected_source_pin_change( + monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch, change: str +) -> None: + current = _vendor("1" * 40, "https://github.com/operator/library.git", "dev") + old = _vendor("2" * 40, "https://github.com/operator/library.git", "dev-old") + original = {"toy": old.to_mapping()} + reviewed = copy.deepcopy(original) + if change == "other-vendor": + reviewed["other"] = current.to_mapping() + elif change == "patch-only": + reviewed["toy"]["digest"] = "sha256-tree-v1:" + "3" * 64 + else: + original = {} + monitor.gh.merge_base_sha = "d" * 40 + locks = {"c" * 40: {"toy": current.to_mapping()}, "d" * 40: original, "a" * 40: reviewed} + monkeypatch.setattr(p, "_main_sha", lambda gh: "c" * 40) + monkeypatch.setattr(p, "_lock_document_at", lambda gh, repo, sha: {"vendors": locks[sha]}) + monitor.inspect(7) + assert not monitor.gh.writes + assert monitor._inputs(monitor.gh.pr) is None + with contextlib.closing(b.Store(monitor.args.workdir / "fresh.sqlite")) as store: + b.Monitor(monitor.args, monitor.gh, store).inspect(7) + assert not monitor.gh.writes + + +def test_refresh_source_update_uses_current_canonical_as_promotion_baseline( + monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch +) -> None: + current = _vendor("1" * 40, "https://github.com/operator/library.git", "dev") + old = _vendor("2" * 40, "https://github.com/operator/library.git", "dev-old") + reviewed = _vendor("3" * 40, "https://github.com/author/library.git", "temporary") + monitor.gh.merge_base_sha = "d" * 40 + locks = {"c" * 40: current, "d" * 40: old, "a" * 40: reviewed} + monkeypatch.setattr(p, "_main_sha", lambda gh: "c" * 40) + monkeypatch.setattr( + p, "_lock_document_at", lambda gh, repo, sha: {"vendors": {"toy": locks[sha].to_mapping()}} + ) + assert monitor._inputs(monitor.gh.pr) == (current, reviewed) + assert not monitor.gh.writes + + def test_waiting_for_prior_promotion_is_maintainer_action_without_review( monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch ) -> None: From d68c1168a889003becdf8def2982fa80f29a5e46 Mon Sep 17 00:00:00 2001 From: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> Date: Fri, 16 Oct 2026 04:47:25 +0000 Subject: [PATCH 08/11] [None][fix] Scope vendor bot monitoring and reconcile obsolete feedback Signed-off-by: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> --- AGENTS.md | 7 + scripts/vendor/README.md | 67 ++- scripts/vendor/bot.py | 230 +++++++-- .../unittest/others/test_maintain_prims_ts.py | 3 + tests/unittest/others/test_vendor_bot.py | 487 +++++++++++++++++- 5 files changed, 727 insertions(+), 67 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b9fd33e230f3..abf96ef3f009 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -216,6 +216,13 @@ the new `--canonical-branch` plus `--migrate-canonical-branch-from OLD_BRANCH`. Start with `--once` without `--publish`; migration verifies the merged lock, blocks unresolved old promotions, and atomically archives branch-dependent cache results before rediscovery. Other identity changes still require another workdir. +Each monitor handles one consumer base branch (`--base-branch`, default `main`); +use separate workdirs for main and release monitors. Launching one never enables +the other. Before a refresh PR, stop its monitor or restart with the explicitly +audited `--refresh-branch NEW_BRANCH` in the configured canonical repository. +This repeatable option excludes refresh PRs from promotion; it does not authorize +publication to a new branch. After merge, the explicit migration is still required. +Migration checks ignore promotion records for other consumer repositories/bases. After cache loss, use the new configuration without the migration flag. For monitoring, use INFO poll summaries/30-second active-operation heartbeats and `bot.py status --workdir ...`. Check the last poll outcome and next deadline; diff --git a/scripts/vendor/README.md b/scripts/vendor/README.md index 0f59ee071bfd..1c10f20eb456 100644 --- a/scripts/vendor/README.md +++ b/scripts/vendor/README.md @@ -197,12 +197,49 @@ crash or reboot; a supervisor is optional. Stop before changing launch settings. The workdir must be empty on first use, owned by the current user, and outside a Git worktree. It holds a disposable SQLite cache, a bare source cache, process-lock and stop files, and isolated promotion worktrees. Runtime state is not committed. -An existing workdir is bound to its operator and vendor configuration; use a -different directory for another operator/vendor. Authentication is inherited +An existing workdir is bound to its operator, vendor, and consumer base branch; +use a different directory for another operator/vendor/base. Authentication is inherited from the launching environment and is not persisted in that state. +### Different consumer base branches + +Each monitor handles one `--base-branch` (default `main`) and one +`--canonical-branch`. Different consumer branches may use different canonical +source branches: run separate instances with separate workdirs. Discovery, +promotion provenance, follow-up PR targets, and migration checks are scoped to +the consumer repository and base branch. Retargeting a tracked PR makes it +ineligible for its old monitor; changing `--base-branch` requires a new workdir. + +For example, a main monitor can use `--base-branch main --canonical-branch +consumer-dev-new`, while a release monitor uses `--base-branch release/1.3 +--canonical-branch consumer-dev`. These are independent opt-in launches; +configuring one does not start the other. If both consumers later share one +canonical source branch, serialize publication across them: separate workdir +locks do not coordinate concurrent pushes to the same source branch. + ### Switching canonical branches after a refresh +Before opening a refresh PR, either stop the monitor for its target base branch +until the refresh merges, or restart it with an explicit refresh target: + +```bash +--canonical-branch consumer-dev \ +--refresh-branch consumer-dev-YYYYMMDD +``` + +`--refresh-branch` is repeatable and applies only to `--canonical-repo`. It tells +the monitor that a PR pointing at that branch is a manual refresh, not a +temporary-branch update requiring promotion metadata. The monitor does not +promote it, skip its CI, or infer refresh targets from contributor metadata or +branch naming conventions. Normal review and CI still apply to the refresh. +The operator must audit these targets; do not list contributor temporary branches. +The option does not authorize publishing to a new branch or start another monitor. +Changing this policy on restart is allowed in the same workdir without migrating +the canonical configuration. Reuse the policy after cache loss. If the monitor +previously requested metadata changes on the open refresh PR, it updates its +existing feedback and dismisses its obsolete automated review when authorized. +If dismissal is forbidden, the comment asks an authorized maintainer to dismiss it. + Keep the same workdir when a periodic source refresh introduces a new canonical branch. Stop the publishing process for the cutover, and wait for the consumer refresh PR to merge. Relaunch with the same configuration except for: @@ -217,10 +254,17 @@ refresh PR to merge. Relaunch with the same configuration except for: First omit `--publish`: migration only changes local configuration/cache state; this pass makes no remote writes. The bot verifies the merged base lock names the new canonical repository/branch and its pin is reachable from that branch. -It refuses outstanding old-branch promotion PRs or unpublished promotion branches. +It refuses outstanding old-branch promotion PRs or unpublished promotion branches +for the same consumer repository and base branch. Promotions for other consumer +bases do not block this migration; missing or malformed scope provenance still +requires maintainer inspection. Finish those operations, or explicitly close superseded follow-up PRs after maintainer review; migration never deletes branches or redirects old promotions. Other operator/vendor identity changes still require a separate workdir. +If the refresh has merged but the old monitor is still running, new updates based +on a configured refresh target report a maintainer cutover action, not a request +to promote the lock back to the old canonical branch. After migration, the +`--refresh-branch` entry for the now-current canonical branch can be omitted. The configuration update and invalidation are one SQLite transaction. Old branch-dependent results are archived, while logs, source caches, promotion @@ -246,7 +290,7 @@ same cutoff on every launch, including recovery. Historical updates belonging to another canonical branch, or superseded pins without a recoverable promotion, are not promoted or sent author feedback. -Open PRs qualify only when they change the selected vendor's source commit +Open PRs qualify for promotion only when they change the selected vendor's source commit relative to their Git merge base. A stale PR head after a canonical refresh is not itself a source update: changes only to other vendors or patch/digest fields do not require promotion metadata. Eligible updates are still checked against @@ -360,6 +404,7 @@ rechecks automatically on later polls; no manual recheck comment is required. | Invalid upstream PR or resolution | Identifies the list entry, duplicated PR, source SHA, missing reason, or unlisted PR, and gives the correction. | | Unresolved attribution | Shows candidate PRs and similarity scores where applicable; explains how to correct the PR list/source pin or supply a targeted author assertion. | | Source-history or lock-selection problem | Explains the rebase or lock-file change and vendor regeneration needed. Rebase guidance identifies the canonical pin. Metadata edits alone cannot fix these failures. | +| Same-commit temporary reference | If the PR redirects the source URL/branch but pins the current canonical commit, asks the author to restore the canonical URL/branch from its target base. Keep the commit, other lock fields, vendored files, and patch unchanged; no upstream-PR metadata is needed. | `unpaired_reason` is appropriate only when there genuinely is no paired upstream PR, not as a workaround for failed matching. The similarity threshold, matching @@ -370,11 +415,19 @@ author-actionable problem. Waiting for a preceding promotion is identified as **maintainer action**, with a comment but no new change-request review. Unexpected API, authentication, network, and local-cache failures stay in operator logs and `status`; they do not post author-fix instructions or request changes. The bot -deduplicates feedback, including after SQLite loss, and never submits `APPROVE` -just because metadata is valid. +deduplicates feedback, including after SQLite loss, updates its existing review +body when the author-actionable problem changes, and never submits `APPROVE` +just because metadata is valid. Same-commit reference checks compare the PR's +own changes against its Git merge base; inherited references and explicitly +configured refresh targets are not treated as unnecessary temporary redirects. After correction it dismisses only its own marked metadata reviews, when -authorized. If dismissal is forbidden, an authorized human must dismiss the +authorized. If an open PR no longer needs promotion, it also reconciles existing +comments/reviews from GitHub, without requiring SQLite history. It updates the +comment to explain that the previous request no longer applies, rather than +claiming metadata validation succeeded. It posts no feedback on an ignored PR +unless it already has a bot comment or an unresolved bot review to reconcile. +If dismissal is forbidden, an authorized human must dismiss the review; the bot reports the failure rather than approving the code. If the authenticated account already has a manual approval/objection, the bot uses comments so it does not supersede that review. It likewise uses comments on the diff --git a/scripts/vendor/bot.py b/scripts/vendor/bot.py index 5831ee288b0a..6df61a6705fc 100644 --- a/scripts/vendor/bot.py +++ b/scripts/vendor/bot.py @@ -193,7 +193,7 @@ def _pages(gh: promote.GitHub, endpoint: str) -> Iterator[dict]: class Monitor: - """One operator-configured vendor; run other vendors in separate workdirs.""" + """One vendor/base-branch pair; run other pairs in separate workdirs.""" def __init__( self, @@ -207,6 +207,7 @@ def __init__( self.actor = gh.get("user")["login"] self.prefix = f"repos/{gh.consumer_repo}" self.marker = f"" + self.refresh_branches = frozenset(args.refresh_branch) settings = { "consumer_repo": gh.consumer_repo, "vendor": gh.vendor_name, @@ -266,6 +267,9 @@ def __init__( "--migrate-canonical-branch-from and use the current canonical configuration." ) store.put("settings", settings) + # Refresh targets are an operator policy, not publication destinations. + # Changing them does not migrate the canonical branch or its provenance. + store.put("refresh_branches", sorted(self.refresh_branches)) if store.get("since") is None: store.put( "since", @@ -344,6 +348,23 @@ def _check_pending_promotions(self, previous_branch: str) -> None: record = promote._record_from_message(message) if record.get("canonical_branch") != previous_branch: continue + consumer_pr = record.get("consumer_pr") + consumer_base = record.get("consumer_base") + consumer = ( + re.fullmatch(r"https://github\.com/([\w.-]+/[\w.-]+)/pull/[1-9][0-9]*", consumer_pr) + if isinstance(consumer_pr, str) + else None + ) + if consumer is None or not isinstance(consumer_base, str) or not consumer_base: + raise ValueError( + f"Missing or invalid consumer provenance on promotion branch {branch}; " + "inspect its promotion commit before migrating." + ) + if ( + consumer.group(1).lower() != self.gh.consumer_repo.lower() + or consumer_base != self.gh.base_branch + ): + continue if record.get("vendor") != self.gh.vendor_name: raise ValueError(f"Unexpected vendor provenance on promotion branch {branch}.") prs = list( @@ -424,9 +445,15 @@ def _transition(self, number: int, state: str, reason: str = "") -> None: self.store.put(f"problem:{number}", observation["reason"] if state == "blocked" else None) def _feedback( - self, pr: dict, problem: Diagnostic | None, *, attribution: dict[str, dict] | None = None + self, + pr: dict, + problem: Diagnostic | None, + *, + attribution: dict[str, dict] | None = None, + no_promotion: bool = False, ) -> None: number = pr["number"] + state = "blocked" if problem else "ignored" if no_promotion else "metadata_valid" message = ( f"@{pr['user']['login']} — vendor promotion needs {problem.owner} action:\n\n{problem}" "\n\nThe bot will recheck automatically on a later poll." @@ -434,6 +461,13 @@ def _feedback( else "Vendor promotion metadata and source attribution are valid. " "This is **not** a code-review approval. Promotion still requires the source PR to merge." ) + if no_promotion: + message = ( + f"This PR does not require vendor promotion for {field(self.gh.vendor_name)} " + "under this monitor's configuration. The previous promotion-metadata request " + "no longer applies; no upstream-PR metadata is required by this monitor. " + "This is **not** a code-review approval. Normal review and CI still apply." + ) if attribution: details = metadata.similarity_feedback(attribution, self.gh.upstream_repo) if details: @@ -446,13 +480,19 @@ def _feedback( if not self.args.publish: self._transition( number, - "blocked" if problem else "metadata_valid", + state, str(problem) if problem else "dry run; no feedback published", ) return # Check immutable head and description immediately before feedback. A # concurrent author edit will be retried on the next poll. current = self.gh.get(f"{self.prefix}/pulls/{number}") + if current["base"]["ref"] != self.gh.base_branch: + self._transition(number, "ignored", "PR targets a different consumer base branch") + return + if no_promotion and current["state"] != "open": + self._transition(number, "ignored", "PR is no longer open") + return if current["head"]["sha"] != pr["head"]["sha"] or current.get("body") != pr.get("body"): self._transition( number, "deferred", "source head or description changed; retry next poll" @@ -466,6 +506,29 @@ def _feedback( ] if len(existing) > 1: raise ValueError("Multiple bot feedback comments; inspect before continuing.") + reviews = ( + [ + item + for item in _pages(self.gh, f"{self.prefix}/pulls/{number}/reviews") + if item["user"]["login"] == self.actor + ] + if current["state"] == "open" and current["user"]["login"] != self.actor + else [] + ) + automated = [ + item + for item in reviews + if item.get("body", "").startswith(self.marker) and item["state"] == "CHANGES_REQUESTED" + ] + manual = [ + item + for item in reviews + if not item.get("body", "").startswith(self.marker) + and item["state"] in ("APPROVED", "CHANGES_REQUESTED") + ] + if no_promotion and not existing and not automated: + self._transition(number, "ignored", "not a promotable update for this canonical vendor") + return if existing: if existing[0]["body"] != body and not ( not problem and existing[0]["body"] == body + dismissal_notice @@ -477,47 +540,43 @@ def _feedback( ) _LOG.info("PR #%s: updated metadata feedback comment", number) else: - self.gh.api( - f"{self.prefix}/issues/{number}/comments", method="POST", payload={"body": body} - ) + existing = [ + self.gh.api( + f"{self.prefix}/issues/{number}/comments", method="POST", payload={"body": body} + ) + ] _LOG.info("PR #%s: posted metadata feedback comment", number) if current["state"] != "open" or current["user"]["login"] == self.actor: self._transition( number, - "blocked" if problem else "metadata_valid", + state, str(problem) if problem else "feedback reconciled; not a code-review approval", ) return - reviews = [ - item - for item in _pages(self.gh, f"{self.prefix}/pulls/{number}/reviews") - if item["user"]["login"] == self.actor - ] - automated = [ - item - for item in reviews - if item.get("body", "").startswith(self.marker) and item["state"] == "CHANGES_REQUESTED" - ] - manual = [ - item - for item in reviews - if not item.get("body", "").startswith(self.marker) - and item["state"] in ("APPROVED", "CHANGES_REQUESTED") - ] - if problem and problem.owner == "author" and not automated and not manual: - self.gh.api( - f"{self.prefix}/pulls/{number}/reviews", - method="POST", - payload={ - "event": "REQUEST_CHANGES", - "commit_id": pr["head"]["sha"], - "body": ( - f"{self.marker}\n{problem.problem}\n\nSee the vendor-bot comment for " - "the specific fix and the latest validation result." - ), - }, + if problem and problem.owner == "author": + review_body = ( + f"{self.marker}\n{problem.problem}\n\nSee the vendor-bot comment for " + "the specific fix and the latest validation result." ) - _LOG.info("PR #%s: requested metadata changes", number) + for review in automated: + if review["body"] != review_body: + self.gh.api( + f"{self.prefix}/pulls/{number}/reviews/{review['id']}", + method="PUT", + payload={"body": review_body}, + ) + _LOG.info("PR #%s: updated metadata review %s", number, review["id"]) + if not automated and not manual: + self.gh.api( + f"{self.prefix}/pulls/{number}/reviews", + method="POST", + payload={ + "event": "REQUEST_CHANGES", + "commit_id": pr["head"]["sha"], + "body": review_body, + }, + ) + _LOG.info("PR #%s: requested metadata changes", number) elif not problem: for review in automated: # Never replace a metadata objection with APPROVE or dismiss a @@ -536,7 +595,8 @@ def _feedback( if "HTTP 403" not in str(error): raise _LOG.warning( - "PR #%s: metadata is valid, but an authorized human must dismiss automated review %s.", + "PR #%s: metadata review is resolved, but an authorized human " + "must dismiss automated review %s.", number, review["id"], ) @@ -558,7 +618,7 @@ def _feedback( ) self._transition( number, - "blocked" if problem else "metadata_valid", + state, str(problem) if problem else "feedback reconciled; not a code-review approval", ) @@ -591,7 +651,19 @@ def _inputs(self, pr: dict) -> tuple[manage.Vendor, manage.Vendor] | None: "regenerate the vendored snapshot if its source changed, and push the PR. " "Editing the PR description will not fix a lock-file error.", ) from error - if previous.commit == reviewed.commit or not reviewed.branch: + canonical = ( + promote._repo_name(previous.url).lower() == self.args.canonical_repo.lower() + and previous.branch == self.args.canonical_branch + ) + target_is_canonical = promote._repo_name( + reviewed.url + ).lower() == self.args.canonical_repo.lower() and ( + reviewed.branch == self.args.canonical_branch + or reviewed.branch in self.refresh_branches + ) + if target_is_canonical: + return None # Periodic refresh/direct canonical update: separate workflow. + if not reviewed.branch or (pr.get("merged") and previous.commit == reviewed.commit): return None # Includes lock-only promotions and tag-only updates. if not pr.get("merged"): comparison = self.gh.compare(self.gh.consumer_repo, base, revision) @@ -608,21 +680,42 @@ def _inputs(self, pr: dict) -> tuple[manage.Vendor, manage.Vendor] | None: raise RuntimeError(f"PR merge-base vendor lock is invalid: {error}") from error # Classify the PR's own changes, but retain the current canonical # pin as the baseline for source-history checks and promotion. - if name not in original or ( - manage._validate_vendor(name, original[name]).commit == reviewed.commit - ): + if name not in original: return None - canonical = ( - promote._repo_name(previous.url).lower() == self.args.canonical_repo.lower() - and previous.branch == self.args.canonical_branch - ) - target_is_canonical = ( - promote._repo_name(reviewed.url).lower() == self.args.canonical_repo.lower() - and reviewed.branch == self.args.canonical_branch - ) - if target_is_canonical: - return None # Periodic refresh/direct canonical update: separate workflow. + original_vendor = manage._validate_vendor(name, original[name]) + reference_changed = ( + promote._repo_name(original_vendor.url).lower(), + original_vendor.branch, + ) != (promote._repo_name(reviewed.url).lower(), reviewed.branch) + if canonical and previous.commit == reviewed.commit and reference_changed: + raise Diagnostic( + f"Temporary source reference adds no new vendor commit: " + f"{field(reviewed.commit)} is already pinned on {field(self.gh.base_branch)}.", + f"In `{promote._LOCK}` at `vendors.{name}`, restore `url` to " + f"{field(previous.url)} and `branch` to {field(previous.branch)}. " + "Change only these source-reference fields; keep the commit and other lock " + "fields, vendored files, and patch unchanged. No upstream-PR metadata or " + "vendor promotion is needed for this same-commit reference change.", + ) + if original_vendor.commit == reviewed.commit: + return None + if previous.commit == reviewed.commit: + return None if not canonical: + if ( + promote._repo_name(previous.url).lower() == self.args.canonical_repo.lower() + and previous.branch in self.refresh_branches + ): + raise Diagnostic( + f"The base lock now uses refresh branch {field(previous.branch)}, but this " + f"monitor still publishes to {field(self.args.canonical_branch)}.", + "A maintainer must stop this monitor and migrate its workdir with " + f"`--canonical-branch {previous.branch} " + f"--migrate-canonical-branch-from {self.args.canonical_branch}`. " + "Verify a dry run before resuming publication. Do not promote back to " + "the old branch or change contributor metadata to bypass this cutover.", + owner="maintainer", + ) if pr.get("merged"): return None # Historical update belonging to another canonical branch. raise Diagnostic( @@ -687,15 +780,21 @@ def inspect(self, number: int) -> None: return self.progress.operation("inspecting", number) pr = self.gh.get(f"{self.prefix}/pulls/{number}") + if pr["base"]["ref"] != self.gh.base_branch: + self._transition(number, "ignored", "PR targets a different consumer base branch") + return if pr["state"] == "closed" and not pr.get("merged"): self._transition(number, "ignored", "closed without merging") return try: inputs = self._inputs(pr) if inputs is None: - self._transition( - number, "ignored", "not a promotable update for this canonical vendor" - ) + if pr["state"] == "open": + self._feedback(pr, None, no_promotion=True) + else: + self._transition( + number, "ignored", "not a promotable update for this canonical vendor" + ) return previous, reviewed = inputs self.progress.count("relevant") @@ -977,9 +1076,15 @@ def _status(workdir: Path) -> dict: sqlite3.connect(f"{database.as_uri()}?mode=ro", uri=True, timeout=1) ) as connection: rows = connection.execute( - "SELECT key, value FROM state WHERE key IN ('health', 'process', 'settings', 'migration')" + "SELECT key, value FROM state WHERE key IN " + "('health', 'process', 'settings', 'migration', 'refresh_branches')" ).fetchall() state = {key: json.loads(value) for key, value in rows} + refresh_branches = state.pop("refresh_branches", []) + if not isinstance(refresh_branches, list) or any( + not isinstance(branch, str) for branch in refresh_branches + ): + raise ValueError("Malformed refresh policy; expected a JSON list of branch names.") if any(not isinstance(value, dict) for value in state.values()): raise ValueError("Malformed status cache; expected JSON objects.") health, process = state.get("health", {}), state.get("process", {}) @@ -1002,6 +1107,8 @@ def _status(workdir: Path) -> dict: vendor=state.get("settings", {}).get("vendor"), canonical_repo=state.get("settings", {}).get("canonical_repo"), canonical_branch=state.get("settings", {}).get("canonical_branch"), + base_branch=state.get("settings", {}).get("base_branch"), + refresh_branches=refresh_branches, migration=state.get("migration"), ) if not active: @@ -1021,6 +1128,15 @@ def _parser() -> argparse.ArgumentParser: run.add_argument("--upstream-repo", required=True) run.add_argument("--canonical-repo", required=True) run.add_argument("--canonical-branch", required=True) + run.add_argument( + "--refresh-branch", + action="append", + default=[], + help=( + "Recognize this branch in --canonical-repo as a refresh target, not a temporary " + "source branch. Repeat for multiple targets; does not enable publication to them." + ), + ) run.add_argument( "--migrate-canonical-branch-from", help="Explicitly migrate this stopped workdir from OLD_BRANCH after a merged refresh.", @@ -1094,6 +1210,8 @@ def main(argv: list[str] | None = None) -> int: promote._repo_name(f"https://github.com/{name}") for branch in (args.canonical_branch, args.base_branch, args.upstream_branch): promote._run(["git", "check-ref-format", f"refs/heads/{branch}"]) + for branch in args.refresh_branch: + promote._run(["git", "check-ref-format", f"refs/heads/{branch}"]) if args.migrate_canonical_branch_from: promote._run( ["git", "check-ref-format", f"refs/heads/{args.migrate_canonical_branch_from}"] diff --git a/tests/unittest/others/test_maintain_prims_ts.py b/tests/unittest/others/test_maintain_prims_ts.py index bbd6c96f3c1e..df4948ab5ac1 100644 --- a/tests/unittest/others/test_maintain_prims_ts.py +++ b/tests/unittest/others/test_maintain_prims_ts.py @@ -339,6 +339,7 @@ def api( node_id="PR_test", state="open", merged=False, + user={"login": "maintainer"}, head={"sha": head}, base={"ref": "main"}, updated_at="2020-01-01T00:00:00Z", @@ -1300,6 +1301,7 @@ def test_bot_recovers_publication_after_cache_loss( b._initialize_workdir(workdir) args = argparse.Namespace( migrate_canonical_branch_from=None, + refresh_branch=[], canonical_repo="maintainer/flashinfer", canonical_branch="trtllm-prims-ts-dev", fork="maintainer/TensorRT-LLM", @@ -1491,6 +1493,7 @@ def test_recovery_rejects_changed_artifacts_without_writes(world: World, tamper: def test_bot_cold_recovery_dry_run_and_superseded_history(world: World) -> None: args = argparse.Namespace( migrate_canonical_branch_from=None, + refresh_branch=[], canonical_repo="maintainer/flashinfer", canonical_branch="trtllm-prims-ts-dev", fork="maintainer/TensorRT-LLM", diff --git a/tests/unittest/others/test_vendor_bot.py b/tests/unittest/others/test_vendor_bot.py index 6dff26cf8feb..bb30a565c83b 100644 --- a/tests/unittest/others/test_vendor_bot.py +++ b/tests/unittest/others/test_vendor_bot.py @@ -18,6 +18,7 @@ import threading import time from pathlib import Path +from urllib.parse import parse_qs import pytest @@ -753,6 +754,10 @@ def api( number = int(endpoint.split("/")[-2]) self.reviews[number - 1]["state"] = "DISMISSED" return {} + if "/reviews/" in endpoint and method == "PUT": + number = int(endpoint.rsplit("/", 1)[1]) + self.reviews[number - 1]["body"] = payload["body"] + return copy.deepcopy(self.reviews[number - 1]) if "/reviews" in endpoint: if method == "GET": return copy.deepcopy(self.reviews) @@ -789,6 +794,7 @@ def monitor(tmp_path: Path): args = argparse.Namespace( canonical_repo="operator/library", canonical_branch="dev", + refresh_branch=[], migrate_canonical_branch_from=None, fork="operator/consumer", repo=tmp_path, @@ -982,7 +988,14 @@ def pages(gh, endpoint): monitor.gh, "optional", lambda endpoint: {"commit": {"message": "record"}}, raising=False ) monkeypatch.setattr( - p, "_record_from_message", lambda message: {"vendor": "toy", "canonical_branch": "dev"} + p, + "_record_from_message", + lambda message: { + "vendor": "toy", + "canonical_branch": "dev", + "consumer_pr": "https://github.com/org/consumer/pull/7", + "consumer_base": "main", + }, ) if state == "closed": b.Monitor(args, monitor.gh, monitor.store) @@ -993,6 +1006,79 @@ def pages(gh, endpoint): assert not monitor.gh.writes +@pytest.mark.parametrize("base_branch", ["main", "release/1.3"]) +@pytest.mark.parametrize("other_scope", ["base", "repository"]) +@pytest.mark.parametrize("origin", ["local", "remote"]) +def test_branch_migration_ignores_promotions_for_other_consumers( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, + base_branch: str, + other_scope: str, + origin: str, +) -> None: + monitor.gh.base_branch = base_branch + other_base = "release/1.3" if base_branch == "main" else "main" + branch = "chore/toy-promote-7-aaaaaaaaaaaa" + record = { + "version": 1, + "vendor": "toy", + "canonical_branch": "dev", + "consumer_pr": "https://github.com/other/consumer/pull/7" + if other_scope == "repository" + else "https://github.com/org/consumer/pull/7", + "consumer_base": other_base if other_scope == "base" else base_branch, + } + + def pages(gh, endpoint): + if endpoint.endswith("/branches"): + return iter( + [{"name": branch, "commit": {"sha": "a" * 40}}] if origin == "remote" else [] + ) + assert endpoint.endswith("/pulls?state=open"), endpoint + return iter([]) + + monkeypatch.setattr(b, "_pages", pages) + monkeypatch.setattr( + p, "_git", lambda *args: f"{branch} {'a' * 40}" if origin == "local" else "" + ) + monkeypatch.setattr( + monitor.gh, + "optional", + lambda endpoint: {"commit": {"message": p._record_text(record)}}, + ) + monitor._check_pending_promotions("dev") + assert not monitor.gh.writes + + +@pytest.mark.parametrize("field", ["consumer_pr", "consumer_base"]) +@pytest.mark.parametrize("value", [None, "", 123]) +def test_branch_migration_rejects_ambiguous_consumer_provenance( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, + field: str, + value: object, +) -> None: + branch = "chore/toy-promote-7-aaaaaaaaaaaa" + record = { + "version": 1, + "vendor": "toy", + "canonical_branch": "dev", + "consumer_pr": "https://github.com/org/consumer/pull/7", + "consumer_base": "main", + field: value, + } + monkeypatch.setattr(b, "_pages", lambda *args: iter([])) + monkeypatch.setattr(p, "_git", lambda *args: f"{branch} {'a' * 40}") + monkeypatch.setattr( + monitor.gh, + "optional", + lambda endpoint: {"commit": {"message": p._record_text(record)}}, + ) + with pytest.raises(ValueError, match="Missing or invalid consumer provenance"): + monitor._check_pending_promotions("dev") + assert not monitor.gh.writes + + def test_branch_migration_sqlite_transaction_rolls_back(monitor: b.Monitor, migration) -> None: import sqlite3 @@ -1171,6 +1257,268 @@ def test_refresh_source_update_uses_current_canonical_as_promotion_baseline( assert not monitor.gh.writes +@pytest.fixture +def refresh_candidate(monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch): + previous = _vendor("1" * 40, "https://github.com/operator/library.git", "dev") + reviewed = _vendor("2" * 40, "https://github.com/operator/library.git", "dev-new") + locks = {"c" * 40: previous, "a" * 40: reviewed} + monkeypatch.setattr(p, "_main_sha", lambda gh: "c" * 40) + monkeypatch.setattr( + p, "_lock_document_at", lambda gh, repo, sha: {"vendors": {"toy": locks[sha].to_mapping()}} + ) + original_get = monitor.gh.get + monkeypatch.setattr( + monitor.gh, + "get", + lambda endpoint: {"parents": [{"sha": "c" * 40}]} + if endpoint.endswith("/commits/" + "a" * 40) + else original_get(endpoint), + ) + return locks + + +@pytest.mark.parametrize("base_branch", ["main", "release/1.3"]) +@pytest.mark.parametrize("redirect", ["repository", "branch", "both"]) +@pytest.mark.parametrize("caught_up", [False, True]) +def test_same_commit_reference_redirect_requests_canonical_lock_not_metadata( + monitor: b.Monitor, refresh_candidate, base_branch: str, redirect: str, caught_up: bool +) -> None: + gh = monitor.gh + gh.base_branch = gh.pr["base"]["ref"] = base_branch + previous = refresh_candidate["c" * 40] + reviewed = dataclasses.replace( + previous, + url="https://github.com/author/library.git" if redirect != "branch" else previous.url, + branch="temporary" if redirect != "repository" else previous.branch, + ) + refresh_candidate["a" * 40] = reviewed + if caught_up: + gh.merge_base_sha = "d" * 40 + refresh_candidate["d" * 40] = dataclasses.replace(previous, commit="0" * 40) + gh.pr["body"] = "Invalid/missing promotion metadata must not obscure the lock fix." + with contextlib.closing(b.Store(":memory:")) as store: + restarted = b.Monitor(monitor.args, gh, store) + restarted.inspect(7) + assert store.get("observation:7")["state"] == "blocked" + body = gh.comments[0]["body"] + assert "Temporary source reference adds no new vendor commit" in body + assert previous.commit in body and base_branch in body + assert p._LOCK in body and "vendors.toy" in body + assert previous.url in body and f"{previous.branch}" in body + assert "keep the commit and other lock fields, vendored files, and patch unchanged" in body + assert "No upstream-PR metadata" in body + assert "vendor-promotion:start" not in body + assert gh.reviews[0]["state"] == "CHANGES_REQUESTED" + writes = copy.deepcopy(gh.writes) + with contextlib.closing(b.Store(":memory:")) as store: + b.Monitor(monitor.args, gh, store).inspect(7) + assert gh.writes == writes + + +@pytest.mark.parametrize( + "case", ["canonical", "equivalent-url", "refresh", "inherited", "merged", "promotion"] +) +def test_same_commit_reference_non_redirects_do_not_request_changes( + monitor: b.Monitor, refresh_candidate, case: str +) -> None: + previous = refresh_candidate["c" * 40] + reviewed = previous + if case == "equivalent-url": + reviewed = dataclasses.replace(previous, url="https://github.com/OPERATOR/library") + elif case == "refresh": + reviewed = dataclasses.replace(previous, branch="dev-new") + monitor.args.refresh_branch = ["dev-new"] + elif case in ("inherited", "merged"): + reviewed = dataclasses.replace(previous, url="https://github.com/author/library.git") + if case == "inherited": + monitor.gh.merge_base_sha = "d" * 40 + refresh_candidate["d" * 40] = reviewed + else: + monitor.gh.pr.update(merged=True, state="closed", merge_commit_sha="a" * 40) + elif case == "promotion": + refresh_candidate["c" * 40] = dataclasses.replace( + previous, url="https://github.com/author/library.git", branch="temporary" + ) + refresh_candidate["a" * 40] = reviewed + with contextlib.closing(b.Store(":memory:")) as store: + b.Monitor(monitor.args, monitor.gh, store).inspect(7) + assert store.get("observation:7")["state"] == "ignored" + assert not monitor.gh.writes + + +def test_same_commit_reference_reconciles_stale_feedback_then_clears_after_fix( + monitor: b.Monitor, refresh_candidate +) -> None: + gh = monitor.gh + monitor._feedback(gh.pr, _issue("Source update is not a fast-forward.")) + previous = refresh_candidate["c" * 40] + refresh_candidate["a" * 40] = dataclasses.replace( + previous, url="https://github.com/author/library.git", branch="temporary" + ) + monitor.inspect(7) + assert len(gh.comments) == len(gh.reviews) == 1 + assert all("adds no new vendor commit" in item["body"] for item in (*gh.comments, *gh.reviews)) + assert "fast-forward" not in gh.reviews[0]["body"] + assert gh.reviews[0]["state"] == "CHANGES_REQUESTED" + assert any( + method == "PUT" and endpoint.endswith("/reviews/1") for endpoint, method, _ in gh.writes + ) + refresh_candidate["a" * 40] = previous + for _ in range(2): + with contextlib.closing(b.Store(":memory:")) as store: + b.Monitor(monitor.args, gh, store).inspect(7) + assert store.get("observation:7")["state"] == "ignored" + assert store.get("problem:7") is None + assert len(gh.comments) == len(gh.reviews) == 1 + assert "does not require vendor promotion" in gh.comments[0]["body"] + assert "metadata and source attribution are valid" not in gh.comments[0]["body"] + assert gh.reviews[0]["state"] == "DISMISSED" + assert not any(payload.get("event") == "APPROVE" for _, _, payload in gh.writes) + writes = copy.deepcopy(gh.writes) + monitor.inspect(7) + assert gh.writes == writes + + +@pytest.mark.parametrize("remaining", ["comment", "review", "both", "neither"]) +def test_ignored_pr_cleans_remote_feedback_without_sqlite( + monitor: b.Monitor, refresh_candidate, remaining: str +) -> None: + monitor._feedback(monitor.gh.pr, _issue("Missing block")) + if remaining in ("review", "neither"): + monitor.gh.comments.clear() + if remaining in ("comment", "neither"): + monitor.gh.reviews.clear() + monitor.gh.writes.clear() + monitor.args.refresh_branch = ["dev-new"] + for _ in range(2): + with contextlib.closing(b.Store(":memory:")) as store: + b.Monitor(monitor.args, monitor.gh, store).inspect(7) + assert store.get("observation:7")["state"] == "ignored" + if remaining == "neither": + assert not monitor.gh.writes + else: + assert len(monitor.gh.comments) == 1 + assert "no longer applies" in monitor.gh.comments[0]["body"] + assert "not** a code-review approval" in monitor.gh.comments[0]["body"] + assert all(review["state"] == "DISMISSED" for review in monitor.gh.reviews) + assert len(monitor.gh.writes) == (2 if remaining in ("review", "both") else 1) + + +@pytest.mark.parametrize("existing", ["manual", "other-actor", "other-vendor"]) +def test_ignored_pr_preserves_unrelated_comments_and_reviews( + monitor: b.Monitor, refresh_candidate, existing: str +) -> None: + actor = "human" if existing == "other-actor" else monitor.actor + body = ( + "Manual review" + if existing == "manual" + else "\nOther vendor" + if existing == "other-vendor" + else f"{monitor.marker}\nAnother operator's feedback" + ) + monitor.gh.comments.append({"id": 1, "user": {"login": actor}, "body": body}) + monitor.gh.reviews.append( + {"id": 1, "user": {"login": actor}, "body": body, "state": "CHANGES_REQUESTED"} + ) + monitor.args.refresh_branch = ["dev-new"] + b.Monitor(monitor.args, monitor.gh, monitor.store).inspect(7) + assert not monitor.gh.writes + assert monitor.gh.reviews[0]["state"] == "CHANGES_REQUESTED" + + +@pytest.mark.parametrize("change", ["head", "body", "base", "closed", "dry-run"]) +def test_ignored_feedback_cleanup_respects_concurrent_changes_and_dry_run( + monitor: b.Monitor, change: str +) -> None: + monitor._feedback(monitor.gh.pr, _issue("Missing block")) + snapshot = copy.deepcopy(monitor.gh.pr) + if change == "head": + monitor.gh.pr["head"]["sha"] = "b" * 40 + elif change == "body": + monitor.gh.pr["body"] = "edited" + elif change == "base": + monitor.gh.pr["base"]["ref"] = "release/1.3" + elif change == "closed": + monitor.gh.pr["state"] = "closed" + else: + monitor.args.publish = False + writes = copy.deepcopy(monitor.gh.writes) + monitor._feedback(snapshot, None, no_promotion=True) + assert monitor.gh.writes == writes + assert monitor.gh.reviews[0]["state"] == "CHANGES_REQUESTED" + + +@pytest.mark.parametrize("base_branch", ["main", "release/1.3"]) +@pytest.mark.parametrize("merged", [False, True]) +def test_explicit_refresh_target_never_requests_metadata_or_promotes( + monitor: b.Monitor, refresh_candidate, base_branch: str, merged: bool +) -> None: + monitor.gh.base_branch = base_branch + monitor.gh.pr["base"]["ref"] = base_branch + monitor.gh.pr.update(merged=merged, merge_commit_sha="a" * 40) + monitor.args.refresh_branch = ["dev-new"] + for _ in range(2): + with contextlib.closing(b.Store(":memory:")) as store: + restarted = b.Monitor(monitor.args, monitor.gh, store) + restarted.inspect(7) + assert store.get("observation:7")["state"] == "ignored" + assert store.get("refresh_branches") == ["dev-new"] + assert not monitor.gh.writes + + +@pytest.mark.parametrize("untrusted", ["branch", "repository"]) +def test_refresh_targets_are_explicit_and_repository_scoped( + monitor: b.Monitor, refresh_candidate, untrusted: str +) -> None: + if untrusted == "repository": + monitor.args.refresh_branch = ["dev-new"] + refresh_candidate["a" * 40] = dataclasses.replace( + refresh_candidate["a" * 40], url="https://github.com/author/library.git" + ) + restarted = b.Monitor(monitor.args, monitor.gh, monitor.store) + assert restarted._inputs(monitor.gh.pr) == ( + refresh_candidate["c" * 40], + refresh_candidate["a" * 40], + ) + restarted.inspect(7) + assert "vendor-promotion:start" in monitor.gh.comments[0]["body"] + assert monitor.gh.reviews[0]["state"] == "CHANGES_REQUESTED" + + +def test_refresh_policy_can_change_without_migrating_workdir( + monitor: b.Monitor, refresh_candidate +) -> None: + monitor.store.put("cursor", "preserved") + monitor.store.put("complete:8", "preserved") + for branches in (["dev-new", "dev-new"], []): + monitor.args.refresh_branch = branches + restarted = b.Monitor(monitor.args, monitor.gh, monitor.store) + assert (restarted._inputs(monitor.gh.pr) is None) == bool(branches) + assert monitor.store.get("settings")["canonical_branch"] == "dev" + assert monitor.store.get("cursor") == "preserved" + assert monitor.store.get("complete:8") == "preserved" + assert monitor.store.get("refresh_branches") == sorted(set(branches)) + assert monitor.store.get("migration") is None + assert not monitor.gh.writes + + +def test_updates_after_refresh_require_maintainer_cutover( + monitor: b.Monitor, refresh_candidate +) -> None: + monitor.args.refresh_branch = ["dev-new"] + refresh_candidate["c" * 40] = refresh_candidate["a" * 40] + refresh_candidate["a" * 40] = _vendor( + "3" * 40, "https://github.com/author/library.git", "temporary" + ) + restarted = b.Monitor(monitor.args, monitor.gh, monitor.store) + restarted.inspect(7) + comment = monitor.gh.comments[0]["body"] + assert "maintainer action" in comment + assert "--canonical-branch dev-new --migrate-canonical-branch-from dev" in comment + assert "Do not promote back" in comment + assert not monitor.gh.reviews + + def test_waiting_for_prior_promotion_is_maintainer_action_without_review( monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -1321,6 +1669,84 @@ def test_discovery_and_persistent_operator_scope(monitor: b.Monitor) -> None: b.Monitor(changed, monitor.gh, monitor.store) +@pytest.mark.parametrize("base_branch", ["main", "release/1.3"]) +def test_monitors_discover_only_their_consumer_base(monitor: b.Monitor, base_branch: str) -> None: + monitor.gh.base_branch = base_branch + with contextlib.closing(b.Store(":memory:")) as store: + restarted = b.Monitor(monitor.args, monitor.gh, store) + monitor.gh.pr["base"]["ref"] = "release/1.3" if base_branch == "main" else "main" + assert restarted._discover() == [] + monitor.gh.pr["base"]["ref"] = base_branch + assert restarted._discover() == [7] + assert not monitor.gh.writes + + +def test_workdir_cannot_change_consumer_base(monitor: b.Monitor) -> None: + monitor.gh.base_branch = "release/1.3" + with pytest.raises(ValueError, match="different operator/vendor configuration"): + b.Monitor(monitor.args, monitor.gh, monitor.store) + assert monitor.store.get("settings")["base_branch"] == "main" + + +@pytest.mark.parametrize("base_branch", ["main", "release/1.3"]) +def test_promotion_identity_and_followup_keep_consumer_base( + monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch, base_branch: str +) -> None: + gh = monitor.gh + gh.base_branch = base_branch + gh.pr.update(merged=True, merge_commit_sha="b" * 40) + gh.pr["base"]["ref"] = base_branch + previous = _vendor("1" * 40, "https://github.com/operator/library.git", "dev") + reviewed = _vendor("2" * 40, "https://github.com/author/library.git", "temporary") + original_get = gh.get + + def get(endpoint: str) -> dict: + if endpoint.endswith("/commits/" + "b" * 40): + return {"parents": [{"sha": "c" * 40}]} + if "/git/ref/heads/" in endpoint: + assert endpoint.endswith(f"/git/ref/heads/{base_branch}") + return {"object": {"sha": "d" * 40}} + return original_get(endpoint) + + monkeypatch.setattr(gh, "get", get) + monkeypatch.setattr( + p, "_lock_at", lambda gh, repo, sha: previous if sha == "c" * 40 else reviewed + ) + args = monitor._arguments(7, reviewed) + plan = p._plan_identity(args, gh) + assert plan.base_branch == plan.record["consumer_base"] == base_branch + assert plan.main_sha == "d" * 40 + + def followups(endpoint: str): + query = parse_qs(endpoint.split("?", 1)[1]) + assert query["base"] == [base_branch] + assert query["head"] == [f"operator:{plan.branch}"] + return [] + + monkeypatch.setattr(gh, "api", followups) + assert p._find_followup(gh, plan) is None + assert not gh.writes + + +def test_tracked_pr_retargeted_to_another_base_is_ignored(monitor: b.Monitor) -> None: + assert monitor._discover() == [7] + monitor.gh.pr["base"]["ref"] = "release/1.3" + monitor.inspect(7) + assert monitor.store.get("observation:7") == { + "state": "ignored", + "reason": "PR targets a different consumer base branch", + } + assert not monitor.gh.writes + + +def test_feedback_rechecks_consumer_base_before_writing(monitor: b.Monitor) -> None: + inspected = copy.deepcopy(monitor.gh.pr) + monitor.gh.pr["base"]["ref"] = "release/1.3" + monitor._feedback(inspected, _issue()) + assert monitor.store.get("observation:7")["state"] == "ignored" + assert not monitor.gh.writes + + def test_cold_discovery_backfills_old_merged_prs_and_honors_cutoff( monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch, @@ -1515,11 +1941,17 @@ def unresolved(*args): assert monitor.gh.reviews[0]["state"] == "DISMISSED" +@pytest.mark.parametrize("no_promotion", [False, True]) +@pytest.mark.parametrize("comment_deleted", [False, True]) def test_forbidden_dismissal_informs_author_without_approving( monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch, + no_promotion: bool, + comment_deleted: bool, ) -> None: monitor._feedback(monitor.gh.pr, _issue("Missing block")) + if comment_deleted: + monitor.gh.comments.clear() original = monitor.gh.api def forbidden(endpoint, **kwargs): @@ -1528,14 +1960,14 @@ def forbidden(endpoint, **kwargs): return original(endpoint, **kwargs) monkeypatch.setattr(monitor.gh, "api", forbidden) - monitor._feedback(monitor.gh.pr, None) + monitor._feedback(monitor.gh.pr, None, no_promotion=no_promotion) assert "lacks dismissal permission" in monitor.gh.comments[0]["body"] assert monitor.gh.reviews[0]["state"] == "CHANGES_REQUESTED" writes = len(monitor.gh.writes) - monitor._feedback(monitor.gh.pr, None) + monitor._feedback(monitor.gh.pr, None, no_promotion=no_promotion) assert len(monitor.gh.writes) == writes monkeypatch.setattr(monitor.gh, "api", original) - monitor._feedback(monitor.gh.pr, None) + monitor._feedback(monitor.gh.pr, None, no_promotion=no_promotion) assert "lacks dismissal permission" not in monitor.gh.comments[0]["body"] assert monitor.gh.reviews[0]["state"] == "DISMISSED" @@ -1940,6 +2372,53 @@ def test_status_is_read_only_and_hides_stale_process_data(tmp_path: Path) -> Non assert (workdir / b._STATE_FILE).read_bytes() == before +def test_status_exposes_consumer_base_and_refresh_policy(monitor: b.Monitor) -> None: + monitor.args.refresh_branch = ["dev-new"] + b.Monitor(monitor.args, monitor.gh, monitor.store) + state = b._status(monitor.args.workdir) + assert state["base_branch"] == "main" + assert state["canonical_branch"] == "dev" + assert state["refresh_branches"] == ["dev-new"] + assert not state["running"] + + +@pytest.mark.parametrize("policy", [{}, [123]]) +def test_status_rejects_malformed_refresh_policy(monitor: b.Monitor, policy: object) -> None: + monitor.store.put("refresh_branches", policy) + state = b._status(monitor.args.workdir) + assert not state["state_available"] + assert "Malformed refresh policy" in state["state_error"] + + +def test_refresh_branch_cli_is_repeatable_and_rejects_invalid_refs(tmp_path: Path) -> None: + workdir = tmp_path / "unused-bot-state" + argv = [ + "run", + "--workdir", + str(workdir), + "--vendor", + "toy", + "--upstream-repo", + "org/library", + "--canonical-repo", + "operator/library", + "--canonical-branch", + "dev", + "--fork", + "operator/consumer", + "--refresh-branch", + "dev-new", + "--refresh-branch", + "../invalid", + "--once", + ] + parsed = b._parser().parse_args(argv) + assert parsed.refresh_branch == ["dev-new", "../invalid"] + assert not parsed.publish and not parsed.daemon + assert b.main(argv) == 1 + assert not workdir.exists() + + @pytest.mark.parametrize("malformed", [False, True]) def test_status_reports_unreadable_cache_without_losing_liveness( tmp_path: Path, malformed: bool From a15227e146376de530c9743ba618358948d2a224 Mon Sep 17 00:00:00 2001 From: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:30:42 +0000 Subject: [PATCH 09/11] [None][doc] Clarify vendor bot scope and refresh workflow Signed-off-by: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> --- 3rdparty/vendor-sources.md | 9 +++++++++ scripts/vendor/README.md | 41 ++++++++++++++++++++++++++++++-------- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/3rdparty/vendor-sources.md b/3rdparty/vendor-sources.md index d80ebfeb8f52..886fd05482fc 100644 --- a/3rdparty/vendor-sources.md +++ b/3rdparty/vendor-sources.md @@ -302,6 +302,15 @@ authentication, configurable private workdirs, daemon launch/stop, and recovery. No GitHub App or systemd setup is required. All remote writes are opt-in with `--publish`; operator settings and credentials are not committed to this repo. +Each monitor handles one vendor and consumer base branch. Main and release +branches can use different canonical source branches, but require separate +workdirs and explicit launches. A periodic upstream refresh remains a separate +code-changing PR with normal CI: stop its monitor or configure an audited +`--refresh-branch` target before opening it, then migrate the canonical branch +in the existing workdir after merge. See the toolkit guide's +[base-branch setup](../scripts/vendor/README.md#different-consumer-base-branches) +and [refresh cutover](../scripts/vendor/README.md#switching-canonical-branches-after-a-refresh). + Use the entry points under `scripts/vendor/` directly. The old top-level scripts and PrimTS-specific CLI aliases are not retained; callers must migrate to the generic commands and supply their vendor and repository configuration explicitly. diff --git a/scripts/vendor/README.md b/scripts/vendor/README.md index 1c10f20eb456..fcff541083af 100644 --- a/scripts/vendor/README.md +++ b/scripts/vendor/README.md @@ -26,7 +26,12 @@ lock and is not migrated by this change. ```mermaid flowchart TD - source[Source-update PR changes a vendor pin] --> metadata[Author lists upstream PRs in description] + candidate[Lock-changing PR for the configured consumer and base branch] --> classify{Selected vendor change} + classify -->|new pin on temporary branch| metadata[Author lists upstream PRs in description] + classify -->|temporary reference at current canonical pin| restore[Author restores canonical URL and branch only] + restore --> cleanup[Reconcile obsolete bot feedback without approving code] + classify -->|no promotion needed| cleanup + classify -->|operator-configured refresh target| refresh[Separate upstream refresh PR and normal CI] metadata --> match[Bot validates metadata and matches lock-selected file changes] match -->|missing or ambiguous| feedback[Request changes with author-facing instructions] feedback -->|author corrects block, pin, or targeted resolution| metadata @@ -41,7 +46,7 @@ flowchart TD recover -->|pending recorded operation| verify recover -->|no record and pin still pending| metadata recover -->|completed promotion| merged - refresh[Separate upstream refresh PR and normal CI] -->|merged| migrate[Stop bot and verify explicit branch migration] + refresh -->|merged| migrate[Stop bot and verify explicit branch migration] migrate --> rediscover[Reuse workdir and revalidate GitHub state] rediscover --> rebase ``` @@ -65,11 +70,13 @@ repository authorization. No GitHub App, repository-admin access, webhook server, or systemd unit is required merely to run the monitor. Repository rules, SSO/token policies, CI-bot authorization, and required owner reviews still apply. -Run only one **publishing** monitor per consumer/vendor/canonical branch. The -workdir lock prevents duplicate processes using the same directory; it is not a -distributed lock between operators or machines. Finish one promotion before -merging the next source update for that vendor. Unexpected concurrent changes -stop publication rather than force-pushing or selecting an arbitrary newer pin. +Run only one **publishing** monitor per consumer repository/base/vendor. If +multiple monitors share a canonical source branch, serialize their publications +as well. The workdir lock prevents duplicate processes using the same directory; +it is not a distributed lock between operators, machines, or workdirs. Finish +one promotion before merging the next source update for that consumer/base/vendor. +Unexpected concurrent changes stop publication rather than force-pushing or +selecting an arbitrary newer pin. ## Contributor metadata @@ -166,6 +173,8 @@ All operator-specific settings are CLI inputs. Start with one dry-run pass: ```bash python scripts/vendor/bot.py run \ --vendor example-library \ + --consumer-repo NVIDIA/TensorRT-LLM \ + --base-branch main \ --upstream-repo UPSTREAM/library \ --canonical-repo MAINTAINER/library \ --canonical-branch consumer-dev \ @@ -297,6 +306,13 @@ do not require promotion metadata. Eligible updates are still checked against the latest canonical lock, so their source branches must include the refreshed canonical history before promotion. +A same-commit temporary URL/branch redirect is not a source update, but can still +need author feedback: if the PR itself changes the reference and its pin equals +the current canonical base pin, restore the base's canonical URL/branch only. +No promotion metadata is required. When an open PR no longer needs promotion, +the bot reconciles any existing feedback instead of leaving a stale objection. +It does not post new feedback on otherwise ignored PRs. + ### Monitoring health `--log-level INFO` is the default, in both foreground and daemon mode. Logs use @@ -326,6 +342,8 @@ The JSON separates process liveness from work completion: | Field | Meaning | | --- | --- | | `running`, `pid`, `publish` | Process lock held, recorded PID, and whether remote writes are enabled; not a health verdict | +| `vendor`, `base_branch`, `canonical_repo`, `canonical_branch` | Cached monitor scope and publishing destination; verify these after a restart or migration | +| `refresh_branches`, `migration` | Explicit refresh-target policy and last verified canonical-branch migration; not authorization to publish to a refresh target | | `operation`, `progress_at` | Current phase/PR/start time and last worker checkpoint | | `counts` | Current poll's scanned records, tracked lock PRs, relevant vendor updates, processed PRs, author-blocked PRs, and errors | | `last_completed_poll` | Last fully scanned/processed poll, even if individual PRs needed attention | @@ -354,7 +372,8 @@ authorizes publication; restart recovery still verifies Git/GitHub provenance. Git/GitHub are the source of truth; SQLite is not required to reconstruct a promotion. Stop the old process before replacing or discarding its database. -Relaunch with the **same account, repositories, canonical branch, and cutoff**. +Relaunch with the **same account, repositories, consumer base branch, canonical +branch, and cutoff**, and re-supply any `--refresh-branch` policy. The bot accepts an existing workdir containing only recognized bot artifacts even if SQLite is missing. It preserves those artifacts and refuses unrelated files and symlinks. A new empty workdir also works; existing registered Git @@ -478,3 +497,9 @@ PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 pytest --noconftest -c /dev/null \ tests/unittest/others/test_maintain_prims_ts.py \ tests/unittest/others/test_vendor_bot.py ``` + +These suites carry `cpu_only` markers and are included in normal CPU CI through +the `unittest/others` parent entry in +`tests/integration/test_lists/test-db/l0_cpu.yml`. The standalone command above +avoids loading the TensorRT-LLM runtime or unrelated pytest plugins; it does not +replace normal CI for changes to the tools. From fef19f024f8104a0fdde0225b49d4952e61d775d Mon Sep 17 00:00:00 2001 From: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> Date: Mon, 19 Oct 2026 05:55:45 +0000 Subject: [PATCH 10/11] [None][doc] Keep vendor guidance in AGENTS concise Signed-off-by: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> --- AGENTS.md | 36 ++++-------------------------------- 1 file changed, 4 insertions(+), 32 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index abf96ef3f009..f4e20f4873ec 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -195,38 +195,10 @@ historical implementation details. ### Vendor source tools and promotion -Use `scripts/vendor/manage.py` for lock/vendor-state changes and -`scripts/vendor/promote.py promote` for lock-only promotion after a source-update -PR merges. Select the vendor, upstream, and canonical fork/branch explicitly. -Manual publication uses `--publish --auto-merge`; dry-run is the default. -`scripts/vendor/bot.py run` provides an optional local monitor with `--workdir`, -`--daemon`, and opt-in `--publish`. Do not launch a publishing daemon unless the -user requested ongoing automation. Never commit operator credentials or runtime -state. The bot asks the PR author to correct metadata or ambiguous attribution; -it never approves code merely because metadata passes. Use the entry points -under `scripts/vendor/`; no legacy script wrappers are provided. See -`scripts/vendor/README.md` and `3rdparty/vendor-sources.md`. Preserve structured -promotion provenance and DCO sign-off in the final squash message. CI skip applies -only to verified lock-only promotions, not changes to these tools. -Git/GitHub promotion commits are authoritative; SQLite is a disposable cache. -After cache loss, stop the old process and relaunch with the same operator/config. -Cold discovery backfills merged PRs; `--since` explicitly limits that history. -After a canonical-branch refresh merges, stop the bot and reuse its workdir with -the new `--canonical-branch` plus `--migrate-canonical-branch-from OLD_BRANCH`. -Start with `--once` without `--publish`; migration verifies the merged lock, -blocks unresolved old promotions, and atomically archives branch-dependent cache -results before rediscovery. Other identity changes still require another workdir. -Each monitor handles one consumer base branch (`--base-branch`, default `main`); -use separate workdirs for main and release monitors. Launching one never enables -the other. Before a refresh PR, stop its monitor or restart with the explicitly -audited `--refresh-branch NEW_BRANCH` in the configured canonical repository. -This repeatable option excludes refresh PRs from promotion; it does not authorize -publication to a new branch. After merge, the explicit migration is still required. -Migration checks ignore promotion records for other consumer repositories/bases. -After cache loss, use the new configuration without the migration flag. -For monitoring, use INFO poll summaries/30-second active-operation heartbeats -and `bot.py status --workdir ...`. Check the last poll outcome and next deadline; -the process lock and heartbeat alone do not establish successful reconciliation. +Use the tools under `scripts/vendor/` to manage locked vendor sources, prepare +lock-only promotions, and optionally monitor source-update PRs. Read +[Vendored Sources](3rdparty/vendor-sources.md) for commands, contributor metadata, +and operator guidance before changing vendor state or publishing promotions. ### Triggering CI From 2bdead8f0311c4c5393c61156940f29ed197ea29 Mon Sep 17 00:00:00 2001 From: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> Date: Mon, 19 Oct 2026 06:01:46 +0000 Subject: [PATCH 11/11] [None][fix] Use CI-compatible capture in vendor promotion tests Signed-off-by: Yuxian Qiu <142763828+yuxianq@users.noreply.github.com> --- tests/unittest/others/test_maintain_prims_ts.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/unittest/others/test_maintain_prims_ts.py b/tests/unittest/others/test_maintain_prims_ts.py index df4948ab5ac1..3751df9bced9 100644 --- a/tests/unittest/others/test_maintain_prims_ts.py +++ b/tests/unittest/others/test_maintain_prims_ts.py @@ -385,7 +385,7 @@ def world(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> World: def test_dry_run_has_no_writes_and_pins_merged_not_live_head( - world: World, capsys: pytest.CaptureFixture + world: World, capfd: pytest.CaptureFixture ) -> None: assert ( m.main( @@ -409,7 +409,7 @@ def test_dry_run_has_no_writes_and_pins_merged_not_live_head( ) == 0 ) - assert "DRY RUN" in capsys.readouterr().out + assert "DRY RUN" in capfd.readouterr().out assert all(method == "GET" for method, _, _ in world.calls) assert not any(args[0] in ("push", "commit", "worktree") for args in world.git_calls) plan = m._make_plan(world.args(), world) @@ -1035,7 +1035,7 @@ def test_canonical_branch_name_on_developer_fork(world: World, same_fork: bool) assert actual == {**plan.reviewed.to_mapping(), "url": plan.previous.url} -def test_unpaired_cli_dry_run(world: World, capsys: pytest.CaptureFixture) -> None: +def test_unpaired_cli_dry_run(world: World, capfd: pytest.CaptureFixture) -> None: assert ( m.main( [ @@ -1058,7 +1058,7 @@ def test_unpaired_cli_dry_run(world: World, capsys: pytest.CaptureFixture) -> No ) == 0 ) - assert '"refresh_policy": "retain"' in capsys.readouterr().out + assert '"refresh_policy": "retain"' in capfd.readouterr().out assert not any(args[0] == "push" for args in world.git_calls) @@ -1086,14 +1086,14 @@ def test_canonical_branch_must_be_explicit_and_match(world: World, branch: str) def test_completed_promotion_ignores_later_live_state( - world: World, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture + world: World, monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture ) -> None: world.state = "CLEAN" args = world.args(publish=True, auto_merge=True) plan = m._make_plan(args, world) m._publish(args, world, plan) merged = world.main - assert f"Squash commit: {merged}" in capsys.readouterr().out + assert f"Squash commit: {merged}" in capfd.readouterr().out _git(world.consumer, "switch", "--detach", world.main) _write(world.consumer, "later.txt", "later main commit\n") world.main = _commit(world.consumer, "later main update")