diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 77bc45f49e75..bab1ef5b3a7f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -475,7 +475,7 @@ /requirements-grpc-smg.txt @NVIDIA/trt-llm-oss-compliance /requirements-openengine.txt @NVIDIA/trt-llm-oss-compliance /requirements.txt @NVIDIA/trt-llm-oss-compliance -/scripts/vendor_sources.py @NVIDIA/trt-llm-infra-devs @NVIDIA/trt-llm-oss-compliance +/scripts/vendor/ @NVIDIA/trt-llm-infra-devs @NVIDIA/trt-llm-oss-compliance /setup.py @NVIDIA/trt-llm-oss-compliance /tensorrt_llm/_torch/attention/backends/prims_ts/** @NVIDIA/trt-llm-torch-attention-devs @NVIDIA/trt-llm-oss-compliance /tests/unittest/api_stability/ @NVIDIA/trt-llm-noncommitted-api-review-committee diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bac469873fd0..c93ef9233a39 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1597,7 +1597,7 @@ repos: pass_filenames: false - id: vendor-sources-check name: verify vendored sources are in sync - entry: python scripts/vendor_sources.py check + entry: python scripts/vendor/manage.py check language: python additional_dependencies: [PyYAML] always_run: true diff --git a/3rdparty/vendor-sources.md b/3rdparty/vendor-sources.md index c37cbb1de2d8..886fd05482fc 100644 --- a/3rdparty/vendor-sources.md +++ b/3rdparty/vendor-sources.md @@ -14,7 +14,7 @@ the selected files, any persistent compatibility patch and its content digest, and a digest of the materialized destination. A short branch or tag may explain where the commit came from, but the full commit is authoritative. -Use `scripts/vendor_sources.py` for every lock or vendor-state change. Do not +Use `scripts/vendor/manage.py` for every lock or vendor-state change. Do not edit the YAML, generated patches, or digests by hand. All examples below use the default lock. For an isolated test or another consumer repository, place `--lock PATH` before the subcommand. @@ -91,10 +91,10 @@ existing compatibility patch, then use `pin`. List entries, or run the offline integrity status for all or one vendor: ```bash -python scripts/vendor_sources.py list -python scripts/vendor_sources.py status -python scripts/vendor_sources.py status VENDOR -python scripts/vendor_sources.py check VENDOR +python scripts/vendor/manage.py list +python scripts/vendor/manage.py status +python scripts/vendor/manage.py status VENDOR +python scripts/vendor/manage.py check VENDOR ``` `status` and the default `check` exit unsuccessfully if the destination has a @@ -107,7 +107,7 @@ When neither the lock entry nor destination exists, create both from an immutable commit and a local upstream checkout: ```bash -python scripts/vendor_sources.py create VENDOR \ +python scripts/vendor/manage.py create VENDOR \ --url https://example.com/organization/repository.git \ --branch main \ --commit FULL_COMMIT \ @@ -124,7 +124,7 @@ If the destination already exists but has no lock entry, adopt it. Use `exact` to require an exact upstream match: ```bash -python scripts/vendor_sources.py create VENDOR \ +python scripts/vendor/manage.py create VENDOR \ --url https://example.com/organization/repository.git \ --commit FULL_COMMIT \ --source path/in/upstream \ @@ -143,7 +143,7 @@ Discard destination edits and reproduce the currently locked upstream commit plus its persistent patch: ```bash -python scripts/vendor_sources.py sync VENDOR --repo /path/to/upstream +python scripts/vendor/manage.py sync VENDOR --repo /path/to/upstream ``` This overwrites the selected destination files. It does not update the lock, @@ -155,21 +155,21 @@ After editing an exact destination for a change that must remain downstream, create its persistent patch: ```bash -python scripts/vendor_sources.py patch VENDOR create --repo /path/to/upstream +python scripts/vendor/manage.py patch VENDOR create --repo /path/to/upstream ``` After intentionally changing an already patched destination, regenerate the patch: ```bash -python scripts/vendor_sources.py patch VENDOR refresh --repo /path/to/upstream +python scripts/vendor/manage.py patch VENDOR refresh --repo /path/to/upstream ``` Drop a no-longer-needed patch only after the destination exactly matches the currently locked upstream selection: ```bash -python scripts/vendor_sources.py patch VENDOR drop --repo /path/to/upstream +python scripts/vendor/manage.py patch VENDOR drop --repo /path/to/upstream ``` Generated patches live under `3rdparty/vendor_patches/`. Review them, but update @@ -184,7 +184,7 @@ branch at the currently locked commit **before** exporting: ```bash git -C /path/to/upstream switch -c trtllm-vendor-fix LOCKED_FULL_COMMIT -python scripts/vendor_sources.py export VENDOR --repo /path/to/upstream +python scripts/vendor/manage.py export VENDOR --repo /path/to/upstream ``` The upstream checkout's selected source must be clean before export and its @@ -205,7 +205,7 @@ git -C /path/to/upstream push -u origin trtllm-vendor-fix Finally, pin the committed revision from that checkout: ```bash -python scripts/vendor_sources.py pin VENDOR \ +python scripts/vendor/manage.py pin VENDOR \ --url https://example.com/my-fork/repository.git \ --branch trtllm-vendor-fix \ --commit NEW_FULL_COMMIT \ @@ -236,7 +236,7 @@ Remove a lock entry and its generated compatibility patch while preserving the destination: ```bash -python scripts/vendor_sources.py remove VENDOR +python scripts/vendor/manage.py remove VENDOR ``` The preserved destination is no longer protected by the lock. Delete or move @@ -247,8 +247,8 @@ it separately as part of the reviewed migration that removes the vendor. The default check is deliberately offline: ```bash -python scripts/vendor_sources.py check -python scripts/vendor_sources.py check --offline +python scripts/vendor/manage.py check +python scripts/vendor/manage.py check --offline ``` It validates the lock schema and path safety, patch metadata, and the checked-in @@ -260,7 +260,7 @@ When network access is available, attempt verification against every recorded upstream: ```bash -python scripts/vendor_sources.py check --upstream +python scripts/vendor/manage.py check --upstream ``` An inaccessible repository is reported as unavailable rather than failing. If @@ -268,14 +268,14 @@ a commit can be obtained, a source, patch, or destination mismatch is an error. Trusted maintainer CI can require access to every source: ```bash -python scripts/vendor_sources.py check --upstream --require-access +python scripts/vendor/manage.py check --upstream --require-access ``` To verify one vendor against an existing checkout without contacting the recorded URL, provide it explicitly: ```bash -python scripts/vendor_sources.py check VENDOR --repo /path/to/upstream +python scripts/vendor/manage.py check VENDOR --repo /path/to/upstream ``` The checkout's configured remote may differ from the lock URL; it only needs to @@ -289,6 +289,32 @@ local repository, and URL or commit changes require vendor CODEOWNER review. Never put credentials in a lock URL. Run checks that use internal credentials only in a trusted environment, not with pull-request-controlled scripts. +## Promotion and local monitoring + +The generic toolkit under `scripts/vendor/` supports every vendor key in this +lock. `promote.py` prepares a verified lock-only follow-up after a source-update +PR merges; `bot.py` can monitor PRs locally, request missing contributor metadata, +match upstream commits, and publish promotions with squash auto-merge. + +See the [toolkit guide](../scripts/vendor/README.md) for the workflow diagram, +contributor description format, author resolution of ambiguous matches, +authentication, configurable private workdirs, daemon launch/stop, and recovery. +No GitHub App or systemd setup is required. All remote writes are opt-in with +`--publish`; operator settings and credentials are not committed to this repo. + +Each monitor handles one vendor and consumer base branch. Main and release +branches can use different canonical source branches, but require separate +workdirs and explicit launches. A periodic upstream refresh remains a separate +code-changing PR with normal CI: stop its monitor or configure an audited +`--refresh-branch` target before opening it, then migrate the canonical branch +in the existing workdir after merge. See the toolkit guide's +[base-branch setup](../scripts/vendor/README.md#different-consumer-base-branches) +and [refresh cutover](../scripts/vendor/README.md#switching-canonical-branches-after-a-refresh). + +Use the entry points under `scripts/vendor/` directly. The old top-level scripts +and PrimTS-specific CLI aliases are not retained; callers must migrate to the +generic commands and supply their vendor and repository configuration explicitly. + ## License and attribution The vendor lock is a reproducibility record, not a license manifest. Before @@ -310,6 +336,6 @@ files remain exact upstream copies. Use the normal commands with `flashinfer-prims-ts`, for example: ```bash -python scripts/vendor_sources.py status flashinfer-prims-ts -python scripts/vendor_sources.py check flashinfer-prims-ts +python scripts/vendor/manage.py status flashinfer-prims-ts +python scripts/vendor/manage.py check flashinfer-prims-ts ``` diff --git a/AGENTS.md b/AGENTS.md index bb41fdcbf983..f4e20f4873ec 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -193,6 +193,13 @@ The bot-to-bot semantic review workflow is disabled. See [.github/semantic-review.md](.github/semantic-review.md) for its current status and historical implementation details. +### Vendor source tools and promotion + +Use the tools under `scripts/vendor/` to manage locked vendor sources, prepare +lock-only promotions, and optionally monitor source-update PRs. Read +[Vendored Sources](3rdparty/vendor-sources.md) for commands, contributor metadata, +and operator guidance before changing vendor state or publishing promotions. + ### Triggering CI CI is triggered by posting comments on the PR. Basic commands: diff --git a/scripts/vendor/README.md b/scripts/vendor/README.md new file mode 100644 index 000000000000..fcff541083af --- /dev/null +++ b/scripts/vendor/README.md @@ -0,0 +1,505 @@ + + +# Vendor source toolkit + +These tools work with any vendor key in `3rdparty/vendor_sources.lock.yaml`. +They contain no operator credentials, personal repository defaults, or private +configuration. Use Python 3.10+, Git, PyYAML, and, for GitHub promotion/monitoring, +an authenticated `gh` CLI. Promotion and monitoring currently support GitHub.com +repositories and canonical **branches**, not tag promotion or periodic rebases. + +| Entry point | Responsibility | +| --- | --- | +| `python scripts/vendor/manage.py` | Create, inspect, sync, export, patch, pin, and verify vendored trees | +| `python scripts/vendor/promote.py promote` | Promote an immutable, merged source update and prepare a lock-only PR | +| `python scripts/vendor/bot.py run` | Monitor source-update PRs, validate metadata, resolve attribution, and promote | + +Use these entry points directly; no backward compatibility is promised for the +old top-level scripts or PrimTS-specific CLI aliases. Supply vendor and repository +configuration explicitly. The separate legacy Triton packager does not use this +lock and is not migrated by this change. + +## Workflow + +```mermaid +flowchart TD + candidate[Lock-changing PR for the configured consumer and base branch] --> classify{Selected vendor change} + classify -->|new pin on temporary branch| metadata[Author lists upstream PRs in description] + classify -->|temporary reference at current canonical pin| restore[Author restores canonical URL and branch only] + restore --> cleanup[Reconcile obsolete bot feedback without approving code] + classify -->|no promotion needed| cleanup + classify -->|operator-configured refresh target| refresh[Separate upstream refresh PR and normal CI] + metadata --> match[Bot validates metadata and matches lock-selected file changes] + match -->|missing or ambiguous| feedback[Request changes with author-facing instructions] + feedback -->|author corrects block, pin, or targeted resolution| metadata + match -->|valid and source PR merged| verify[Verify ancestry, current lock, and materialization] + verify --> durable[Commit provenance and publish the promotion branch] + durable --> ff[Fast-forward configured canonical source branch] + ff --> promotion[Create or reuse URL/branch-only promotion PR] + promotion --> ci[Request CI skip and enable squash auto-merge] + ci -->|required checks and owner reviews| merged[Verify final lock and squash provenance] + merged --> rebase[Other source-update authors rebase onto the canonical pin] + restart[Restart after local cache loss] --> recover[Rediscover PRs and verify Git provenance] + recover -->|pending recorded operation| verify + recover -->|no record and pin still pending| metadata + recover -->|completed promotion| merged + refresh -->|merged| migrate[Stop bot and verify explicit branch migration] + migrate --> rediscover[Reuse workdir and revalidate GitHub state] + rediscover --> rebase +``` + +Promotion preserves the source SHA, selected files, compatibility patch, and +digests. It does not re-vendor code. A periodic upstream refresh is a separate +code-changing operation and must not use the promotion CI-skip path. + +## Operator setup + +Use your own approved GitHub credentials (`gh auth login`, `GH_CONFIG_DIR`, or +the usual GitHub CLI token environment). Configure Git authentication for pushes +and a real Git name/email for DCO-signed commits. Tokens are never accepted in +repository URLs, stored in SQLite, or copied into repository configuration by +these tools. Do not put credentials in command-line arguments. + +The operator needs permission to push the configured canonical source branch +and consumer publishing fork, create consumer PRs/comments/reviews, request the +CI-skip path, and enable auto-merge. Review dismissal can require additional +repository authorization. No GitHub App, repository-admin access, webhook +server, or systemd unit is required merely to run the monitor. Repository rules, +SSO/token policies, CI-bot authorization, and required owner reviews still apply. + +Run only one **publishing** monitor per consumer repository/base/vendor. If +multiple monitors share a canonical source branch, serialize their publications +as well. The workdir lock prevents duplicate processes using the same directory; +it is not a distributed lock between operators, machines, or workdirs. Finish +one promotion before merging the next source update for that consumer/base/vendor. +Unexpected concurrent changes stop publication rather than force-pushing or +selecting an arbitrary newer pin. + +## Contributor metadata + +Authors add exactly one block to the source-update PR description. Multiple +vendors can share it; each monitor reads its configured vendor entry: + +````markdown + +```yaml +schema_version: 1 +vendors: + example-library: + upstream_prs: + - https://github.com/UPSTREAM/library/pull/123 + - https://github.com/UPSTREAM/library/pull/456 +``` + +```` + +No complete commit map is required. The bot compares only files selected by the +vendor lock's `source` directory and `include` patterns, using the same selection +rules as vendoring, on both sides of each comparison. For `flashinfer-prims-ts`, +this currently selects `flashinfer/attention/prims_ts/**/*.py`: differences in +external tests, benchmarks, or unselected documentation do not prevent a match. +Changing `source` or `include` during the source update requires a separate +selection change; automatic attribution refuses to silently narrow its scope. + +The selected source delta is compared against immutable upstream PR snapshots. +Matching uses commit identity, zero-context edits, and whole-series aggregate +changes for squash/split cases. +Edit matching ignores unchanged neighboring lines, line numbers, blob IDs, and +Git-generated hunk labels. It preserves changed-line whitespace, file paths/modes, +binary data, and hunk boundaries. Function definitions that are actually added or +removed remain part of the comparison; only labels in hunk headers are ignored. +Identical edits at different locations in the same file can therefore match; +this establishes textual attribution, not semantic equivalence. Exact matches +take priority; multiple exact matches remain ambiguous. +Merged upstream PRs can still be paired using their original head history. + +When a source commit has no exact match, the bot compares its normalized patch +lines against each upstream commit and PR aggregate. A unique upstream PR with +similarity **>= 0.9** is accepted. The metric is Python +`SequenceMatcher(autojunk=False)`: `2 * matched_lines / (source_lines + upstream_lines)`. +Whitespace remains significant, repeated lines are not discarded, and empty +patches never establish similarity. The unrounded score determines acceptance; +comments display rounded percentages. Whole-source-series matching still requires +exact equality, so aggregate similarity cannot hide an unmatched individual commit. + +Comments show accepted non-exact matches with their scores and threshold. Below +the threshold, or when multiple PRs qualify, author-action feedback shows the +closest upstream PRs and scores. Similarity matches record the score, metric, +threshold, compared immutable revisions, and `retain-until-verified` refresh policy +in promotion provenance. They count as attribution for promotion, not code-review +approval or proof that changes can be dropped during refresh. Commit titles and +cherry-pick trailers are not evidence. Authors still resolve ambiguous or +below-threshold matches explicitly. + +Commits with no lock-selected changes need no upstream attribution or author +resolution. Empty selected patches never count as an upstream match; omit PRs +that cover only excluded changes. The promotion record saves `attribution_scope` +and marks excluded-only commits `outside-vendor-scope`, in an unpaired group with +`refresh_policy: retain`. All source commits still undergo the linear-history +and ancestry checks, and the promoted SHA and vendored-tree verification remain +unchanged. Scoped attribution does not prove whole-commit equivalence or authorize +dropping changes to excluded files during a future source-branch refresh. + +If attribution is unresolved, feedback lists the full source SHAs and candidate +PRs. **The PR author**, not the operator, resolves it by adding missing upstream +PRs, removing unrelated PRs, correcting/rebasing the pinned source, or adding a +targeted explanation under that vendor: + +```yaml +resolutions: + FULL_40_CHARACTER_SOURCE_SHA: + upstream_pr: https://github.com/UPSTREAM/library/pull/123 + reason: Explain the downstream adaptation or why this PR owns the change. +``` + +This is an explicit author assertion, not an automatic equivalence claim. The +final provenance records its reason and `retain-until-verified` refresh policy. +When there is no upstream PR for the update, replace `upstream_prs` with a +nonempty `unpaired_reason`. Do not combine those alternatives. Unpaired changes +are retained during future refresh until equivalence is established separately. + +Before a new operation is committed, the source head and normalized metadata +are rechecked before feedback and publication. Metadata corrections rerun +validation. Durable promotion records remain authoritative; later description +edits or upstream PR changes do not rewrite a recorded operation's provenance. + +## Launch the local monitor + +All operator-specific settings are CLI inputs. Start with one dry-run pass: + +```bash +python scripts/vendor/bot.py run \ + --vendor example-library \ + --consumer-repo NVIDIA/TensorRT-LLM \ + --base-branch main \ + --upstream-repo UPSTREAM/library \ + --canonical-repo MAINTAINER/library \ + --canonical-branch consumer-dev \ + --fork YOUR_ACCOUNT/TensorRT-LLM \ + --workdir /path/to/private/vendor-bot-state \ + --once +``` + +For PrimTS, use vendor `flashinfer-prims-ts`, upstream +`flashinfer-ai/flashinfer`, and the operator's current canonical fork/branch. +The canonical repository is always explicit: the bot never adopts a contributor +fork as the publishing destination merely because it appears in a lock or PR. + +Omit `--once` to run continuously in the foreground. Add **both** `--daemon` and +`--publish` to detach and enable remote writes. `--daemon` alone remains dry-run. +The bot always enables squash auto-merge when publishing. It retains the +complete signed-off provenance message and verifies the final merge later. + +```bash +python scripts/vendor/bot.py status --workdir /path/to/private/vendor-bot-state +python scripts/vendor/bot.py stop --workdir /path/to/private/vendor-bot-state +``` + +`stop` requests a graceful exit after the current operation; it does not kill an +unrelated process using a stale PID. Logs go to `WORKDIR/bot.log` in daemon mode. +The daemon survives terminal closure but does not automatically restart after a +crash or reboot; a supervisor is optional. Stop before changing launch settings. + +The workdir must be empty on first use, owned by the current user, and outside a +Git worktree. It holds a disposable SQLite cache, a bare source cache, process-lock +and stop files, and isolated promotion worktrees. Runtime state is not committed. +An existing workdir is bound to its operator, vendor, and consumer base branch; +use a different directory for another operator/vendor/base. Authentication is inherited +from the launching environment and is not persisted in that state. + +### Different consumer base branches + +Each monitor handles one `--base-branch` (default `main`) and one +`--canonical-branch`. Different consumer branches may use different canonical +source branches: run separate instances with separate workdirs. Discovery, +promotion provenance, follow-up PR targets, and migration checks are scoped to +the consumer repository and base branch. Retargeting a tracked PR makes it +ineligible for its old monitor; changing `--base-branch` requires a new workdir. + +For example, a main monitor can use `--base-branch main --canonical-branch +consumer-dev-new`, while a release monitor uses `--base-branch release/1.3 +--canonical-branch consumer-dev`. These are independent opt-in launches; +configuring one does not start the other. If both consumers later share one +canonical source branch, serialize publication across them: separate workdir +locks do not coordinate concurrent pushes to the same source branch. + +### Switching canonical branches after a refresh + +Before opening a refresh PR, either stop the monitor for its target base branch +until the refresh merges, or restart it with an explicit refresh target: + +```bash +--canonical-branch consumer-dev \ +--refresh-branch consumer-dev-YYYYMMDD +``` + +`--refresh-branch` is repeatable and applies only to `--canonical-repo`. It tells +the monitor that a PR pointing at that branch is a manual refresh, not a +temporary-branch update requiring promotion metadata. The monitor does not +promote it, skip its CI, or infer refresh targets from contributor metadata or +branch naming conventions. Normal review and CI still apply to the refresh. +The operator must audit these targets; do not list contributor temporary branches. +The option does not authorize publishing to a new branch or start another monitor. +Changing this policy on restart is allowed in the same workdir without migrating +the canonical configuration. Reuse the policy after cache loss. If the monitor +previously requested metadata changes on the open refresh PR, it updates its +existing feedback and dismisses its obsolete automated review when authorized. +If dismissal is forbidden, the comment asks an authorized maintainer to dismiss it. + +Keep the same workdir when a periodic source refresh introduces a new canonical +branch. Stop the publishing process for the cutover, and wait for the consumer +refresh PR to merge. Relaunch with the same configuration except for: + +```bash +--canonical-branch consumer-dev-YYYYMMDD \ +--migrate-canonical-branch-from consumer-dev \ +--workdir /path/to/private/vendor-bot-state \ +--once +``` + +First omit `--publish`: migration only changes local configuration/cache state; +this pass makes no remote writes. The bot verifies the merged base lock names +the new canonical repository/branch and its pin is reachable from that branch. +It refuses outstanding old-branch promotion PRs or unpublished promotion branches +for the same consumer repository and base branch. Promotions for other consumer +bases do not block this migration; missing or malformed scope provenance still +requires maintainer inspection. +Finish those operations, or explicitly close superseded follow-up PRs after +maintainer review; migration never deletes branches or redirects old promotions. +Other operator/vendor identity changes still require a separate workdir. +If the refresh has merged but the old monitor is still running, new updates based +on a configured refresh target report a maintainer cutover action, not a request +to promote the lock back to the old canonical branch. After migration, the +`--refresh-branch` entry for the now-current canonical branch can be omitted. + +The configuration update and invalidation are one SQLite transaction. Old +branch-dependent results are archived, while logs, source caches, promotion +worktrees, tracked candidates, and the historical `--since` cutoff are retained. +Discovery restarts and rechecks candidates against Git/GitHub. Old successful +polls are cleared from active health status; `status` reports the canonical +repository/branch and the verified migration revision. Existing Git promotion +provenance is never rewritten. Repeating the same migration is a no-op and does +not discard subsequent discovery progress. + +After inspecting the dry-run results, restart with the new branch and +`--daemon --publish`, omitting `--migrate-canonical-branch-from`. Verify a completed +poll under the new configuration, then notify outstanding source-update PRs to +rebase their temporary source branches, update their consumer PRs and pins, and +regenerate vendor snapshots. If SQLite is lost, launch normally using the new +canonical configuration and the same cutoff; omit the migration flag. + +The default interval is 120 seconds (`--interval`, minimum 30). A cold start +discovers existing open PRs and backfills merged PRs from all history. This scan +can be expensive; subsequent polls use a cached cursor and tracked candidates. +An explicit `--since 2026-01-01T00:00:00Z` restricts historical discovery; use the +same cutoff on every launch, including recovery. Historical updates belonging +to another canonical branch, or superseded pins without a recoverable promotion, +are not promoted or sent author feedback. + +Open PRs qualify for promotion only when they change the selected vendor's source commit +relative to their Git merge base. A stale PR head after a canonical refresh is +not itself a source update: changes only to other vendors or patch/digest fields +do not require promotion metadata. Eligible updates are still checked against +the latest canonical lock, so their source branches must include the refreshed +canonical history before promotion. + +A same-commit temporary URL/branch redirect is not a source update, but can still +need author feedback: if the PR itself changes the reference and its pin equals +the current canonical base pin, restore the base's canonical URL/branch only. +No promotion metadata is required. When an open PR no longer needs promotion, +the bot reconciles any existing feedback instead of leaving a stale objection. +It does not post new feedback on otherwise ignored PRs. + +### Monitoring health + +`--log-level INFO` is the default, in both foreground and daemon mode. Logs use +UTC timestamps. INFO includes poll start/end, discovery counts, elapsed time, +PR metadata/feedback and promotion transitions, and the next polling deadline. +Unchanged PR outcomes are not logged repeatedly. `WARNING`/`ERROR` suppress the +INFO heartbeat and summaries; DEBUG is not needed for normal monitoring. Git +and promotion subprocess output still goes to the daemon log independently of +the Python logging threshold. + +During an active operation, a log-only heartbeat runs every 30 seconds, including +while the worker waits for GitHub or Git. It reports the phase, PR number (when +known), operation age, and counters. A heartbeat proves only that the reporting +thread is alive: unchanged counters and growing operation age may indicate a +slow or stuck operation. It does not advance the progress or completion timestamp. +Idle intervals are described by `Next poll at ...`, not continuous heartbeat noise. + +`status` reads the local cache without creating or repairing a database: + +```bash +python scripts/vendor/bot.py status --workdir /path/to/private/vendor-bot-state +tail -n 100 /path/to/private/vendor-bot-state/bot.log +``` + +The JSON separates process liveness from work completion: + +| Field | Meaning | +| --- | --- | +| `running`, `pid`, `publish` | Process lock held, recorded PID, and whether remote writes are enabled; not a health verdict | +| `vendor`, `base_branch`, `canonical_repo`, `canonical_branch` | Cached monitor scope and publishing destination; verify these after a restart or migration | +| `refresh_branches`, `migration` | Explicit refresh-target policy and last verified canonical-branch migration; not authorization to publish to a refresh target | +| `operation`, `progress_at` | Current phase/PR/start time and last worker checkpoint | +| `counts` | Current poll's scanned records, tracked lock PRs, relevant vendor updates, processed PRs, author-blocked PRs, and errors | +| `last_completed_poll` | Last fully scanned/processed poll, even if individual PRs needed attention | +| `last_successful_poll` | Last completed poll with no blocked PRs or processing errors; not a code-review approval or guarantee that promotions have merged | +| `last_poll_summary` | Duration, counts, and outcome: `ok`, `needs_attention`, `failed`, or `interrupted` | +| `last_error` | Historical error timestamp, PR, and bounded single-line diagnostic; retained after recovery | +| `next_poll_at` | Next scheduled poll, including exponential backoff after discovery failures | +| `state_available`, `state_error` | Whether cached telemetry could be read; missing/corrupt state does not mean a held process lock is healthy | + +`scanned` counts PR records visited, not unique PRs (the initial open/history +scans overlap). `tracked` includes lock changes for other vendors; `relevant` +counts the selected vendor's eligible updates before promotion-state checks. +Blocked PRs are author/metadata issues; errors are failed processing attempts. +Logs include their PR numbers and reasons. Diagnostic summaries redact configured +GitHub tokens, recognizable token formats, URL credentials, and authorization +headers, and omit multiline YAML excerpts; no PR bodies or authentication +configuration are intentionally logged. + +For monitoring, check the latest poll outcome and deadline along with the log. +Do not interpret `running: true` or an old `last_error` alone as health/failure. +Stopped processes have no current PID, operation, or next deadline in `status`, +but retain historical poll/error information. Telemetry is disposable and never +authorizes publication; restart recovery still verifies Git/GitHub provenance. + +### Recovery after cache loss + +Git/GitHub are the source of truth; SQLite is not required to reconstruct a +promotion. Stop the old process before replacing or discarding its database. +Relaunch with the **same account, repositories, consumer base branch, canonical +branch, and cutoff**, and re-supply any `--refresh-branch` policy. +The bot accepts an existing workdir containing only recognized bot artifacts +even if SQLite is missing. It preserves those artifacts and refuses unrelated +files and symlinks. A new empty workdir also works; existing registered Git +worktrees are validated and reused, or recreated when their directories are gone. + +Recovery checks the deterministic promotion PR/branch and its signed-off commit +before parsing the source PR description or consulting upstream PR heads. It +verifies the operation identity, lock-only diff, and provenance, then resumes +the recorded operation. Completed promotions are verified against their squash +commit without requiring the old source fork or mutable PR metadata. Existing +feedback, reviews, CI-skip comments, and auto-merge settings are discovered from +GitHub rather than duplicated from missing local IDs. + +Before the first canonical-branch push, the signed promotion commit is pushed to +the configured publishing fork. Therefore loss after any remote publication +step retains a durable provenance record. A locally committed but not yet pushed +operation can also be recovered from the consumer repository's Git branch. +Before any provenance commit exists, attribution is recomputed from current +remote inputs; a previous successful match in SQLite is never authoritative. + +Recovery assumes a single publishing operator and accessible, unmodified Git/GitHub +artifacts. It is not recovery from deleted remote history or a replacement +publishing account/fork. Tampered commits, unexpected branch moves, closed-unmerged +promotion PRs, and unrelated worktree edits still stop the operation for inspection. + +One process monitors one vendor; separate workdirs can monitor different vendors. +`--consumer-repo`, `--base-branch`, and `--upstream-branch` also support consumers +and upstreams whose default branch is not `main`. `--repo` selects a **trusted** +local consumer checkout; no code or hooks from source forks are executed. + +## Feedback and review safety + +With `--publish`, recognized author-actionable problems produce an updatable +comment containing the specific problem, its location, and a **Fix** section. +Examples are included only when relevant, using the configured vendor and +upstream repository. Missing metadata gets a complete copyable marked block; +field errors get focused instructions; attribution failures get a correctly +nested resolution example. Existing entries for other vendors must be preserved. +Edit the **PR description**, not a comment, when correcting metadata. The bot +rechecks automatically on later polls; no manual recheck comment is required. + +| Problem | What the bot tells the author to fix | +|---------|-------------------------------------| +| Missing markers | Names exactly which of `` and `` is missing, and where to insert it. | +| Duplicate or reversed markers | Reports marker counts or ordering; asks the author to consolidate metadata into one block without losing vendor entries. | +| Malformed YAML or schema | Reports YAML line/column or the affected field path and the expected structure, without copying raw parser excerpts. YAML line 1 is the first line inside the fence. | +| Invalid upstream PR or resolution | Identifies the list entry, duplicated PR, source SHA, missing reason, or unlisted PR, and gives the correction. | +| Unresolved attribution | Shows candidate PRs and similarity scores where applicable; explains how to correct the PR list/source pin or supply a targeted author assertion. | +| Source-history or lock-selection problem | Explains the rebase or lock-file change and vendor regeneration needed. Rebase guidance identifies the canonical pin. Metadata edits alone cannot fix these failures. | +| Same-commit temporary reference | If the PR redirects the source URL/branch but pins the current canonical commit, asks the author to restore the canonical URL/branch from its target base. Keep the commit, other lock fields, vendored files, and patch unchanged; no upstream-PR metadata is needed. | + +`unpaired_reason` is appropriate only when there genuinely is no paired upstream +PR, not as a workaround for failed matching. The similarity threshold, matching +rules, and retained provenance are unchanged by diagnostic formatting. + +Eligible open PRs receive one metadata-only `REQUEST_CHANGES` review for an +author-actionable problem. Waiting for a preceding promotion is identified as +**maintainer action**, with a comment but no new change-request review. Unexpected +API, authentication, network, and local-cache failures stay in operator logs and +`status`; they do not post author-fix instructions or request changes. The bot +deduplicates feedback, including after SQLite loss, updates its existing review +body when the author-actionable problem changes, and never submits `APPROVE` +just because metadata is valid. Same-commit reference checks compare the PR's +own changes against its Git merge base; inherited references and explicitly +configured refresh targets are not treated as unnecessary temporary redirects. + +After correction it dismisses only its own marked metadata reviews, when +authorized. If an open PR no longer needs promotion, it also reconciles existing +comments/reviews from GitHub, without requiring SQLite history. It updates the +comment to explain that the previous request no longer applies, rather than +claiming metadata validation succeeded. It posts no feedback on an ignored PR +unless it already has a bot comment or an unresolved bot review to reconcile. +If dismissal is forbidden, an authorized human must dismiss the +review; the bot reports the failure rather than approving the code. If the +authenticated account already has a manual approval/objection, the bot uses +comments so it does not supersede that review. It likewise uses comments on the +operator's own PRs. Polling and review permissions do not create a mandatory +pre-merge gate: strict blocking requires repository-admin rules. Promotion still +refuses invalid metadata even if the source PR merges between polls. + +## Manual promotion and recovery + +The same generic engine can run without a daemon: + +```bash +python scripts/vendor/promote.py promote \ + --vendor example-library \ + --source-pr 123 \ + --upstream-repo UPSTREAM/library \ + --canonical-repo MAINTAINER/library \ + --canonical-branch consumer-dev \ + --fork YOUR_ACCOUNT/TensorRT-LLM \ + --from-pr-description \ + --publish --auto-merge +``` + +Omit the final flags for a dry run. Alternatively supply repeated `--upstream-pr` +with `--auto-match`, or an explicit `--unpaired-reason`. The existing +`--map-upstream SHA=PR` interface remains available for manual provenance +assertions but cannot accompany automatic matching. `--source-repo` selects a +local immutable source checkout for materialization; `--worktree` overrides the +follow-up checkout location. `--wait` optionally waits for and verifies merge. + +Publication checks the merged source PR, expected previous canonical lock, +linear fast-forward ancestry, current consumer pin, complete materialization, +and full lock-only diff. It uses non-force pushes and DCO-signed commits. The +`VENDOR PROMOTION V1` JSON block in the commit/squash message records vendor, +source/consumer identities, source SHAs, upstream snapshots, and attribution. +Historical `PRIMTS PROMOTION V1` messages remain readable. No provenance file is +added to the consumer repository. + +## Tests + +Tests use temporary Git repositories and fake GitHub responses, including a real +detached-process smoke test with a fake `gh`; no credentials, network, or GPU: + +```bash +PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 pytest --noconftest -c /dev/null \ + -p no:cacheprovider -o 'markers=cpu_only: CPU-only test' \ + tests/unittest/others/test_vendor_sources.py \ + tests/unittest/others/test_maintain_prims_ts.py \ + tests/unittest/others/test_vendor_bot.py +``` + +These suites carry `cpu_only` markers and are included in normal CPU CI through +the `unittest/others` parent entry in +`tests/integration/test_lists/test-db/l0_cpu.yml`. The standalone command above +avoids loading the TensorRT-LLM runtime or unrelated pytest plugins; it does not +replace normal CI for changes to the tools. diff --git a/scripts/vendor/__init__.py b/scripts/vendor/__init__.py new file mode 100644 index 000000000000..424dcded61bf --- /dev/null +++ b/scripts/vendor/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Source vendoring, reviewed-revision promotion, and local automation.""" diff --git a/scripts/vendor/bot.py b/scripts/vendor/bot.py new file mode 100644 index 000000000000..6df61a6705fc --- /dev/null +++ b/scripts/vendor/bot.py @@ -0,0 +1,1271 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Local, restartable vendor-promotion monitor. Remote writes require --publish.""" + +from __future__ import annotations + +import argparse +import contextlib +import datetime +import fcntl +import json +import logging +import os +import re +import signal +import sqlite3 +import subprocess +import sys +import threading +import time +from collections.abc import Iterator +from pathlib import Path + +if __package__: + from . import manage, metadata, promote + from .diagnostics import Diagnostic, field +else: + import manage + import metadata + import promote + from diagnostics import Diagnostic, field + +_LOG = logging.getLogger("vendor-bot") +_STATE_FILE = "state.sqlite" +_STOP = threading.Event() +_HEARTBEAT_SECONDS = 30 + + +def _now() -> str: + return datetime.datetime.now(datetime.timezone.utc).isoformat() + + +def _summary(error: object) -> str: + """Keep diagnostics single-line; exclude YAML excerpts and authentication secrets.""" + text = str(error).splitlines()[0] if str(error) else type(error).__name__ + for name in ("GH_TOKEN", "GITHUB_TOKEN", "GH_ENTERPRISE_TOKEN", "GITHUB_ENTERPRISE_TOKEN"): + if os.environ.get(name): + text = text.replace(os.environ[name], "[REDACTED]") + text = re.sub(r"https?://[^\s/]+@", "https://[REDACTED]@", text) + text = re.sub(r"\b(?:gh[pousr]_[A-Za-z0-9_]+|github_pat_[A-Za-z0-9_]+)", "[REDACTED]", text) + text = re.sub(r"(?i)(authorization\s*[:=]\s*).+", r"\1[REDACTED]", text) + return text[:500] + + +class Store: + """Disposable reconciliation cache; Git/GitHub own promotion provenance and state.""" + + def __init__(self, path: Path) -> None: + self.connection = sqlite3.connect(path) + self.connection.execute( + "CREATE TABLE IF NOT EXISTS state (key TEXT PRIMARY KEY, value TEXT NOT NULL)" + ) + + def get(self, key: str, default: object = None) -> object: + row = self.connection.execute("SELECT value FROM state WHERE key = ?", (key,)).fetchone() + return json.loads(row[0]) if row else default + + def put(self, key: str, value: object) -> None: + with self.connection: + self.connection.execute( + "INSERT INTO state VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", + (key, json.dumps(value, sort_keys=True)), + ) + + def close(self) -> None: + """Close SQLite before releasing the process lock.""" + self.connection.close() + + def migrate_branch(self, settings: dict, record: dict) -> None: + """Atomically archive derived results and start reconciliation for a new branch.""" + keys = ("cursor", "bootstrapped", "health", "last_poll") + prefixes = ("complete:", "promotion:", "observation:", "problem:") + rows = self.connection.execute("SELECT key, value FROM state").fetchall() + archived = { + key: json.loads(value) for key, value in rows if key in keys or key.startswith(prefixes) + } + history = self.get("branch_migrations", []) + history.append({**record, "previous_results": archived}) + with self.connection: + self.connection.executemany( + "DELETE FROM state WHERE key = ?", [(key,) for key in archived] + ) + self.connection.executemany( + "INSERT INTO state VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", + [ + (key, json.dumps(value, sort_keys=True)) + for key, value in ( + ("settings", settings), + ("branch_migrations", history), + ("migration", record), + ) + ], + ) + + +class Progress: + """Main-thread status checkpoints with an independent, log-only heartbeat.""" + + def __init__(self, store: Store) -> None: + self.store = store + previous = store.get("health", {}) + self.state = { + "operation": None, + "progress_at": None, + "poll_started_at": None, + "last_completed_poll": previous.get("last_completed_poll"), + "last_successful_poll": previous.get("last_successful_poll"), + "last_poll_summary": previous.get("last_poll_summary"), + "last_error": previous.get("last_error"), + "next_poll_at": None, + "counts": {}, + } + self.lock = threading.Lock() + self.last_checkpoint = time.monotonic() + + def update(self, **fields: object) -> None: + with self.lock: + self.state.update(fields) + self.store.put("health", self.state) + self.last_checkpoint = time.monotonic() + + def operation(self, phase: str, number: int | None = None) -> None: + self.update( + operation={"phase": phase, "pr": number, "started_at": _now()}, progress_at=_now() + ) + + def count(self, name: str, amount: int = 1) -> None: + with self.lock: + self.state["counts"][name] = self.state["counts"].get(name, 0) + amount + if time.monotonic() - self.last_checkpoint >= _HEARTBEAT_SECONDS: + self.update(progress_at=_now()) + + def error(self, error: object, number: int | None = None) -> None: + self.update(last_error={"at": _now(), "pr": number, "message": _summary(error)}) + + def heartbeat(self) -> None: + with self.lock: + operation = self.state["operation"] + if operation is None or operation["phase"] in ("waiting", "stopped"): + return + age = ( + datetime.datetime.now(datetime.timezone.utc) + - datetime.datetime.fromisoformat(operation["started_at"]) + ).total_seconds() + _LOG.info( + "Heartbeat: phase=%s pr=%s operation_age=%.1fs counts=%s; " + "worker alive, operation completion not yet confirmed", + operation["phase"], + operation["pr"], + age, + self.state["counts"], + ) + + @contextlib.contextmanager + def heartbeats(self) -> Iterator[None]: + finished = threading.Event() + + def report() -> None: + while not finished.wait(_HEARTBEAT_SECONDS): + self.heartbeat() + + thread = threading.Thread(target=report, name="vendor-bot-heartbeat", daemon=True) + thread.start() + try: + yield + finally: + finished.set() + thread.join() + + +def _pages(gh: promote.GitHub, endpoint: str) -> Iterator[dict]: + separator = "&" if "?" in endpoint else "?" + page = 1 + while True: + values = gh.api(f"{endpoint}{separator}per_page=100&page={page}") + if not isinstance(values, list): + raise ValueError(f"Expected a list from {endpoint}.") + yield from values + if len(values) < 100: + return + page += 1 + + +class Monitor: + """One vendor/base-branch pair; run other pairs in separate workdirs.""" + + def __init__( + self, + args: argparse.Namespace, + gh: promote.GitHub, + store: Store, + progress: Progress | None = None, + ) -> None: + self.args, self.gh, self.store = args, gh, store + self.progress = progress or Progress(store) + self.actor = gh.get("user")["login"] + self.prefix = f"repos/{gh.consumer_repo}" + self.marker = f"" + self.refresh_branches = frozenset(args.refresh_branch) + settings = { + "consumer_repo": gh.consumer_repo, + "vendor": gh.vendor_name, + "upstream_repo": gh.upstream_repo, + "base_branch": gh.base_branch, + "upstream_branch": gh.upstream_branch, + "canonical_repo": args.canonical_repo, + "canonical_branch": args.canonical_branch, + "fork": args.fork, + "actor": self.actor, + "repo": str(args.repo), + } + saved = store.get("settings") + migration_from = args.migrate_canonical_branch_from + if saved is not None and saved != settings: + changed = { + key for key in saved.keys() | settings.keys() if saved.get(key) != settings.get(key) + } + if changed != {"canonical_branch"}: + raise ValueError( + "Workdir belongs to a different operator/vendor configuration; choose another." + ) + if migration_from != saved["canonical_branch"]: + raise ValueError( + "Canonical branch changed; stop the old process and restart with " + f"--migrate-canonical-branch-from {saved['canonical_branch']} after the refresh merges." + ) + record = self._verify_branch_migration(migration_from) + store.migrate_branch(settings, record) + self.progress.update( + last_completed_poll=None, + last_successful_poll=None, + last_poll_summary=None, + last_error=None, + next_poll_at=None, + ) + _LOG.info( + "Canonical branch migrated: %s -> %s; rediscovery required", + migration_from, + args.canonical_branch, + ) + elif migration_from: + record = store.get("migration", {}) + if ( + saved == settings + and record.get("from") == migration_from + and record.get("to") == args.canonical_branch + ): + _LOG.info( + "Canonical branch migration already applied: %s -> %s", + migration_from, + args.canonical_branch, + ) + else: + raise ValueError( + "No matching saved branch migration. For a fresh/cache-loss start, omit " + "--migrate-canonical-branch-from and use the current canonical configuration." + ) + store.put("settings", settings) + # Refresh targets are an operator policy, not publication destinations. + # Changing them does not migrate the canonical branch or its provenance. + store.put("refresh_branches", sorted(self.refresh_branches)) + if store.get("since") is None: + store.put( + "since", + args.since or "1970-01-01T00:00:00Z", + ) + + def _verify_branch_migration(self, previous_branch: str) -> dict: + """Read authoritative remote state before changing the local configuration.""" + self.progress.operation("verifying_branch_migration") + revision = promote._main_sha(self.gh) + current = promote._lock_at(self.gh, self.gh.consumer_repo, revision) + if ( + promote._repo_name(current.url).lower() != self.args.canonical_repo.lower() + or current.branch != self.args.canonical_branch + ): + raise ValueError( + "Branch migration requires the merged base-branch lock to point to the new " + "canonical repository/branch. Finish the refresh before restarting." + ) + ref = self.gh.get( + f"repos/{self.args.canonical_repo}/git/ref/heads/{self.args.canonical_branch}" + ) + tip = promote._sha(ref["object"]["sha"]) + if self.gh.compare(self.args.canonical_repo, current.commit, tip)["status"] not in ( + "identical", + "ahead", + ): + raise ValueError( + "New canonical branch does not contain the merged lock's pinned commit." + ) + self._check_pending_promotions(previous_branch) + latest = promote._lock_at(self.gh, self.gh.consumer_repo, promote._main_sha(self.gh)) + if latest.to_mapping() != current.to_mapping(): + raise ValueError("Base vendor lock changed during migration; inspect it and retry.") + return { + "from": previous_branch, + "to": self.args.canonical_branch, + "at": _now(), + "consumer_revision": revision, + "canonical_tip": tip, + "pin": current.commit, + } + + def _check_pending_promotions(self, previous_branch: str) -> None: + """Do not strand open follow-ups or unpublished promotion branches.""" + prefix = f"chore/{self.gh.vendor_name}-promote-" + branches = { + item["name"]: item["commit"]["sha"] + for item in _pages(self.gh, f"repos/{self.args.fork}/branches") + if item["name"].startswith(prefix) + } + local = promote._git( + self.args.repo, + "for-each-ref", + "--format=%(refname:short) %(objectname)", + f"refs/heads/{prefix}*", + ) + for line in local.splitlines(): + branch, sha = line.split() + if branch not in branches: + branches[branch] = sha + # Open follow-ups can survive deletion of their publishing branch. + for pr in _pages(self.gh, f"{self.prefix}/pulls?state=open"): + branch = pr["head"]["ref"] + if pr["base"]["ref"] == self.gh.base_branch and branch.startswith(prefix): + branches[branch] = pr["head"]["sha"] + for branch, sha in branches.items(): + remote = self.gh.optional(f"{self.prefix}/commits/{sha}") + if remote is None: + remote = self.gh.optional(f"repos/{self.args.fork}/commits/{sha}") + message = ( + remote["commit"]["message"] + if remote + else promote._git(self.args.repo, "log", "-1", "--format=%B", sha) + ) + record = promote._record_from_message(message) + if record.get("canonical_branch") != previous_branch: + continue + consumer_pr = record.get("consumer_pr") + consumer_base = record.get("consumer_base") + consumer = ( + re.fullmatch(r"https://github\.com/([\w.-]+/[\w.-]+)/pull/[1-9][0-9]*", consumer_pr) + if isinstance(consumer_pr, str) + else None + ) + if consumer is None or not isinstance(consumer_base, str) or not consumer_base: + raise ValueError( + f"Missing or invalid consumer provenance on promotion branch {branch}; " + "inspect its promotion commit before migrating." + ) + if ( + consumer.group(1).lower() != self.gh.consumer_repo.lower() + or consumer_base != self.gh.base_branch + ): + continue + if record.get("vendor") != self.gh.vendor_name: + raise ValueError(f"Unexpected vendor provenance on promotion branch {branch}.") + prs = list( + _pages( + self.gh, + f"{self.prefix}/pulls?state=all&head={self.args.fork.split('/')[0]}:{branch}" + f"&base={self.gh.base_branch}", + ) + ) + if len(prs) != 1 or prs[0]["state"] != "closed": + location = prs[0]["html_url"] if len(prs) == 1 else branch + raise ValueError( + f"Unresolved old-branch promotion: {location}. Finish it, or explicitly " + "close its superseded PR before migrating. Unpublished branches need " + "maintainer inspection; migration never deletes or redirects them." + ) + + def _discover(self) -> list[int]: + # Advance only after a complete scan, with overlap for edits arriving + # during pagination. Tracked operations are reconciled independently. + cutoff = ( + datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(minutes=5) + ).strftime("%Y-%m-%dT%H:%M:%SZ") + tracked = set(self.store.get("tracked", [])) + endpoints = [f"{self.prefix}/pulls?state=all&sort=updated&direction=desc"] + if not self.store.get("bootstrapped", False): + endpoints.insert(0, f"{self.prefix}/pulls?state=open") + since = self.store.get("cursor", self.store.get("since")) + _LOG.info( + "Discovery started: since=%s cold_start=%s", + since, + not self.store.get("bootstrapped", False), + ) + for endpoint in endpoints: + self.progress.operation( + "discover_open" if "state=open" in endpoint else "discover_history" + ) + for pr in _pages(self.gh, endpoint): + if _STOP.is_set() or (self.args.workdir / "stop").exists(): + _STOP.set() + return sorted(tracked) + if "state=all" in endpoint and pr["updated_at"] < since: + break + self.progress.count("scanned") + if pr["base"]["ref"] != self.gh.base_branch: + continue + number = pr["number"] + if pr["state"] == "closed" and not pr.get("merged_at"): + continue + head = pr["head"]["sha"] + key = f"candidate:{number}" + cached = self.store.get(key, {}) + if cached.get("head") != head: + relevant = any( + item["filename"] == promote._LOCK + for item in _pages(self.gh, f"{self.prefix}/pulls/{number}/files") + ) + cached = {"head": head, "relevant": relevant} + self.store.put(key, cached) + if cached["relevant"]: + tracked.add(number) + self.store.put("tracked", sorted(tracked)) + self.store.put("bootstrapped", True) + self.store.put("cursor", cutoff) + self.progress.update(progress_at=_now()) + _LOG.info( + "Discovery completed: scanned=%s tracked_lock_prs=%s", + self.progress.state["counts"].get("scanned", 0), + len(tracked), + ) + return sorted(tracked) + + def _transition(self, number: int, state: str, reason: str = "") -> None: + observation = {"state": state, "reason": _summary(reason) if reason else ""} + if self.store.get(f"observation:{number}") != observation: + _LOG.info("PR #%s: %s%s", number, state, f": {observation['reason']}" if reason else "") + self.store.put(f"observation:{number}", observation) + self.store.put(f"problem:{number}", observation["reason"] if state == "blocked" else None) + + def _feedback( + self, + pr: dict, + problem: Diagnostic | None, + *, + attribution: dict[str, dict] | None = None, + no_promotion: bool = False, + ) -> None: + number = pr["number"] + state = "blocked" if problem else "ignored" if no_promotion else "metadata_valid" + message = ( + f"@{pr['user']['login']} — vendor promotion needs {problem.owner} action:\n\n{problem}" + "\n\nThe bot will recheck automatically on a later poll." + if problem + else "Vendor promotion metadata and source attribution are valid. " + "This is **not** a code-review approval. Promotion still requires the source PR to merge." + ) + if no_promotion: + message = ( + f"This PR does not require vendor promotion for {field(self.gh.vendor_name)} " + "under this monitor's configuration. The previous promotion-metadata request " + "no longer applies; no upstream-PR metadata is required by this monitor. " + "This is **not** a code-review approval. Normal review and CI still apply." + ) + if attribution: + details = metadata.similarity_feedback(attribution, self.gh.upstream_repo) + if details: + message += "\n\n" + details + body = f"{self.marker}\n{message}" + dismissal_notice = ( + "\n\nAn authorized reviewer must dismiss the automated metadata " + "review; this account lacks dismissal permission." + ) + if not self.args.publish: + self._transition( + number, + state, + str(problem) if problem else "dry run; no feedback published", + ) + return + # Check immutable head and description immediately before feedback. A + # concurrent author edit will be retried on the next poll. + current = self.gh.get(f"{self.prefix}/pulls/{number}") + if current["base"]["ref"] != self.gh.base_branch: + self._transition(number, "ignored", "PR targets a different consumer base branch") + return + if no_promotion and current["state"] != "open": + self._transition(number, "ignored", "PR is no longer open") + return + if current["head"]["sha"] != pr["head"]["sha"] or current.get("body") != pr.get("body"): + self._transition( + number, "deferred", "source head or description changed; retry next poll" + ) + return + comments = list(_pages(self.gh, f"{self.prefix}/issues/{number}/comments")) + existing = [ + item + for item in comments + if item["user"]["login"] == self.actor and item.get("body", "").startswith(self.marker) + ] + if len(existing) > 1: + raise ValueError("Multiple bot feedback comments; inspect before continuing.") + reviews = ( + [ + item + for item in _pages(self.gh, f"{self.prefix}/pulls/{number}/reviews") + if item["user"]["login"] == self.actor + ] + if current["state"] == "open" and current["user"]["login"] != self.actor + else [] + ) + automated = [ + item + for item in reviews + if item.get("body", "").startswith(self.marker) and item["state"] == "CHANGES_REQUESTED" + ] + manual = [ + item + for item in reviews + if not item.get("body", "").startswith(self.marker) + and item["state"] in ("APPROVED", "CHANGES_REQUESTED") + ] + if no_promotion and not existing and not automated: + self._transition(number, "ignored", "not a promotable update for this canonical vendor") + return + if existing: + if existing[0]["body"] != body and not ( + not problem and existing[0]["body"] == body + dismissal_notice + ): + self.gh.api( + f"{self.prefix}/issues/comments/{existing[0]['id']}", + method="PATCH", + payload={"body": body}, + ) + _LOG.info("PR #%s: updated metadata feedback comment", number) + else: + existing = [ + self.gh.api( + f"{self.prefix}/issues/{number}/comments", method="POST", payload={"body": body} + ) + ] + _LOG.info("PR #%s: posted metadata feedback comment", number) + if current["state"] != "open" or current["user"]["login"] == self.actor: + self._transition( + number, + state, + str(problem) if problem else "feedback reconciled; not a code-review approval", + ) + return + if problem and problem.owner == "author": + review_body = ( + f"{self.marker}\n{problem.problem}\n\nSee the vendor-bot comment for " + "the specific fix and the latest validation result." + ) + for review in automated: + if review["body"] != review_body: + self.gh.api( + f"{self.prefix}/pulls/{number}/reviews/{review['id']}", + method="PUT", + payload={"body": review_body}, + ) + _LOG.info("PR #%s: updated metadata review %s", number, review["id"]) + if not automated and not manual: + self.gh.api( + f"{self.prefix}/pulls/{number}/reviews", + method="POST", + payload={ + "event": "REQUEST_CHANGES", + "commit_id": pr["head"]["sha"], + "body": review_body, + }, + ) + _LOG.info("PR #%s: requested metadata changes", number) + elif not problem: + for review in automated: + # Never replace a metadata objection with APPROVE or dismiss a + # human review. GitHub may require an authorized human dismissal. + try: + self.gh.api( + f"{self.prefix}/pulls/{number}/reviews/{review['id']}/dismissals", + method="PUT", + payload={ + "message": "Automated vendor metadata issue resolved; code review remains required.", + "event": "DISMISS", + }, + ) + _LOG.info("PR #%s: dismissed resolved metadata review %s", number, review["id"]) + except RuntimeError as error: + if "HTTP 403" not in str(error): + raise + _LOG.warning( + "PR #%s: metadata review is resolved, but an authorized human " + "must dismiss automated review %s.", + number, + review["id"], + ) + self._transition( + number, "blocked", "Authorized human must dismiss metadata review" + ) + if existing and existing[0]["body"] != body + dismissal_notice: + self.gh.api( + f"{self.prefix}/issues/comments/{existing[0]['id']}", + method="PATCH", + payload={"body": body + dismissal_notice}, + ) + return + if existing and existing[0]["body"] == body + dismissal_notice: + self.gh.api( + f"{self.prefix}/issues/comments/{existing[0]['id']}", + method="PATCH", + payload={"body": body}, + ) + self._transition( + number, + state, + str(problem) if problem else "feedback reconciled; not a code-review approval", + ) + + def _inputs(self, pr: dict) -> tuple[manage.Vendor, manage.Vendor] | None: + for change in _pages(self.gh, f"{self.prefix}/pulls/{pr['number']}/files"): + if change["filename"] == promote._LOCK and change.get("status") in ("added", "removed"): + return None # Creating/removing the entire lock is not a source update. + if pr.get("merged"): + revision = promote._sha(pr["merge_commit_sha"]) + commit = self.gh.get(f"{self.prefix}/commits/{revision}") + base = commit["parents"][0]["sha"] + else: + base, revision = promote._main_sha(self.gh), pr["head"]["sha"] + try: + before = promote._lock_document_at(self.gh, self.gh.consumer_repo, base)["vendors"] + except Diagnostic as error: + raise RuntimeError(f"Canonical/base vendor lock is invalid: {error}") from error + after = promote._lock_document_at(self.gh, self.gh.consumer_repo, revision)["vendors"] + name = self.gh.vendor_name + if name not in before or name not in after: + return None # Vendor creation/removal is not promotion. + previous = manage._validate_vendor(name, before[name]) + try: + reviewed = manage._validate_vendor(name, after[name]) + promote._repo_name(reviewed.url) + except (manage.VendorError, ValueError) as error: + raise Diagnostic( + f"Invalid lock entry at `vendors.{name}`: {field(_summary(error))}.", + f"Correct that entry in `{promote._LOCK}` with `scripts/vendor/manage.py`, " + "regenerate the vendored snapshot if its source changed, and push the PR. " + "Editing the PR description will not fix a lock-file error.", + ) from error + canonical = ( + promote._repo_name(previous.url).lower() == self.args.canonical_repo.lower() + and previous.branch == self.args.canonical_branch + ) + target_is_canonical = promote._repo_name( + reviewed.url + ).lower() == self.args.canonical_repo.lower() and ( + reviewed.branch == self.args.canonical_branch + or reviewed.branch in self.refresh_branches + ) + if target_is_canonical: + return None # Periodic refresh/direct canonical update: separate workflow. + if not reviewed.branch or (pr.get("merged") and previous.commit == reviewed.commit): + return None # Includes lock-only promotions and tag-only updates. + if not pr.get("merged"): + comparison = self.gh.compare(self.gh.consumer_repo, base, revision) + merge_base = promote._sha(comparison["merge_base_commit"]["sha"]) + try: + original = ( + before + if merge_base == base + else promote._lock_document_at(self.gh, self.gh.consumer_repo, merge_base)[ + "vendors" + ] + ) + except Diagnostic as error: + raise RuntimeError(f"PR merge-base vendor lock is invalid: {error}") from error + # Classify the PR's own changes, but retain the current canonical + # pin as the baseline for source-history checks and promotion. + if name not in original: + return None + original_vendor = manage._validate_vendor(name, original[name]) + reference_changed = ( + promote._repo_name(original_vendor.url).lower(), + original_vendor.branch, + ) != (promote._repo_name(reviewed.url).lower(), reviewed.branch) + if canonical and previous.commit == reviewed.commit and reference_changed: + raise Diagnostic( + f"Temporary source reference adds no new vendor commit: " + f"{field(reviewed.commit)} is already pinned on {field(self.gh.base_branch)}.", + f"In `{promote._LOCK}` at `vendors.{name}`, restore `url` to " + f"{field(previous.url)} and `branch` to {field(previous.branch)}. " + "Change only these source-reference fields; keep the commit and other lock " + "fields, vendored files, and patch unchanged. No upstream-PR metadata or " + "vendor promotion is needed for this same-commit reference change.", + ) + if original_vendor.commit == reviewed.commit: + return None + if previous.commit == reviewed.commit: + return None + if not canonical: + if ( + promote._repo_name(previous.url).lower() == self.args.canonical_repo.lower() + and previous.branch in self.refresh_branches + ): + raise Diagnostic( + f"The base lock now uses refresh branch {field(previous.branch)}, but this " + f"monitor still publishes to {field(self.args.canonical_branch)}.", + "A maintainer must stop this monitor and migrate its workdir with " + f"`--canonical-branch {previous.branch} " + f"--migrate-canonical-branch-from {self.args.canonical_branch}`. " + "Verify a dry run before resuming publication. Do not promote back to " + "the old branch or change contributor metadata to bypass this cutover.", + owner="maintainer", + ) + if pr.get("merged"): + return None # Historical update belonging to another canonical branch. + raise Diagnostic( + "Waiting for the preceding vendor promotion: the base lock still points " + f"to repository {field(previous.url)}, branch {field(previous.branch)}.", + f"A maintainer must finish the preceding lock-only promotion back to " + f"{field(self.args.canonical_repo)} / {field(self.args.canonical_branch)}. " + "After it merges, the author should rebase this consumer PR onto the latest " + "base branch and rebase its temporary source branch onto the new canonical " + "pin, then update the lock/vendor snapshot. No metadata edit can unblock " + "the preceding promotion.", + owner="maintainer", + ) + return previous, reviewed + + def _arguments(self, number: int, reviewed: manage.Vendor) -> argparse.Namespace: + return argparse.Namespace( + source_pr=str(number), + canonical_repo=self.args.canonical_repo, + canonical_branch=self.args.canonical_branch, + upstream_pr=[], + unpaired_reason=None, + map_upstream=[], + repo=self.args.repo, + fork=self.args.fork, + source_repo=None, + worktree=self.args.workdir / f"promotion-{number}-{reviewed.commit[:12]}", + publish=self.args.publish, + auto_merge=True, + wait=False, + timeout=3600, + ) + + def _publish(self, arguments: argparse.Namespace, plan: promote.Promotion) -> None: + number = plan.number + if plan.completed_pr is not None: + self.store.put(f"complete:{number}", plan.completed_pr["html_url"]) + self._transition(number, "completed", plan.completed_pr["html_url"]) + return + if not self.args.publish: + self._transition(number, "promotion_ready", f"dry run: {plan.branch}") + return + self.progress.operation("publishing_promotion", number) + self._transition(number, "promoting", "create or resume verified lock-only promotion") + followup = promote._publish(arguments, self.gh, plan) + self.store.put(f"promotion:{number}", followup["number"]) + latest = self.gh.get(f"{self.prefix}/pulls/{followup['number']}") + if latest.get("merged"): + promote._verify_remote_commit( + self.gh, plan, self.gh.consumer_repo, latest["merge_commit_sha"] + ) + self.store.put(f"complete:{number}", latest["html_url"]) + self._transition( + number, "completed" if latest.get("merged") else "promotion_pending", latest["html_url"] + ) + + def inspect(self, number: int) -> None: + """Reconcile one source-update PR and its deterministic promotion operation.""" + completed = self.store.get(f"complete:{number}") + if completed: + self._transition(number, "completed", completed) + return + self.progress.operation("inspecting", number) + pr = self.gh.get(f"{self.prefix}/pulls/{number}") + if pr["base"]["ref"] != self.gh.base_branch: + self._transition(number, "ignored", "PR targets a different consumer base branch") + return + if pr["state"] == "closed" and not pr.get("merged"): + self._transition(number, "ignored", "closed without merging") + return + try: + inputs = self._inputs(pr) + if inputs is None: + if pr["state"] == "open": + self._feedback(pr, None, no_promotion=True) + else: + self._transition( + number, "ignored", "not a promotable update for this canonical vendor" + ) + return + previous, reviewed = inputs + self.progress.count("relevant") + except Diagnostic as error: + self._feedback(pr, error) + return + arguments = self._arguments(number, reviewed) + if pr.get("merged"): + self.progress.operation("recovering_promotion", number) + plan = promote._plan_identity(arguments, self.gh) + if promote._recover_plan(arguments, self.gh, plan): + self._transition(number, "recovered", "verified durable promotion provenance") + self._publish(arguments, plan) + return + current = promote._lock_at(self.gh, self.gh.consumer_repo, plan.main_sha) + if current.to_mapping() != reviewed.to_mapping(): + self._transition(number, "ignored", "historical pin is no longer pending promotion") + return + promote._check_current(self.gh, plan) + try: + self.progress.operation("validating_metadata", number) + entry = metadata.parse(pr.get("body") or "", self.gh.vendor_name, self.gh.upstream_repo) + cache = self.args.workdir / "sources.git" + # Until provenance is committed, decisions follow current remote + # inputs, not whichever upstream heads a previous process cached. + _, evidence = metadata.resolve(self.gh, previous, reviewed, entry, cache) + except Diagnostic as error: + self._feedback(pr, error) + return + self.progress.operation("reconciling_feedback", number) + self._feedback(pr, None, attribution=evidence) + if not pr.get("merged"): + return + # Re-read mutable description/head before any promotion. Immutable lock + # checks inside promote also run before each remote write. + current = self.gh.get(f"{self.prefix}/pulls/{number}") + if current.get("body") != pr.get("body") or current["head"]["sha"] != pr["head"]["sha"]: + self._transition( + number, "deferred", "source head or description changed; retry next poll" + ) + return + arguments.upstream_pr = [str(item) for item in entry["upstream_prs"]] + arguments.unpaired_reason = entry["unpaired_reason"] + arguments.source_repo = cache if cache.exists() else None + self.progress.operation("planning_promotion", number) + plan = promote._make_plan( + arguments, + self.gh, + auto_match=bool(entry["upstream_prs"]), + resolutions=entry["resolutions"], + ) + # Attribution evidence is versioned with the durable promotion record. + # Reuse the original published record on retries rather than replacing it. + if plan.completed_pr is None: + plan.record["contributor_metadata"] = { + "source_head": pr["head"]["sha"], + "metadata_digest": metadata.fingerprint(entry), + "pr_author": pr["user"]["login"], + } + self._publish(arguments, plan) + + def poll(self) -> None: + """Process candidates serially; preserve failures for safe later retries.""" + started = time.monotonic() + self.progress.update( + poll_started_at=_now(), + next_poll_at=None, + counts=dict(scanned=0, tracked=0, relevant=0, processed=0, blocked=0, errors=0), + ) + _LOG.info("Poll started") + completed = False + try: + candidates = self._discover() + self.progress.count("tracked", len(candidates)) + for number in candidates: + if _STOP.is_set() or (self.args.workdir / "stop").exists(): + _STOP.set() + break + try: + self.inspect(number) + if self.store.get(f"problem:{number}"): + self.progress.count("blocked") + except ( + RuntimeError, + ValueError, + OSError, + KeyError, + subprocess.TimeoutExpired, + ) as error: + self.progress.count("errors") + self.progress.error(error, number) + self._transition(number, "blocked", _summary(error)) + _LOG.error("PR #%s paused: %s", number, _summary(error)) + self.progress.count("processed") + completed = not _STOP.is_set() + except (RuntimeError, OSError, ValueError, KeyError, subprocess.TimeoutExpired) as error: + self.progress.count("errors") + self.progress.error(error) + raise + finally: + counts = self.progress.state["counts"].copy() + outcome = ( + "failed" + if not completed and not _STOP.is_set() + else "interrupted" + if _STOP.is_set() + else "needs_attention" + if counts["errors"] or counts["blocked"] + else "ok" + ) + summary = { + "outcome": outcome, + "finished_at": _now(), + "duration_seconds": round(time.monotonic() - started, 3), + **counts, + } + fields = {"last_poll_summary": summary, "progress_at": _now()} + if completed: + fields["last_completed_poll"] = summary["finished_at"] + self.store.put("last_poll", summary["finished_at"]) + if outcome == "ok": + fields["last_successful_poll"] = summary["finished_at"] + self.progress.update(**fields) + _LOG.info("Poll finished: %s", summary) + + +def _initialize_workdir(path: Path) -> None: + path.mkdir(parents=True, exist_ok=True, mode=0o700) + if path.stat().st_uid != os.getuid(): + raise ValueError("Workdir must be owned by the current user.") + result = subprocess.run( + ["git", "-C", str(path), "rev-parse", "--is-inside-work-tree"], + capture_output=True, + text=True, + ) + if result.returncode == 0: + raise ValueError( + "Bot workdir must be outside a Git worktree to keep runtime state private." + ) + files = { + _STATE_FILE, + "state.sqlite-wal", + "state.sqlite-shm", + "state.sqlite-journal", + "bot.log", + "bot.lock", + "stop", + } + for entry in path.iterdir(): + directory = entry.name == "sources.git" or re.fullmatch( + r"promotion-[0-9]+-[0-9a-f]{12}", entry.name + ) + if ( + entry.is_symlink() + or entry.stat().st_uid != os.getuid() + or not (entry.is_dir() if directory else entry.name in files and entry.is_file()) + ): + raise ValueError( + "Choose an empty workdir or an existing vendor-bot workdir; unexpected entry: " + + entry.name + ) + path.chmod(0o700) + + +@contextlib.contextmanager +def _locked(workdir: Path) -> Iterator[None]: + with (workdir / "bot.lock").open("a+") as handle: + try: + fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as error: + raise ValueError("A bot already owns this workdir.") from error + yield + + +def _running(workdir: Path) -> bool: + if not (workdir / "bot.lock").exists(): + return False + try: + with _locked(workdir): + return False + except ValueError: + return True + + +def _serve(args: argparse.Namespace) -> None: + with _locked(args.workdir): + (args.workdir / "stop").unlink(missing_ok=True) + with contextlib.closing(Store(args.workdir / _STATE_FILE)) as store: + store.put("process", {"pid": None, "publish": args.publish}) + progress = Progress(store) + progress.operation("starting") + with progress.heartbeats(): + try: + gh = promote.GitHub( + args.consumer_repo, + args.vendor, + args.upstream_repo, + args.base_branch, + args.upstream_branch, + ) + monitor = Monitor(args, gh, store, progress) + store.put("process", {"pid": os.getpid(), "publish": args.publish}) + _LOG.info( + "Started vendor %s as %s; remote writes %s; interval=%ss", + args.vendor, + monitor.actor, + args.publish, + args.interval, + ) + failures = 0 + while not _STOP.is_set(): + try: + monitor.poll() + failures = 0 + except ( + RuntimeError, + OSError, + ValueError, + KeyError, + subprocess.TimeoutExpired, + ) as error: + failures += 1 + _LOG.error("Poll paused: %s", _summary(error)) + if args.once: + raise + if args.once or _STOP.is_set(): + break + delay = min(args.interval * 2 ** min(failures, 5), 3600) + deadline = time.monotonic() + delay + progress.operation("waiting") + progress.update( + next_poll_at=( + datetime.datetime.now(datetime.timezone.utc) + + datetime.timedelta(seconds=delay) + ).isoformat() + ) + _LOG.info( + "Next poll at %s (in %ss; consecutive_poll_failures=%s)", + progress.state["next_poll_at"], + delay, + failures, + ) + while time.monotonic() < deadline and not _STOP.wait(1): + if (args.workdir / "stop").exists(): + _STOP.set() + break + except ( + RuntimeError, + OSError, + ValueError, + KeyError, + subprocess.TimeoutExpired, + ) as error: + progress.error(error) + raise + finally: + progress.operation("stopped") + progress.update(next_poll_at=None) + store.put("process", {"pid": None, "publish": args.publish}) + _LOG.info("Stopped vendor %s", args.vendor) + + +def _status(workdir: Path) -> dict: + """Read cached telemetry without creating or repairing the state database.""" + active = _running(workdir) + result = { + "running": active, + "stop_requested": active and (workdir / "stop").exists(), + "pid": None, + "publish": None, + "operation": None, + } + database = workdir / _STATE_FILE + if not database.exists(): + result["state_available"] = False + return result + try: + with contextlib.closing( + sqlite3.connect(f"{database.as_uri()}?mode=ro", uri=True, timeout=1) + ) as connection: + rows = connection.execute( + "SELECT key, value FROM state WHERE key IN " + "('health', 'process', 'settings', 'migration', 'refresh_branches')" + ).fetchall() + state = {key: json.loads(value) for key, value in rows} + refresh_branches = state.pop("refresh_branches", []) + if not isinstance(refresh_branches, list) or any( + not isinstance(branch, str) for branch in refresh_branches + ): + raise ValueError("Malformed refresh policy; expected a JSON list of branch names.") + if any(not isinstance(value, dict) for value in state.values()): + raise ValueError("Malformed status cache; expected JSON objects.") + health, process = state.get("health", {}), state.get("process", {}) + for key in ( + "operation", + "progress_at", + "poll_started_at", + "last_completed_poll", + "last_successful_poll", + "last_poll_summary", + "last_error", + "next_poll_at", + "counts", + ): + result[key] = health.get(key) + result.update( + state_available=True, + pid=process.get("pid") if active else None, + publish=process.get("publish"), + vendor=state.get("settings", {}).get("vendor"), + canonical_repo=state.get("settings", {}).get("canonical_repo"), + canonical_branch=state.get("settings", {}).get("canonical_branch"), + base_branch=state.get("settings", {}).get("base_branch"), + refresh_branches=refresh_branches, + migration=state.get("migration"), + ) + if not active: + result["operation"] = None + result["next_poll_at"] = None + except (sqlite3.Error, ValueError, OSError) as error: + result.update(state_available=False, state_error=_summary(error)) + return result + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + run = commands.add_parser("run", help="Monitor in foreground, or detach with --daemon.") + run.add_argument("--workdir", type=Path, required=True) + run.add_argument("--vendor", required=True) + run.add_argument("--upstream-repo", required=True) + run.add_argument("--canonical-repo", required=True) + run.add_argument("--canonical-branch", required=True) + run.add_argument( + "--refresh-branch", + action="append", + default=[], + help=( + "Recognize this branch in --canonical-repo as a refresh target, not a temporary " + "source branch. Repeat for multiple targets; does not enable publication to them." + ), + ) + run.add_argument( + "--migrate-canonical-branch-from", + help="Explicitly migrate this stopped workdir from OLD_BRANCH after a merged refresh.", + ) + run.add_argument("--fork", required=True, help="Consumer publishing fork OWNER/REPO.") + run.add_argument("--consumer-repo", default="NVIDIA/TensorRT-LLM") + run.add_argument("--base-branch", default="main") + run.add_argument("--upstream-branch", default="main") + run.add_argument( + "--repo", + type=Path, + default=Path(__file__).resolve().parents[2], + help="Trusted consumer checkout used for isolated promotion worktrees.", + ) + run.add_argument( + "--interval", type=int, default=120, help="Polling interval, seconds (minimum 30)." + ) + run.add_argument( + "--log-level", + type=str.upper, + choices=("DEBUG", "INFO", "WARNING", "ERROR"), + default="INFO", + help="Daemon/foreground log threshold (default: INFO).", + ) + run.add_argument( + "--since", + help=( + "Optional historical discovery cutoff, UTC YYYY-MM-DDTHH:MM:SSZ; " + "default all history. Reuse the same cutoff after cache loss." + ), + ) + run.add_argument( + "--publish", + action="store_true", + help="Allow feedback/reviews, pushes, PR creation, CI skip, and squash auto-merge.", + ) + mode = run.add_mutually_exclusive_group() + mode.add_argument("--daemon", action="store_true") + mode.add_argument( + "--once", + action="store_true", + help="One reconciliation pass, useful for testing or a scheduler.", + ) + for name in ("status", "stop"): + command = commands.add_parser(name) + command.add_argument("--workdir", required=True, type=Path) + return parser + + +def main(argv: list[str] | None = None) -> int: + """Launch/control a POSIX daemon without storing credentials or requiring systemd.""" + args = _parser().parse_args(argv) + args.workdir = args.workdir.expanduser().resolve() + logging.basicConfig( + level=getattr(args, "log_level", "INFO"), format="%(asctime)sZ %(levelname)s %(message)s" + ) + logging.Formatter.converter = time.gmtime + try: + if args.command != "run": + status = _status(args.workdir) + if args.command == "stop" and status["running"]: + (args.workdir / "stop").touch(mode=0o600) + status["stop_requested"] = True + print(json.dumps(status)) + return 0 + if args.interval < 30: + raise ValueError("--interval must be at least 30 seconds.") + if not manage._NAME_PATTERN.fullmatch(args.vendor): + raise ValueError("Invalid vendor key.") + for name in (args.consumer_repo, args.upstream_repo, args.canonical_repo, args.fork): + promote._repo_name(f"https://github.com/{name}") + for branch in (args.canonical_branch, args.base_branch, args.upstream_branch): + promote._run(["git", "check-ref-format", f"refs/heads/{branch}"]) + for branch in args.refresh_branch: + promote._run(["git", "check-ref-format", f"refs/heads/{branch}"]) + if args.migrate_canonical_branch_from: + promote._run( + ["git", "check-ref-format", f"refs/heads/{args.migrate_canonical_branch_from}"] + ) + if args.since: + datetime.datetime.strptime(args.since, "%Y-%m-%dT%H:%M:%SZ") + args.repo = args.repo.expanduser().resolve() + _initialize_workdir(args.workdir) + if args.daemon: + # Initialize the state marker before launching so two competing + # starts can both reach the authoritative child-held flock. + with contextlib.closing(Store(args.workdir / _STATE_FILE)): + pass + if _running(args.workdir): + raise ValueError("A bot already owns this workdir.") + command = [ + sys.executable, + str(Path(__file__).resolve()), + *list(argv if argv is not None else sys.argv[1:]), + ] + command.remove("--daemon") + with (args.workdir / "bot.log").open("ab", buffering=0) as log: + child = subprocess.Popen( + command, + stdin=subprocess.DEVNULL, + stdout=log, + stderr=log, + start_new_session=True, + close_fds=True, + umask=0o077, + ) + for _ in range(100): + if child.poll() is not None: + raise ValueError( + f"Daemon exited during startup; inspect {args.workdir / 'bot.log'}." + ) + with contextlib.closing(Store(args.workdir / _STATE_FILE)) as state: + process = state.get("process", {}) + if _running(args.workdir) and process.get("pid") == child.pid: + print(f"Started daemon PID {child.pid}; log: {args.workdir / 'bot.log'}") + return 0 + time.sleep(0.1) + raise ValueError( + "Daemon startup not confirmed; inspect log and status before retrying." + ) + signal.signal(signal.SIGTERM, lambda *_: _STOP.set()) + signal.signal(signal.SIGINT, lambda *_: _STOP.set()) + _STOP.clear() + _serve(args) + return 0 + except (ValueError, RuntimeError, OSError, sqlite3.Error, subprocess.TimeoutExpired) as error: + _LOG.error("Bot stopped: %s", _summary(error)) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/vendor/diagnostics.py b/scripts/vendor/diagnostics.py new file mode 100644 index 000000000000..19dd7b1698bc --- /dev/null +++ b/scripts/vendor/diagnostics.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Actionable diagnostics safe to publish separately from operator failures.""" + +from __future__ import annotations + +import html +from typing import Literal + +import yaml + + +class Diagnostic(ValueError): + """A recognized problem with a corrective action and an optional Markdown example.""" + + def __init__( + self, + problem: str, + action: str, + *, + example: str = "", + owner: Literal["author", "maintainer"] = "author", + ) -> None: + self.problem, self.action, self.example, self.owner = problem, action, example, owner + super().__init__(problem) + + def __str__(self) -> str: + return f"{self.problem}\n\n**Fix:** {self.action}" + ( + f"\n\n{self.example}" if self.example else "" + ) + + +def field(value: object) -> str: + """Quote a bounded field identifier without interpreting author-supplied Markdown.""" + text = str(value).replace("\r", "\\r").replace("\n", "\\n")[:160] + return "" + html.escape(text) + "" + + +def yaml_problem(error: yaml.YAMLError) -> str: + """Describe YAML failures and locations without echoing source excerpts or values.""" + detail = "invalid YAML syntax or unsupported YAML value" + if isinstance(error, yaml.MarkedYAMLError): + problem = error.problem or "" + if problem.startswith("found duplicate key"): + detail = "duplicate mapping key; keep one value for each key" + elif "merge keys are not allowed" in problem: + detail = "YAML merge keys are not allowed; write fields explicitly" + elif problem == "found an unhashable mapping key": + detail = "mapping keys must be scalar field names, not lists or mappings" + elif "could not find expected ':'" in problem: + detail = "missing ':' after a mapping key" + elif problem.startswith("expected ',' or ']'"): + detail = "expected a comma between list items or a closing ']'" + elif problem.startswith("expected ',' or '}'"): + detail = "expected a comma between mapping entries or a closing '}'" + elif problem.startswith("expected "): + detail = "unexpected field or indentation; align sibling fields with spaces" + mark = error.problem_mark + if mark is not None: + detail += f" at YAML line {mark.line + 1}, column {mark.column + 1}" + return detail diff --git a/scripts/vendor_sources.py b/scripts/vendor/manage.py similarity index 99% rename from scripts/vendor_sources.py rename to scripts/vendor/manage.py index 392d6b8565ac..722b2b1a7552 100644 --- a/scripts/vendor_sources.py +++ b/scripts/vendor/manage.py @@ -36,7 +36,7 @@ import yaml -_REPO_ROOT = Path(__file__).resolve().parent.parent +_REPO_ROOT = Path(__file__).resolve().parents[2] _DEFAULT_LOCK = _REPO_ROOT / "3rdparty" / "vendor_sources.lock.yaml" _SCHEMA_VERSION = 1 _COMMIT_PATTERN = re.compile(r"[0-9a-f]{40}\Z") diff --git a/scripts/vendor/metadata.py b/scripts/vendor/metadata.py new file mode 100644 index 000000000000..544cc3dc3ee6 --- /dev/null +++ b/scripts/vendor/metadata.py @@ -0,0 +1,762 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Strict contributor metadata and conservative upstream commit attribution. + +Matching establishes provenance, not permission to drop changes during refresh. +Source code and commit messages are data and are never executed. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from difflib import SequenceMatcher +from pathlib import Path, PurePosixPath + +import yaml + +if __package__: + from . import manage, promote + from .diagnostics import Diagnostic, field, yaml_problem +else: + import manage + import promote + from diagnostics import Diagnostic, field, yaml_problem + +BEGIN = "" +END = "" +SIMILARITY_THRESHOLD = 0.9 +_SIMILARITY_METRIC = "sequence-matcher-lines-v1" + + +def template(vendor_name: str, upstream_repo: str) -> str: + """Return a copyable PR-description block; repository settings stay operator-owned.""" + return ( + f"{BEGIN}\n```yaml\nschema_version: 1\nvendors:\n {vendor_name}:\n" + f" upstream_prs:\n - https://github.com/{upstream_repo}/pull/123\n" + f"```\n{END}" + ) + + +def _vendor_example(vendor_name: str, content: str) -> str: + return f"```yaml\nvendors:\n {vendor_name}:\n{content}\n```" + + +def _resolution_example(vendor_name: str, upstream_repo: str) -> str: + return _vendor_example( + vendor_name, + f" upstream_prs:\n - https://github.com/{upstream_repo}/pull/123\n" + " resolutions:\n FULL_40_CHARACTER_SOURCE_SHA:\n" + f" upstream_pr: https://github.com/{upstream_repo}/pull/123\n" + " reason: Explain the downstream adaptation or why this PR owns the change.", + ) + + +def _pr_reference( + reference: object, upstream_repo: str, path: str, *, require_url: bool = True +) -> int: + if isinstance(reference, str) and ( + not require_url or reference.startswith(f"https://github.com/{upstream_repo}/pull/") + ): + try: + return promote._pr_number(reference, upstream_repo) + except ValueError: + pass + raise Diagnostic( + f"Invalid upstream PR URL at {field(path)}.", + f"In the PR description, use a full URL such as " + f"`https://github.com/{upstream_repo}/pull/123` with the actual PR number " + "and no query string or fragment.", + ) + + +def parse(body: str, vendor_name: str, upstream_repo: str) -> dict: + """Read exactly one marked block and validate the selected vendor's inputs.""" + if len(body) > 65536: + raise Diagnostic( + "PR description exceeds the supported metadata size (65,536 characters).", + "Shorten the PR description to at most 65,536 characters, keeping the complete " + "vendor-promotion block. Move long logs to an attachment or a comment.", + ) + body = body.replace("\r\n", "\n").replace("\r", "\n") + counts = {BEGIN: body.count(BEGIN), END: body.count(END)} + missing = [marker for marker, count in counts.items() if count == 0] + duplicates = [marker for marker, count in counts.items() if count > 1] + full_example = f"````markdown\n{template(vendor_name, upstream_repo)}\n````" + if missing or duplicates: + problems, actions = [], [] + if missing: + problems.append( + "Missing vendor-promotion markers in the PR description: " + + ", ".join(f"`{marker}`" for marker in missing) + + "." + ) + if len(missing) == 2: + actions.append( + "Edit this PR's description—not a comment—and insert the complete block " + "below. Replace the example upstream PR URL with your actual PR URL. " + "If there is genuinely no paired upstream PR, replace `upstream_prs` " + "with a nonempty `unpaired_reason` explaining why." + ) + else: + position = "before" if missing[0] == BEGIN else "after" + actions.append( + f"Edit this PR's description—not a comment—and insert `{missing[0]}` " + f"on its own line {position} the existing fenced YAML block. " + "Keep the existing metadata and the other marker." + ) + if duplicates: + problems.append( + "Duplicate vendor-promotion markers: " + + ", ".join(f"`{marker}` appears {counts[marker]} times" for marker in duplicates) + + "." + ) + actions.append( + "Consolidate all vendor entries into one `vendors` mapping in one marked " + "block in the PR description. Remove extra markers/blocks, preserving the " + "other vendors' metadata; keep one start marker and one end marker." + ) + raise Diagnostic( + " ".join(problems), + " ".join(actions), + example=full_example if len(missing) == 2 else "", + ) + start, end = body.index(BEGIN) + len(BEGIN), body.index(END) + if start >= end: + raise Diagnostic( + "The vendor-promotion end marker appears before the start marker.", + f"Move `{BEGIN}` before the fenced YAML block and `{END}` after it " + "in the PR description. Preserve the metadata between them.", + ) + block = body[start:end].strip() + if not block.startswith("```yaml\n") or not block.endswith("\n```"): + raise Diagnostic( + "The marked block must contain one fenced yaml mapping.", + "In the PR description, put the metadata between an opening line of three " + "backticks followed by `yaml` and a closing line of three backticks. " + "Keep both markers outside that fence, as shown below; retain your real PR URLs.", + example=full_example, + ) + try: + content = block[len("```yaml\n") : -len("\n```")] + if any( + isinstance(token, (yaml.tokens.AliasToken, yaml.tokens.AnchorToken)) + for token in yaml.scan(content) + ): + raise Diagnostic( + "Promotion YAML aliases and anchors are not allowed.", + "In the PR description's metadata block, replace YAML anchors (`&name`) " + "and aliases (`*name`) with explicit values.", + ) + data = yaml.load(content, Loader=manage._UniqueKeyLoader) + except yaml.YAMLError as error: + raise Diagnostic( + f"Invalid promotion YAML: {yaml_problem(error)}.", + "Correct the metadata block in the PR description at the location above " + "(line 1 is the first line inside the YAML fence). Use space indentation, " + "balanced brackets/quotes, and unique mapping keys.", + ) from error + if not isinstance(data, dict): + raise Diagnostic( + "Promotion YAML must be a mapping, not a list or scalar.", + "Use top-level `schema_version: 1` and `vendors:` fields in the PR description, " + "following this structure. Replace the example URL.", + example=full_example, + ) + missing_keys = {"schema_version", "vendors"} - set(data) + extra_keys = set(data) - {"schema_version", "vendors"} + if missing_keys or extra_keys: + raise Diagnostic( + "Invalid top-level promotion YAML fields. " + + ( + "Missing: " + ", ".join(field(key) for key in sorted(missing_keys)) + ". " + if missing_keys + else "" + ) + + ( + "Unsupported: " + ", ".join(field(key) for key in sorted(extra_keys, key=str)) + "." + if extra_keys + else "" + ), + "In the PR description, keep only top-level `schema_version: 1` and `vendors:`; " + "put vendor-specific fields under `vendors.`.", + ) + if type(data["schema_version"]) is not int or data["schema_version"] != 1: + raise Diagnostic( + "Unsupported promotion schema_version; expected integer 1.", + "Set `schema_version: 1` (without quotes) in the PR description's metadata block.", + ) + entries = data["vendors"] + if not isinstance(entries, dict) or vendor_name not in entries: + raise Diagnostic( + f"Promotion YAML must include vendors.{vendor_name} as a mapping entry.", + f"Add `{vendor_name}:` under `vendors:` in the existing PR-description block. " + "Preserve any other vendor entries and replace the example URL.", + example=_vendor_example( + vendor_name, + f" upstream_prs:\n - https://github.com/{upstream_repo}/pull/123", + ), + ) + invalid_names = [ + name + for name in entries + if not isinstance(name, str) or not manage._NAME_PATTERN.fullmatch(name) + ] + if invalid_names: + raise Diagnostic( + "Invalid vendor keys under `vendors`: " + + ", ".join(field(name) for name in invalid_names) + + ".", + "Use the exact vendor names from `3rdparty/vendor_sources.lock.yaml` as " + "string keys under `vendors` in the PR description.", + ) + entry = entries[vendor_name] + path = f"vendors.{vendor_name}" + if not isinstance(entry, dict) or set(entry) - { + "upstream_prs", + "unpaired_reason", + "resolutions", + }: + extra = ( + set(entry) - {"upstream_prs", "unpaired_reason", "resolutions"} + if isinstance(entry, dict) + else set() + ) + raise Diagnostic( + f"Invalid vendor metadata at {field(path)}. " + + ( + "Unsupported fields: " + + ", ".join(field(key) for key in sorted(extra, key=str)) + + "." + if extra + else "Expected a mapping." + ), + "Use only `upstream_prs`, `unpaired_reason`, and `resolutions` under this vendor " + "in the PR description. Repository/operator settings do not belong here.", + ) + if "unpaired_reason" in entry: + if ( + set(entry) != {"unpaired_reason"} + or not isinstance(entry["unpaired_reason"], str) + or not entry["unpaired_reason"].strip() + ): + raise Diagnostic( + f"Invalid {field(path + '.unpaired_reason')}: it must be nonempty and " + "cannot accompany `upstream_prs` or `resolutions`.", + "If there genuinely is no paired upstream PR, keep only `unpaired_reason` " + "with an explanation under this vendor. Otherwise remove `unpaired_reason` " + "and list the actual upstream PRs. Do not use it to bypass failed matching.", + ) + return { + "upstream_prs": [], + "resolutions": {}, + "unpaired_reason": entry["unpaired_reason"].strip(), + } + references = entry.get("upstream_prs") + if not isinstance(references, list) or not references: + raise Diagnostic( + f"{field(path + '.upstream_prs')} must be a nonempty list of upstream PR URLs.", + "In the PR description, list every related upstream PR using YAML list items " + "(`- URL`); replace the example URL. If there genuinely is no paired PR, use " + "only a nonempty `unpaired_reason` instead.", + example=_vendor_example( + vendor_name, + f" upstream_prs:\n - https://github.com/{upstream_repo}/pull/123", + ), + ) + numbers = [] + for index, reference in enumerate(references): + number = _pr_reference(reference, upstream_repo, f"{path}.upstream_prs[{index}]") + if number in numbers: + raise Diagnostic( + f"{field(path + '.upstream_prs')} contains duplicate upstream PR #{number}.", + f"Remove the repeated `https://github.com/{upstream_repo}/pull/{number}` " + "entry from the PR description; keep each upstream PR once.", + ) + numbers.append(number) + resolutions = entry.get("resolutions", {}) + if not isinstance(resolutions, dict): + raise Diagnostic( + f"{field(path + '.resolutions')} must be a mapping keyed by full source commit SHA.", + "Replace it with the structure below inside the existing vendor entry. Use a " + "full 40-character source SHA, a listed upstream PR, and a specific explanation.", + example=_resolution_example(vendor_name, upstream_repo), + ) + normalized = {} + for sha, resolution in resolutions.items(): + resolution_path = f"{path}.resolutions.{sha}" + try: + promote._sha(sha) + except ValueError as error: + raise Diagnostic( + f"Invalid source commit key at {field(resolution_path)}.", + "Use the full lowercase 40-character SHA from the bot's unresolved-commit " + "list, not a short SHA, branch name, or upstream commit SHA.", + ) from error + if not isinstance(resolution, dict) or set(resolution) != {"upstream_pr", "reason"}: + details = "Expected a mapping." + if isinstance(resolution, dict): + absent = {"upstream_pr", "reason"} - set(resolution) + unknown = set(resolution) - {"upstream_pr", "reason"} + details = ( + "Missing: " + ", ".join(field(key) for key in sorted(absent)) + ". " + if absent + else "" + ) + ( + "Unsupported: " + + ", ".join(field(key) for key in sorted(unknown, key=str)) + + "." + if unknown + else "" + ) + raise Diagnostic( + f"{field(resolution_path)} requires exactly `upstream_pr` and `reason`. {details}", + "Add the missing fields or remove extra fields in this resolution, " + "following the structure below. Preserve other vendor metadata.", + example=_resolution_example(vendor_name, upstream_repo), + ) + reference, reason = resolution["upstream_pr"], resolution["reason"] + number = _pr_reference( + reference, upstream_repo, f"{resolution_path}.upstream_pr", require_url=False + ) + if number not in numbers: + raise Diagnostic( + f"{field(resolution_path)} names upstream PR #{number}, which is not listed.", + f"Add `https://github.com/{upstream_repo}/pull/{number}` to " + f"`{path}.upstream_prs` if it owns this change, or correct the resolution " + "to reference a listed PR.", + ) + if not isinstance(reason, str) or not reason.strip(): + raise Diagnostic( + f"{field(resolution_path + '.reason')} must be a nonempty string.", + "Explain the downstream adaptation or why this upstream PR owns the " + "unmatched change. Author assertions are retained until separately verified.", + ) + normalized[sha] = {"upstream_pr": number, "reason": reason.strip()} + return {"upstream_prs": sorted(numbers), "resolutions": normalized, "unpaired_reason": None} + + +def fingerprint(value: dict) -> str: + """Hash normalized metadata without recording credentials or unrelated PR prose.""" + return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest() + + +def _patch( + repo: Path, base: str, head: str, *, source: str = ".", include: tuple[str, ...] = ("**/*",) +) -> str: + """Compare lock-selected edits, retaining bytes, paths, modes, and hunk boundaries.""" + changed = promote._run( + [ + "git", + "-C", + str(repo), + "diff", + "--no-ext-diff", + "--no-textconv", + "--no-renames", + "--name-only", + "-z", + base, + head, + "--", + ], + strip=False, + binary_output=True, + ) + # Use the vendoring glob semantics rather than Git's different pathspec + # rules. Enumerate both sides without rename detection to include deletions + # and moves across the selection boundary; pass the results as literal paths. + selected = [] + for path in changed.split("\0"): + if not path or not PurePosixPath(path).is_relative_to(source): + continue + relative = PurePosixPath(path).relative_to(source).as_posix() + if relative != "." and manage._matches(relative, include): + selected.append(f":(top,literal){path}") + if not selected: + return "" + diff = promote._run( + [ + "git", + "-C", + str(repo), + "diff", + "--no-ext-diff", + "--no-textconv", + "--no-renames", + "--binary", + "--full-index", + "--no-color", + "--unified=0", + "--inter-hunk-context=0", + base, + head, + "--", + *selected, + ], + strip=False, + binary_output=True, + ) + # Hunk labels are Git-generated context, not changed code. Keep only the + # boundary marker; edit bytes, including changed function definitions, remain. + return "\n".join( + re.sub(r"^@@ -[0-9]+(?:,[0-9]+)? \+[0-9]+(?:,[0-9]+)? @@.*$", "@@", line) + for line in diff.split("\n") + if not line.startswith("index ") + ) + + +def _commits(repo: Path, base: str, head: str) -> list[str]: + return promote._git(repo, "rev-list", "--reverse", f"{base}..{head}").splitlines() + + +def _similarity(source_patch: str, upstream_patch: str) -> float: + """Score normalized patch lines without discarding whitespace or frequent lines.""" + if not source_patch or not upstream_patch: + return 0.0 + if source_patch == upstream_patch: + return 1.0 + return SequenceMatcher( + None, + source_patch.removesuffix("\n").split("\n"), + upstream_patch.removesuffix("\n").split("\n"), + autojunk=False, + ).ratio() + + +def _upstream_reference(number: int, repository: str | None) -> str: + label = f"upstream PR #{number}" + return f"[{label}](https://github.com/{repository}/pull/{number})" if repository else label + + +def similarity_feedback(evidence: dict[str, dict], upstream_repo: str | None = None) -> str: + """Describe accepted non-exact matches for feedback and durable provenance.""" + lines = [ + f"- `{sha}`: {_upstream_reference(item['upstream_pr'], upstream_repo)}, similarity " + f"**{item['similarity']:.2%}** — accepted " + f"(threshold {item['similarity_threshold']:.2%}; {item['method']})." + for sha, item in evidence.items() + if "similarity" in item + ] + if not lines: + return "" + return ( + "No exact match was found for the following commits; similarity matching accepted them:\n\n" + + "\n".join(lines) + + "\n\nScores compare normalized patch lines in lock-selected files, not semantic " + "equivalence. These changes retain `retain-until-verified` refresh policy." + ) + + +def match( + repo: Path, + previous: str, + reviewed: str, + upstream_prs: dict[int, dict], + resolutions: dict[str, dict] | None = None, + *, + source: str = ".", + include: tuple[str, ...] = ("**/*",), + upstream_repo: str | None = None, + vendor_name: str = "VENDOR_NAME", +) -> tuple[dict[str, int | None], dict[str, dict]]: + """Match a linear source range against fetched immutable upstream PR snapshots. + + Only lock-selected files are compared. None assignments record commits with + no selected changes, not upstream equivalence for the rest of those commits. + Ambiguities remain author-actionable; titles and cherry-pick trailers are not proof. + """ + commits = _commits(repo, previous, reviewed) + if not commits: + raise Diagnostic( + "Source update has no new commits to attribute.", + "Check the source SHA in `3rdparty/vendor_sources.lock.yaml`. Pin the intended " + "new source commit, or remove the unnecessary source update from this PR.", + ) + source_parents = {} + for sha in commits: + parents = promote._git(repo, "show", "-s", "--format=%P", sha).split() + if len(parents) != 1: + raise Diagnostic( + f"Source history must be linear; commit `{sha}` has {len(parents)} parents.", + f"Rebase the temporary source branch onto canonical pin `{previous}` without " + "merge commits. Then use `scripts/vendor/manage.py` to update the lock and " + "vendored snapshot to the new source SHA, and push the consumer PR.", + ) + source_parents[sha] = parents[0] + resolutions = resolutions or {} + if set(resolutions) - set(commits): + raise Diagnostic( + "Resolutions include commits outside the reviewed source delta: " + + ", ".join(field(sha) for sha in sorted(set(resolutions) - set(commits))) + + ".", + f"Remove those stale entries from `vendors.{vendor_name}.resolutions` in the PR " + "description. After rebasing, use the new source SHAs reported by the bot.", + ) + candidates: dict[str, set[int]] = {sha: set() for sha in commits} + patches = {sha: _patch(repo, f"{sha}^", sha, source=source, include=include) for sha in commits} + if any(not patches[sha] for sha in resolutions): + raise Diagnostic( + "Remove resolutions for commits with no lock-selected file changes: " + + ", ".join(f"`{sha}`" for sha in resolutions if not patches[sha]) + + ".", + f"Delete only these entries from `vendors.{vendor_name}.resolutions`. " + "Excluded-only changes do not need upstream attribution.", + ) + series = _patch(repo, previous, reviewed, source=source, include=include) + methods: dict[tuple[str, int], str] = {} + comparisons: dict[int, list[tuple[str, str, str, str]]] = {} + for number, pr in upstream_prs.items(): + base, head = promote._sha(pr["base"]["sha"]), promote._sha(pr["head"]["sha"]) + merge_base = promote._git(repo, "merge-base", base, head) + upstream_commits = _commits(repo, merge_base, head) + upstream_patches = set() + comparisons[number] = [] + for sha in upstream_commits: + parents = promote._git(repo, "show", "-s", "--format=%P", sha).split() + if len(parents) == 1: + patch = _patch(repo, parents[0], sha, source=source, include=include) + upstream_patches.add(patch) + if patch: + comparisons[number].append(("similar-edits", parents[0], sha, patch)) + aggregate = _patch(repo, merge_base, head, source=source, include=include) + if aggregate: + comparisons[number].append(("similar-upstream-aggregate", merge_base, head, aggregate)) + for sha in commits: + if not patches[sha]: + continue + if sha in upstream_commits: + candidates[sha].add(number) + methods[sha, number] = "same-commit" + elif patches[sha] and patches[sha] in upstream_patches: + candidates[sha].add(number) + methods[sha, number] = "same-edits" + elif patches[sha] and patches[sha] == aggregate: + candidates[sha].add(number) + methods[sha, number] = "upstream-aggregate" + # A whole reviewed series may be squashed into one upstream PR. Do not + # invent individual correspondences: identify the shared group explicitly. + if aggregate and series == aggregate: + for sha in commits: + if not patches[sha]: + continue + candidates[sha].add(number) + methods.setdefault((sha, number), "source-series-aggregate") + # Exact matches take priority across all listed PRs. Approximate attribution + # is considered only when none exists, and is not permission to drop changes. + similarities: dict[str, dict[int, dict]] = {} + for sha in commits: + if not patches[sha] or sha in resolutions or candidates[sha]: + continue + similarities[sha] = {} + for number, choices in comparisons.items(): + best = {"similarity": 0.0} + for method, base, head, patch in choices: + score = _similarity(patches[sha], patch) + if score > best["similarity"]: + best = { + "method": method, + "similarity": score, + "comparison": { + "source_base": source_parents[sha], + "source_head": sha, + "upstream_base": base, + "upstream_head": head, + }, + } + similarities[sha][number] = best + if best["similarity"] >= SIMILARITY_THRESHOLD: + candidates[sha].add(number) + assignments, evidence, unresolved = {}, {}, [] + for sha in commits: + if not patches[sha]: + assignments[sha] = None + evidence[sha] = {"method": "outside-vendor-scope", "refresh_policy": "retain"} + elif sha in resolutions: + resolution = resolutions[sha] + number = resolution["upstream_pr"] + if number not in upstream_prs: + raise Diagnostic( + f"Resolution for `{sha}` names unlisted upstream PR #{number}.", + f"Add that PR to `vendors.{vendor_name}.upstream_prs` if it owns this " + "change, or correct the resolution to reference a listed PR.", + ) + assignments[sha] = number + evidence[sha] = { + "method": "author-assertion", + "reason": resolution["reason"], + "refresh_policy": "retain-until-verified", + } + elif len(candidates[sha]) == 1: + number = next(iter(candidates[sha])) + assignments[sha] = number + evidence[sha] = { + "upstream_head": upstream_prs[number]["head"]["sha"], + "upstream_base": upstream_prs[number]["base"]["sha"], + } + if sha in similarities: + evidence[sha].update( + similarities[sha][number], + upstream_pr=number, + similarity_threshold=SIMILARITY_THRESHOLD, + similarity_metric=_SIMILARITY_METRIC, + refresh_policy="retain-until-verified", + ) + else: + evidence[sha]["method"] = methods[sha, number] + else: + options = ( + ", ".join( + _upstream_reference(number, upstream_repo) for number in sorted(candidates[sha]) + ) + or "no exact match" + ) + detail = f"- `{sha}`: {options}" + if sha in similarities: + closest = sorted( + similarities[sha].items(), key=lambda item: (-item[1]["similarity"], item[0]) + )[:3] + scores = ( + ", ".join( + f"{_upstream_reference(number, upstream_repo)}: {item['similarity']:.2%}" + for number, item in closest + ) + or "no upstream candidates" + ) + outcome = "multiple PRs meet" if candidates[sha] else "no PR meets" + detail += ( + f"; best similarities: {scores}; {outcome} the " + f"{SIMILARITY_THRESHOLD:.2%} similarity threshold" + ) + unresolved.append(detail) + accepted = similarity_feedback(evidence, upstream_repo) + if unresolved: + raise Diagnostic( + "Upstream attribution is unresolved for these source commits:\n\n" + + "\n".join(unresolved) + + ("\n\n" + accepted if accepted else ""), + f"In the PR description, add missing PRs to `vendors.{vendor_name}.upstream_prs`, " + "remove incorrectly listed PRs, or correct/rebase the source pin and update the " + "vendored snapshot. For an intentional adaptation or ambiguous ownership, merge " + "a targeted `resolutions` entry into the existing vendor metadata as shown below. " + "Replace the SHA and URL and give a specific reason; preserve other entries. " + "No complete commit_map is required. A resolution records an author assertion, " + "not verified equivalence; it is retained during refresh until separately verified.", + example=_resolution_example(vendor_name, upstream_repo or "OWNER/REPO"), + ) + if set(assignments.values()) - {None} != set(upstream_prs): + unused = set(upstream_prs) - set(assignments.values()) + raise Diagnostic( + "Some listed upstream PRs cover no new lock-selected changes: " + + ", ".join(_upstream_reference(number, upstream_repo) for number in sorted(unused)) + + "." + + ("\n\n" + accepted if accepted else ""), + f"Remove these unrelated PRs from `vendors.{vendor_name}.upstream_prs` in the " + "PR description. PRs covering only excluded tests/docs do not need to be listed.", + ) + return assignments, evidence + + +def resolve( + gh: promote.GitHub, + previous: manage.Vendor, + reviewed: manage.Vendor, + entry: dict, + cache: Path, +) -> tuple[dict[str, int | None], dict[str, dict]]: + """Fetch only Git data into a private bare cache and resolve declared PRs.""" + if previous.source != reviewed.source or set(previous.include) != set(reviewed.include): + changed = (["source"] if previous.source != reviewed.source else []) + ( + ["include"] if set(previous.include) != set(reviewed.include) else [] + ) + raise Diagnostic( + "Automatic attribution requires unchanged lock source/include selection. " + "Changed fields: " + ", ".join(f"`{name}`" for name in changed) + ".", + "Separate the selection change into a different PR. In this source-update PR, " + "restore the canonical `source`/`include` selection in " + "`3rdparty/vendor_sources.lock.yaml` and regenerate the vendor snapshot.", + ) + cache.mkdir(parents=True, exist_ok=True) + if not (cache / "HEAD").exists(): + promote._git(cache, "init", "--bare", "--quiet") + if promote._git(cache, "rev-parse", "--is-bare-repository") != "true": + raise ValueError("Matching cache must be a dedicated bare repository.") + source_repo = promote._repo_name(reviewed.url) + try: + promote._check_source_repo(gh, source_repo) + except Diagnostic as error: + raise Diagnostic( + f"The source repository {field(source_repo)} is not a fork of {field(gh.upstream_repo)}.", + "Push the temporary source branch to a fork of the configured upstream and " + "update the lock URL, branch, SHA, and vendored snapshot with " + "`scripts/vendor/manage.py`.", + ) from error + for sha in (previous.commit, reviewed.commit): + promote._git(cache, "fetch", "--quiet", "--no-tags", reviewed.url, sha) + # An empty successful query means unrelated histories; Git failures remain + # operator errors rather than being relabeled as author mistakes. + common = promote._git( + cache, "merge-base", previous.commit, reviewed.commit, allowed_returncodes=(0, 1) + ) + rebase_action = ( + f"Rebase the temporary source branch {field(reviewed.branch)} onto canonical " + f"repository {field(previous.url)}, branch {field(previous.branch)}, SHA " + f"`{previous.commit}`. Then use `scripts/vendor/manage.py` to update " + "`3rdparty/vendor_sources.lock.yaml` and the vendored snapshot to the rebased SHA, " + "and push the source branch and consumer PR. Editing metadata alone will not fix this." + ) + if not common: + raise Diagnostic("Source revisions have no merge base.", rebase_action) + if common != previous.commit: + raise Diagnostic("Source update is not a fast-forward.", rebase_action) + if entry["unpaired_reason"]: + commits = _commits(cache, previous.commit, reviewed.commit) + if not commits or any( + len(promote._git(cache, "show", "-s", "--format=%P", sha).split()) != 1 + for sha in commits + ): + raise Diagnostic( + "Unpaired source updates still require a nonempty linear history.", rebase_action + ) + return {}, {} + prs = {} + for number in entry["upstream_prs"]: + pr = gh.get(f"repos/{gh.upstream_repo}/pulls/{number}") + if pr["base"]["ref"] != gh.upstream_branch or ( + pr["state"] == "closed" and not pr.get("merged") + ): + raise Diagnostic( + f"{_upstream_reference(number, gh.upstream_repo)} must target " + f"`{gh.upstream_branch}` and be open or merged; currently targets " + f"{field(pr['base']['ref'])} and is " + + ( + "closed without merging." + if pr["state"] == "closed" and not pr.get("merged") + else f"{field(pr['state'])}." + ), + "Correct the upstream PR target or reopen the upstream PR if appropriate; " + "otherwise replace its URL in the consumer PR description with the correct " + "open/merged upstream PR. Do not remove attribution for changes still included.", + ) + for sha in (pr["base"]["sha"], pr["head"]["sha"]): + promote._git( + cache, + "fetch", + "--quiet", + "--no-tags", + f"https://github.com/{gh.upstream_repo}.git", + promote._sha(sha), + ) + prs[number] = pr + return match( + cache, + previous.commit, + reviewed.commit, + prs, + entry["resolutions"], + source=reviewed.source, + include=reviewed.include, + upstream_repo=gh.upstream_repo, + vendor_name=gh.vendor_name, + ) diff --git a/scripts/vendor/promote.py b/scripts/vendor/promote.py new file mode 100644 index 000000000000..d06ed02200c4 --- /dev/null +++ b/scripts/vendor/promote.py @@ -0,0 +1,1243 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Promote reviewed vendor source revisions; dry-run unless --publish is supplied. + +Only Git, gh, and PyYAML are required. GitHub reads and temporary source fetches +are allowed in dry-run; remote writes and consumer worktrees require --publish. +Provenance lives in signed-off commits, not in a separate tracking file. +""" + +from __future__ import annotations + +import argparse +import base64 +import contextlib +import json +import os +import re +import subprocess +import sys +import tempfile +import time +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import urlencode + +import yaml + +if __package__: + from . import manage as vendor + from .diagnostics import Diagnostic, field, yaml_problem +else: + import manage as vendor + from diagnostics import Diagnostic, field, yaml_problem + +_LOCK = "3rdparty/vendor_sources.lock.yaml" +_BEGIN = "----- BEGIN VENDOR PROMOTION V1 -----" +_END = "----- END VENDOR PROMOTION V1 -----" +_SHA = re.compile(r"[0-9a-f]{40}\Z") +_CI_SKIP_COMMAND = '/bot skip --comment "skip CI since no code change"' + + +def _run( + command: list[str], + *, + cwd: Path | None = None, + text: str | None = None, + timeout: int = 300, + stream: bool = False, + strip: bool = True, + binary_output: bool = False, + allowed_returncodes: tuple[int, ...] = (0,), +) -> str: + # Do not use manage._run_git here: its synthetic identity is for + # patch generation. Real promotion commits must use the maintainer's identity. + env = os.environ.copy() + for key in vendor._GIT_LOCAL_ENVIRONMENT_VARIABLES: + env.pop(key, None) + env.setdefault("GH_CONFIG_DIR", str(Path.home() / ".config/gh")) + env.update(GIT_TERMINAL_PROMPT="0", GH_PROMPT_DISABLED="1", GIT_NO_REPLACE_OBJECTS="1") + if command[0] == "git": + # Lock-only worktrees need source text, not unrelated LFS assets. + env["GIT_LFS_SKIP_SMUDGE"] = "1" + result = subprocess.run( + command, + cwd=cwd, + env=env, + input=text.encode() if binary_output and text is not None else text, + text=not binary_output, + capture_output=not stream, + timeout=timeout, + ) + if result.returncode not in allowed_returncodes: + detail = (result.stderr or result.stdout or "See streamed output above.").strip() + raise RuntimeError(f"{command[0]} failed: {detail}") + output = result.stdout or "" + if isinstance(output, bytes): + output = output.decode("utf-8", errors="surrogateescape") + return output.strip() if strip else output + + +def _git( + repo: Path, *args: str, text: str | None = None, allowed_returncodes: tuple[int, ...] = (0,) +) -> str: + return _run( + ["git", "-C", str(repo), *args], + text=text, + timeout=1800 if args[:2] == ("worktree", "add") else 300, + stream=args[:2] == ("worktree", "add") or args[0] in ("fetch", "commit", "push"), + allowed_returncodes=allowed_returncodes, + ) + + +class GitHub: + """Small gh-backed client; respects GH_CONFIG_DIR without handling tokens.""" + + def __init__( + self, + consumer_repo: str = "NVIDIA/TensorRT-LLM", + vendor_name: str = "", + upstream_repo: str = "", + base_branch: str = "main", + upstream_branch: str = "main", + ) -> None: + self.consumer_repo = _repo_name(f"https://github.com/{consumer_repo}") + self.vendor_name = vendor_name + self.upstream_repo = upstream_repo + self.base_branch = base_branch + self.upstream_branch = upstream_branch + + def api( + self, endpoint: str, *, method: str = "GET", payload: dict | None = None + ) -> dict | list: + command = ["gh", "api", "--hostname", "github.com", "--method", method, endpoint] + if payload is not None: + command += ["--input", "-"] + result = json.loads(_run(command, text=None if payload is None else json.dumps(payload))) + if not isinstance(result, (dict, list)): + raise ValueError(f"Unexpected GitHub response for {endpoint}") + return result + + def get(self, endpoint: str) -> dict: + result = self.api(endpoint) + if not isinstance(result, dict): + raise ValueError(f"Expected an object from {endpoint}") + return result + + def optional(self, endpoint: str) -> dict | None: + try: + return self.get(endpoint) + except RuntimeError as error: + if "HTTP 404" not in str(error): + raise + return None + + def compare(self, repo: str, base: str, head: str, *, all_commits: bool = False) -> dict: + first = self.get(f"repos/{repo}/compare/{base}...{head}?per_page=100&page=1") + if all_commits: + count = int(first["total_commits"]) + for page in range(2, (count + 99) // 100 + 1): + more = self.get(f"repos/{repo}/compare/{base}...{head}?per_page=100&page={page}") + first["commits"].extend(more["commits"]) + if len(first["commits"]) != count: + raise ValueError("Incomplete GitHub comparison; refusing partial provenance.") + return first + + +def _sha(value: str) -> str: + if not isinstance(value, str) or not _SHA.fullmatch(value): + raise ValueError(f"Expected a full immutable commit SHA, got {value!r}.") + return value + + +def _repo_name(url: str) -> str: + match = re.fullmatch( + r"(?:https://github\.com/|git@github\.com:)([\w.-]+/[\w.-]+?)(?:\.git)?/?", url + ) + if not match: + raise ValueError(f"Expected a credential-free GitHub repository URL, got {url!r}.") + return match[1] + + +def _pr_number(value: str, repo: str) -> int: + for prefix in (f"https://github.com/{repo}/pull/", f"{repo}#"): + if value.startswith(prefix): + value = value[len(prefix) :] + break + if not re.fullmatch(r"[1-9][0-9]*", value): + raise ValueError(f"Expected a PR number or PR URL in {repo}: {value!r}.") + return int(value) + + +def _lock_document_at(gh: GitHub, repo: str, commit: str) -> dict: + response = gh.get(f"repos/{repo}/contents/{_LOCK}?ref={_sha(commit)}") + if response.get("encoding") != "base64": + raise ValueError("GitHub did not return the complete vendor lock.") + try: + content = base64.b64decode("".join(response["content"].split()), validate=True) + if any( + isinstance(token, (yaml.tokens.AliasToken, yaml.tokens.AnchorToken)) + for token in yaml.scan(content) + ): + raise Diagnostic( + "Vendor lock YAML aliases and anchors are not allowed.", + f"Replace anchors and aliases with explicit values in `{_LOCK}` and push the PR.", + ) + data = yaml.load(content, Loader=vendor._UniqueKeyLoader) + except yaml.YAMLError as error: + raise Diagnostic( + f"Invalid vendor lock YAML: {yaml_problem(error)}.", + f"Correct the YAML at that location in `{_LOCK}`, keeping keys unique and " + "using spaces for indentation; then push the PR.", + ) from error + if ( + not isinstance(data, dict) + or set(data) != {"schema_version", "vendors"} + or type(data["schema_version"]) is not int + or data["schema_version"] != 1 + or not isinstance(data["vendors"], dict) + ): + raise Diagnostic( + "Unsupported vendor lock schema.", + f"In `{_LOCK}`, use only top-level `schema_version: 1` and a `vendors` mapping. " + "Preserve other vendors' entries and push the corrected file.", + ) + return data + + +def _lock_at(gh: GitHub, repo: str, commit: str) -> vendor.Vendor: + data = _lock_document_at(gh, repo, commit) + return vendor._validate_vendor(gh.vendor_name, data["vendors"][gh.vendor_name]) + + +def _main_sha(gh: GitHub) -> str: + return _sha(gh.get(f"repos/{gh.consumer_repo}/git/ref/heads/{gh.base_branch}")["object"]["sha"]) + + +def _check_fork(gh: GitHub, repo: str, upstream: str) -> None: + info = gh.get(f"repos/{repo}") + if ( + repo.lower() == upstream.lower() + or info.get("source", {}).get("full_name", "").lower() != upstream.lower() + ): + raise Diagnostic( + f"{field(repo)} must be a personal fork of {field(upstream)}, not upstream itself.", + "Use a personal fork of the configured upstream repository for publication.", + ) + + +def _check_source_repo(gh: GitHub, repo: str) -> None: + if repo.lower() != gh.upstream_repo.lower(): + _check_fork(gh, repo, gh.upstream_repo) + + +def _record_from_message(message: str) -> dict: + legacy = "----- BEGIN PRIMTS PROMOTION V1 -----" + if legacy in message: + message = message.replace(legacy, _BEGIN).replace( + "----- END PRIMTS PROMOTION V1 -----", _END + ) + if message.count(_BEGIN) != 1 or message.count(_END) != 1: + raise ValueError("Missing or ambiguous Vendor promotion metadata in commit message.") + record = json.loads(message.split(_BEGIN, 1)[1].split(_END, 1)[0]) + if not isinstance(record, dict) or record.get("version") != 1: + raise ValueError("Unsupported Vendor promotion metadata version.") + if "trtllm_pr" in record: + record["consumer_pr"] = record.pop("trtllm_pr") + record["consumer_merge"] = record.pop("trtllm_merge") + record["upstream_repository"] = "flashinfer-ai/flashinfer" + record["consumer_base"] = "main" + record["upstream_branch"] = "main" + return record + + +def _record_text(record: dict) -> str: + return f"{_BEGIN}\n{json.dumps(record, indent=2, sort_keys=True)}\n{_END}" + + +def _map_commits( + commits: list[dict], prs: dict[int, dict], mappings: list[str], upstream_repo: str +) -> dict[str, int]: + assignments: dict[str, int] = {} + for mapping in mappings: + prefix, separator, reference = mapping.partition("=") + matches = [item["sha"] for item in commits if item["sha"].startswith(prefix)] + if not separator or not re.fullmatch(r"[0-9a-f]{7,40}", prefix) or len(matches) != 1: + raise ValueError(f"Ambiguous or invalid --map-upstream {mapping!r}.") + number = _pr_number(reference, upstream_repo) + if number not in prs or matches[0] in assignments: + raise ValueError(f"Duplicate mapping or PR missing from --upstream-pr: {mapping!r}.") + assignments[matches[0]] = number + for item in commits: + sha = item["sha"] + if sha in assignments: + continue + if len(prs) == 1: + assignments[sha] = next(iter(prs)) + elif sys.stdin.isatty(): + title = item["commit"]["message"].splitlines()[0] + answer = input(f"Upstream PR for {sha[:12]} ({title}), choose {sorted(prs)}: ") + number = _pr_number(answer, upstream_repo) + if number not in prs: + raise ValueError("Answer must identify a supplied upstream PR.") + assignments[sha] = number + else: + raise ValueError(f"Provide --map-upstream {sha}=PR for every unmapped commit.") + if set(assignments.values()) != set(prs): + raise ValueError("Every supplied upstream PR must map to at least one new source commit.") + return assignments + + +@dataclass +class Promotion: + """An immutable source revision and the expected consumer/canonical states.""" + + number: int + main_sha: str + previous: vendor.Vendor + reviewed: vendor.Vendor + canonical_repo: str + fork: str + record: dict + consumer_repo: str = "NVIDIA/TensorRT-LLM" + base_branch: str = "main" + completed_pr: dict | None = None + + @property + def branch(self) -> str: + return f"chore/{self.previous.name}-promote-{self.number}-{self.reviewed.commit[:12]}" + + @property + def title(self) -> str: + return f"[None][chore] promote {self.previous.name} source from #{self.number}" + + @property + def promoted(self) -> dict: + result = self.reviewed.to_mapping() + result.update(url=self.previous.url, branch=self.previous.branch) + return result + + +def _check_current(gh: GitHub, plan: Promotion) -> None: + current = _lock_at(gh, gh.consumer_repo, _main_sha(gh)).to_mapping() + if current not in (plan.reviewed.to_mapping(), plan.promoted): + raise ValueError( + "Consumer branch's vendor lock has been superseded; do not promote this PR." + ) + ref = gh.get(f"repos/{plan.canonical_repo}/git/ref/heads/{plan.previous.branch}") + if ref["object"]["sha"] not in (plan.previous.commit, plan.reviewed.commit): + raise ValueError( + "Canonical branch moved unexpectedly; coordinate with the other maintainer." + ) + + +def _plan_identity(args: argparse.Namespace, gh: GitHub) -> Promotion: + """Identify an operation from immutable consumer history, without live attribution.""" + number = _pr_number(args.source_pr, gh.consumer_repo) + pr = gh.get(f"repos/{gh.consumer_repo}/pulls/{number}") + if not pr.get("merged") or pr["base"]["ref"] != gh.base_branch: + raise ValueError( + "--source-pr must identify a PR already merged into the consumer base branch." + ) + merge_sha = _sha(pr["merge_commit_sha"]) + merge = gh.get(f"repos/{gh.consumer_repo}/commits/{merge_sha}") + previous = _lock_at(gh, gh.consumer_repo, merge["parents"][0]["sha"]) + reviewed = _lock_at(gh, gh.consumer_repo, merge_sha) + canonical_repo = _repo_name(previous.url) + source_repo = _repo_name(reviewed.url) + expected_repo = _repo_name(f"https://github.com/{args.canonical_repo}") + _run(["git", "check-ref-format", f"refs/heads/{args.canonical_branch}"]) + if canonical_repo.lower() != expected_repo.lower() or previous.branch != args.canonical_branch: + raise ValueError( + "Previous lock does not match the explicit canonical repository/branch; " + "finish the preceding promotion first." + ) + if ( + previous.commit == reviewed.commit + or not reviewed.branch + or (canonical_repo.lower() == source_repo.lower() and reviewed.branch == previous.branch) + ): + raise ValueError("The PR must introduce a new source SHA on a temporary branch.") + fork = args.fork or _repo_name(_git(args.repo, "remote", "get-url", "fork")) + record = { + "version": 1, + "kind": "promotion", + "vendor": gh.vendor_name, + "consumer_pr": f"https://github.com/{gh.consumer_repo}/pull/{number}", + "consumer_merge": merge_sha, + "consumer_base": gh.base_branch, + "upstream_repository": gh.upstream_repo, + "upstream_branch": gh.upstream_branch, + "canonical_url": previous.url, + "canonical_branch": previous.branch, + "previous": previous.commit, + "promoted": reviewed.commit, + "source_url": reviewed.url, + "source_branch": reviewed.branch, + } + return Promotion( + number, + _main_sha(gh), + previous, + reviewed, + canonical_repo, + fork, + record, + consumer_repo=gh.consumer_repo, + base_branch=gh.base_branch, + ) + + +def _make_plan( + args: argparse.Namespace, + gh: GitHub, + *, + auto_match: bool = False, + resolutions: dict[str, dict] | None = None, +) -> Promotion: + if args.unpaired_reason is not None: + if not args.unpaired_reason.strip() or args.upstream_pr or args.map_upstream: + raise ValueError( + "--unpaired-reason requires a nonempty reason and cannot be combined " + "with --upstream-pr or --map-upstream." + ) + elif not args.upstream_pr: + raise ValueError("Provide --upstream-pr or an explicit --unpaired-reason.") + plan = _plan_identity(args, gh) + previous, reviewed, record = plan.previous, plan.reviewed, plan.record + canonical_repo, fork = plan.canonical_repo, plan.fork + source_repo = _repo_name(reviewed.url) + # A completed operation is historical: later promotions, deleted temporary + # forks, or changed upstream PR state must not make it run again or fail. + existing = _find_followup(gh, plan) + if existing: + existing = gh.get(f"repos/{gh.consumer_repo}/pulls/{existing['number']}") + if existing.get("merged"): + _verify_remote_commit(gh, plan, gh.consumer_repo, existing["merge_commit_sha"]) + plan.completed_pr = existing + return plan + _check_source_repo(gh, canonical_repo) + _check_source_repo(gh, source_repo) + _check_fork(gh, fork, gh.consumer_repo) + comparison = gh.compare(canonical_repo, previous.commit, reviewed.commit, all_commits=True) + if comparison["merge_base_commit"]["sha"] != previous.commit: + raise ValueError( + "Reviewed source is not a fast-forward of the previous canonical revision." + ) + commits = comparison["commits"] + if not commits or any(len(item["parents"]) != 1 for item in commits): + raise ValueError( + "Promote requires a nonempty, linear source commit range; rebase merges first." + ) + prs = {} + for reference in args.upstream_pr: + upstream_number = _pr_number(reference, gh.upstream_repo) + if upstream_number in prs: + raise ValueError("Duplicate --upstream-pr.") + upstream = gh.get(f"repos/{gh.upstream_repo}/pulls/{upstream_number}") + if upstream["base"]["ref"] != gh.upstream_branch or ( + upstream["state"] == "closed" and not upstream.get("merged") + ): + raise ValueError( + f"Upstream PR must target {gh.upstream_branch} and be open (draft allowed) or merged." + ) + prs[upstream_number] = upstream + if auto_match and prs: + if __package__: + from . import metadata + else: + import metadata + with tempfile.TemporaryDirectory(prefix="vendor-attribution-") as directory: + entry = { + "upstream_prs": list(prs), + "unpaired_reason": None, + "resolutions": resolutions or {}, + } + assignments, evidence = metadata.resolve(gh, previous, reviewed, entry, Path(directory)) + record["attribution"] = evidence + record["attribution_scope"] = {"source": reviewed.source, "include": list(reviewed.include)} + else: + assignments = _map_commits(commits, prs, args.map_upstream, gh.upstream_repo) if prs else {} + upstream_main = _sha( + gh.get(f"repos/{gh.upstream_repo}/git/ref/heads/{gh.upstream_branch}")["object"]["sha"] + ) + upstream_base = gh.compare(canonical_repo, previous.commit, upstream_main)["merge_base_commit"][ + "sha" + ] + groups = [] + if args.unpaired_reason is not None: + groups.append( + { + "commits": [item["sha"] for item in commits], + "upstream_pr": None, + "upstream_base": None, + "upstream_head": None, + "upstream_merge": None, + "unpaired_reason": args.unpaired_reason.strip(), + "refresh_policy": "retain", + } + ) + outside_scope = [ + item["sha"] + for item in commits + if item["sha"] in assignments and assignments[item["sha"]] is None + ] + if outside_scope: + groups.append( + { + "commits": outside_scope, + "upstream_pr": None, + "upstream_base": None, + "upstream_head": None, + "upstream_merge": None, + "unpaired_reason": ( + "No changes to files selected by the vendor lock source/include. " + "No whole-commit upstream equivalence is claimed." + ), + "refresh_policy": "retain", + } + ) + for upstream_number, upstream in prs.items(): + groups.append( + { + "commits": [ + item["sha"] for item in commits if assignments[item["sha"]] == upstream_number + ], + "upstream_pr": f"https://github.com/{gh.upstream_repo}/pull/{upstream_number}", + "upstream_base": _sha(upstream["base"]["sha"]), + "upstream_head": _sha(upstream["head"]["sha"]), + "upstream_merge": upstream.get("merge_commit_sha") + if upstream.get("merged") + else None, + } + ) + record.update( + { + "upstream_base": _sha(upstream_base), + "upstream_observed_main": upstream_main, + "changes": groups, + } + ) + _check_current(gh, plan) + return plan + + +def _find_followup(gh: GitHub, plan: Promotion) -> dict | None: + query = urlencode( + { + "state": "all", + "base": plan.base_branch, + "head": f"{plan.fork.split('/')[0]}:{plan.branch}", + } + ) + prs = gh.api(f"repos/{gh.consumer_repo}/pulls?{query}") + if not isinstance(prs, list) or len(prs) > 1: + raise ValueError("Ambiguous existing promotion PRs.") + if prs and prs[0]["state"] == "closed" and not prs[0].get("merged_at"): + raise ValueError("The promotion PR was closed without merging; inspect it before retrying.") + return prs[0] if prs else None + + +def _local_branch(repo: Path, branch: str) -> str | None: + reference = f"refs/heads/{branch}" + for line in _git( + repo, "for-each-ref", "--format=%(refname) %(objectname)", reference + ).splitlines(): + name, sha = line.split() + if name == reference: + return _sha(sha) + return None + + +def _recover_plan(args: argparse.Namespace, gh: GitHub, plan: Promotion) -> bool: + """Restore verified provenance from Git/GitHub before consulting mutable metadata.""" + existing = _find_followup(gh, plan) + if existing: + existing = gh.get(f"repos/{gh.consumer_repo}/pulls/{existing['number']}") + sha = existing["merge_commit_sha"] if existing.get("merged") else existing["head"]["sha"] + _verify_remote_commit(gh, plan, gh.consumer_repo, sha) + if existing.get("merged"): + plan.completed_pr = existing + return True + else: + remote = gh.optional(f"repos/{plan.fork}/git/ref/heads/{plan.branch}") + if remote: + _verify_remote_commit(gh, plan, plan.fork, remote["object"]["sha"]) + else: + sha = _local_branch(args.repo, plan.branch) + if sha is None: + return False + _git( + args.repo, + "fetch", + "--quiet", + "--no-tags", + f"https://github.com/{plan.consumer_repo}.git", + plan.main_sha, + ) + if _git(args.repo, "merge-base", sha, plan.main_sha) == sha: + return False + parents = _git(args.repo, "show", "-s", "--format=%P", sha).split() + if ( + len(parents) != 1 + or _git(args.repo, "diff", "--name-only", parents[0], sha) != _LOCK + ): + raise ValueError("Local promotion commit must change only the vendor lock.") + _git(args.repo, "merge-base", "--is-ancestor", parents[0], plan.main_sha) + _check_document_change( + _local_lock_document(args.repo, parents[0]), + _local_lock_document(args.repo, sha), + plan, + ) + message = _git(args.repo, "show", "-s", "--format=%B", sha) + _adopt_record(plan, message) + if not re.search(r"^Signed-off-by: .+ <.+>$", message, re.MULTILINE): + raise ValueError("Local promotion commit has no DCO sign-off.") + _check_source_repo(gh, plan.canonical_repo) + _check_source_repo(gh, _repo_name(plan.reviewed.url)) + _check_fork(gh, plan.fork, gh.consumer_repo) + _check_current(gh, plan) + return True + + +def _adopt_record(plan: Promotion, message: str) -> None: + """Keep the original upstream snapshot when resuming an already-published run.""" + existing = _record_from_message(message) + stable_keys = set(plan.record) - {"upstream_base", "upstream_observed_main", "changes"} + if any(existing.get(key) != plan.record[key] for key in stable_keys): + raise ValueError("Existing promotion metadata does not describe this source update.") + _validate_snapshot(existing, plan.record["upstream_repository"]) + if "attribution_scope" in existing and existing["attribution_scope"] != { + "source": plan.reviewed.source, + "include": list(plan.reviewed.include), + }: + raise ValueError("Promotion attribution scope does not match the reviewed vendor lock.") + if "changes" not in plan.record: + # A verified operation identity can restore its entire durable snapshot. + plan.record = existing + return + expected = { + group["upstream_pr"]: ( + group["commits"], + group.get("unpaired_reason"), + group.get("refresh_policy"), + ) + for group in plan.record["changes"] + } + actual = { + group["upstream_pr"]: ( + group["commits"], + group.get("unpaired_reason"), + group.get("refresh_policy"), + ) + for group in existing["changes"] + } + if actual != expected: + raise ValueError("Existing promotion has different upstream PR/commit mappings.") + plan.record = existing + + +def _validate_snapshot(record: dict, upstream_repo: str) -> None: + _sha(record["upstream_base"]) + _sha(record["upstream_observed_main"]) + if "attribution_scope" in record: + scope = record["attribution_scope"] + if not isinstance(scope, dict) or set(scope) != {"source", "include"}: + raise ValueError("Invalid promotion attribution scope.") + vendor._validate_relative_path(scope["source"], "Attribution source") + vendor._validate_include(scope["include"], record["vendor"]) + commits = [] + if not isinstance(record["changes"], list) or not record["changes"]: + raise ValueError("Promotion metadata must contain source changes.") + for group in record["changes"]: + if not isinstance(group["commits"], list) or not group["commits"]: + raise ValueError("Promotion change group has no commits.") + commits.extend(_sha(sha) for sha in group["commits"]) + if group["upstream_pr"] is None: + if ( + group.get("refresh_policy") != "retain" + or not group.get("unpaired_reason", "").strip() + ): + raise ValueError( + "Unpaired promotion changes must retain an explicit reason/policy." + ) + else: + _pr_number(group["upstream_pr"], upstream_repo) + _sha(group["upstream_base"]) + _sha(group["upstream_head"]) + if group["upstream_merge"] is not None: + _sha(group["upstream_merge"]) + if len(set(commits)) != len(commits) or record["promoted"] not in commits: + raise ValueError("Promotion metadata has duplicate or missing source commits.") + + +def _verify_remote_commit(gh: GitHub, plan: Promotion, repo: str, sha: str) -> str: + commit = gh.get(f"repos/{repo}/commits/{_sha(sha)}") + if len(commit["parents"]) != 1 or [item["filename"] for item in commit["files"]] != [_LOCK]: + raise ValueError("Promotion commit must change only the vendor lock and have one parent.") + parent = commit["parents"][0]["sha"] + comparison = gh.compare(gh.consumer_repo, parent, _main_sha(gh)) + if comparison["merge_base_commit"]["sha"] != parent: + raise ValueError( + "Promotion parent is not on the consumer base branch; unrelated commits may be present." + ) + before = _lock_document_at(gh, repo, parent) + after = _lock_document_at(gh, repo, sha) + _check_document_change(before, after, plan) + message = commit["commit"]["message"] + _adopt_record(plan, message) + if not re.search(r"^Signed-off-by: .+ <.+>$", message, re.MULTILINE): + raise ValueError("Promotion commit has no DCO sign-off.") + return message + + +def _check_document_change(before: dict, after: dict, plan: Promotion) -> None: + if before["vendors"][plan.previous.name] != plan.reviewed.to_mapping(): + raise ValueError("Promotion parent does not contain the reviewed temporary pin.") + expected = {**before, "vendors": {**before["vendors"], plan.previous.name: plan.promoted}} + if expected != after: + raise ValueError("Promotion commit is not the expected branch/URL-only lock change.") + + +@contextlib.contextmanager +def _source_repo(args: argparse.Namespace, plan: Promotion) -> Iterator[Path]: + with tempfile.TemporaryDirectory(prefix="vendor-promote-source-") as temporary: + repo = args.source_repo or Path(temporary) + if args.source_repo is None: + print(f"Fetching reviewed vendor source {plan.reviewed.commit}...", flush=True) + _git(repo, "init", "--quiet") + _git(repo, "fetch", "--quiet", "--no-tags", plan.reviewed.url, plan.reviewed.commit) + _git(repo, "cat-file", "-e", f"{plan.reviewed.commit}^{{commit}}") + _git(repo, "merge-base", "--is-ancestor", plan.previous.commit, plan.reviewed.commit) + yield repo + + +def _local_lock_document(worktree: Path, revision: str | None = None) -> dict: + content = ( + (worktree / _LOCK).read_text() + if revision is None + else _git(worktree, "show", f"{revision}:{_LOCK}") + ) + return yaml.load(content, Loader=vendor._UniqueKeyLoader) + + +def _pending_paths(worktree: Path) -> set[str]: + # A file restored in the working tree can still have an unrelated staged edit. + return set(_git(worktree, "diff", "--name-only", "HEAD").splitlines()) | set( + _git(worktree, "diff", "--cached", "--name-only", "HEAD").splitlines() + ) + + +def _check_lock_only(worktree: Path, before: dict, plan: Promotion) -> None: + _check_document_change(before, _local_lock_document(worktree), plan) + if _pending_paths(worktree) != {_LOCK}: + raise ValueError("Promotion must change only the vendor lock.") + if _git(worktree, "ls-files", "--others", "--exclude-standard"): + raise ValueError("Unexpected untracked files in the promotion worktree.") + + +def _prepare_commit(args: argparse.Namespace, plan: Promotion, source: Path) -> tuple[Path, str]: + print(f"Fetching consumer base {plan.main_sha}...", flush=True) + _git( + args.repo, + "fetch", + "--quiet", + "--no-tags", + f"https://github.com/{plan.consumer_repo}.git", + plan.main_sha, + ) + worktree = args.worktree or args.repo.parent / plan.branch.removeprefix("chore/") + worktree = worktree.resolve() + branch_exists = _local_branch(args.repo, plan.branch) is not None + missing_worktree = False + if branch_exists and not worktree.exists(): + for block in _git(args.repo, "worktree", "list", "--porcelain", "-z").split("\0\0"): + fields = dict(line.split(" ", 1) for line in block.split("\0") if " " in line) + if fields.get("branch") != f"refs/heads/{plan.branch}": + continue + registered = Path(fields["worktree"]) + if registered.exists(): + worktree = registered + else: + missing_worktree = True + break + if worktree.exists(): + if _git(worktree, "branch", "--show-current") != plan.branch or _git( + worktree, "rev-parse", "--path-format=absolute", "--git-common-dir" + ) != _git(args.repo, "rev-parse", "--path-format=absolute", "--git-common-dir"): + raise ValueError( + f"Existing worktree is not this repository's promotion branch: {worktree}" + ) + head = _git(worktree, "rev-parse", "HEAD") + if _git(worktree, "merge-base", head, plan.main_sha) != head: + if _git(worktree, "status", "--porcelain"): + raise ValueError( + f"Completed promotion worktree contains unexpected edits: {worktree}" + ) + _adopt_record(plan, _git(worktree, "log", "-1", "--format=%B")) + parent = _git(worktree, "rev-parse", "HEAD^") + _git(worktree, "merge-base", "--is-ancestor", parent, plan.main_sha) + before = _local_lock_document(worktree, "HEAD^") + _check_committed_lock(worktree, before, plan) + lock = vendor._load_lock(worktree / _LOCK) + vendor._verify_source(lock, lock.vendors[plan.previous.name], source) + print(f"Reusing completed promotion commit {head} in {worktree}.", flush=True) + return worktree, head + print(f"Resuming incomplete promotion in {worktree}...", flush=True) + else: + print(f"Creating promotion worktree {worktree} (LFS downloads disabled)...", flush=True) + if branch_exists: + # Force is restricted to a registration whose directory is gone; + # existing worktrees are reused and validated above, never replaced. + flags = ("--force",) if missing_worktree else () + _git(args.repo, "worktree", "add", *flags, str(worktree), plan.branch) + return _prepare_commit(args, plan, source) + _git(args.repo, "worktree", "add", "-b", plan.branch, str(worktree), plan.main_sha) + before = _local_lock_document(worktree, "HEAD") + if before["vendors"][plan.previous.name] != plan.reviewed.to_mapping(): + raise ValueError("Worktree base no longer contains the reviewed temporary pin.") + # Recover only the two states this tool writes. Preserve arbitrary user or + # hook edits, including staged-only changes, instead of overwriting them. + for revision in (None, ""): + pending = _local_lock_document(worktree, revision) + if pending != before: + _check_document_change(before, pending, plan) + if _pending_paths(worktree) - {_LOCK}: + raise ValueError( + "Incomplete promotion has changes outside the vendor lock; inspect before retrying." + ) + if _git(worktree, "ls-files", "--others", "--exclude-standard"): + raise ValueError("Incomplete promotion has untracked files; inspect before retrying.") + lock = vendor._load_lock(worktree / _LOCK) + print("Verifying source materialization and the lock-only change...", flush=True) + vendor._verify_source(lock, lock.vendors[plan.previous.name], source) + result = vendor.main( + [ + "--lock", + str(worktree / _LOCK), + "pin", + plan.previous.name, + "--url", + plan.previous.url, + "--branch", + plan.previous.branch, + "--commit", + plan.reviewed.commit, + "--repo", + str(source), + ] + ) + if result: + raise ValueError("Vendor pin failed; worktree preserved for inspection.") + _check_lock_only(worktree, before, plan) + _git(worktree, "add", "--", _LOCK) + print("Creating signed-off promotion commit...", flush=True) + _git(worktree, "commit", "-s", "-F", "-", text=f"{plan.title}\n\n{_record_text(plan.record)}\n") + if _git(worktree, "status", "--porcelain"): + raise ValueError(f"Commit hooks left changes in {worktree}; inspect and retry.") + _check_committed_lock(worktree, before, plan) + return worktree, _git(worktree, "rev-parse", "HEAD") + + +def _check_committed_lock(worktree: Path, before: dict, plan: Promotion) -> None: + _check_document_change(before, _local_lock_document(worktree), plan) + if len(_git(worktree, "rev-list", "--parents", "-n", "1", "HEAD").split()) != 2: + raise ValueError("Promotion commit must have exactly one parent.") + if _git(worktree, "diff", "--name-only", "HEAD^", "HEAD") != _LOCK: + raise ValueError("Commit hooks added changes outside the vendor lock.") + message = _git(worktree, "log", "-1", "--format=%B") + if _record_from_message(message) != plan.record: + raise ValueError("Commit hooks changed the promotion provenance.") + if not re.search(r"^Signed-off-by: .+ <.+>$", message, re.MULTILINE): + raise ValueError("Promotion commit has no DCO sign-off.") + + +def _check_auto_merge(gh: GitHub) -> None: + settings = gh.get(f"repos/{gh.consumer_repo}") + if not settings.get("allow_auto_merge") or not settings.get("allow_squash_merge"): + raise ValueError("Repository must allow auto-merge and squash merging.") + rules = gh.api(f"repos/{gh.consumer_repo}/rules/branches/{gh.base_branch}") + if not isinstance(rules, list) or any(rule["type"] == "merge_queue" for rule in rules): + raise ValueError( + "Merge queues are unsupported: custom squash-message retention is required." + ) + + +def _disable_auto_merge(gh: GitHub, pr: dict) -> None: + gh.api( + "graphql", + method="POST", + payload={ + "query": ( + "mutation($id:ID!){disablePullRequestAutoMerge(" + "input:{pullRequestId:$id}){clientMutationId}}" + ), + "variables": {"id": pr["node_id"]}, + }, + ) + + +def _report_merged(pr: dict) -> None: + print( + f"Promotion merged: {pr['html_url']}\n" + f"Squash commit: {pr['merge_commit_sha']}\n" + "Final lock-only change, provenance, and DCO sign-off verified.", + flush=True, + ) + + +def _arm_auto_merge(gh: GitHub, plan: Promotion, pr: dict, message: str) -> None: + _check_auto_merge(gh) + _check_current(gh, plan) + number, head = pr["number"], pr["head"]["sha"] + state = gh.api( + "graphql", + method="POST", + payload={ + "query": "query($id:ID!){node(id:$id){... on PullRequest{headRefOid mergeStateStatus}}}", + "variables": {"id": pr["node_id"]}, + }, + ) + node = state["data"]["node"] + if node["headRefOid"] != head: + raise ValueError("Follow-up PR head changed; revalidate before enabling auto-merge.") + title, _, body = message.partition("\n") + saved = pr.get("auto_merge") + if saved: + if ( + saved.get("merge_method") == "squash" + and saved.get("commit_title") == title + and saved.get("commit_message", "").strip() == body.strip() + ): + print(f"Existing squash auto-merge message verified: {pr['html_url']}") + return + _disable_auto_merge(gh, pr) + raise ValueError("Existing auto-merge message differs; disabled. Inspect before retrying.") + # Query only necessary fields: gh pr merge's broader PR query can require + # read:org even when the token already has permission to merge this PR. + operation = ( + "mergePullRequest" if node["mergeStateStatus"] == "CLEAN" else "enablePullRequestAutoMerge" + ) + input_type = ( + "MergePullRequestInput" + if operation == "mergePullRequest" + else "EnablePullRequestAutoMergeInput" + ) + gh.api( + "graphql", + method="POST", + payload={ + "query": f"mutation($input:{input_type}!){{{operation}(input:$input){{clientMutationId}}}}", + "variables": { + "input": { + "pullRequestId": pr["node_id"], + "expectedHeadOid": head, + "mergeMethod": "SQUASH", + "commitHeadline": title, + "commitBody": body.strip(), + } + }, + }, + ) + updated = gh.get(f"repos/{gh.consumer_repo}/pulls/{number}") + if updated.get("merged"): + _verify_remote_commit(gh, plan, gh.consumer_repo, updated["merge_commit_sha"]) + _report_merged(updated) + else: + saved = updated.get("auto_merge") or {} + if ( + updated["head"]["sha"] != head + or saved.get("merge_method") != "squash" + or saved.get("commit_title") != title + or saved.get("commit_message", "").strip() != body.strip() + ): + _disable_auto_merge(gh, pr) + raise ValueError("Auto-merge message/head verification failed; auto-merge disabled.") + print(f"Squash auto-merge enabled and message verified: {updated['html_url']}") + + +def _pr_body(plan: Promotion, message: str) -> str: + unpaired = "" + if any(group["upstream_pr"] is None for group in plan.record["changes"]): + unpaired = ( + "Some source commits have no paired upstream PR. Their provenance " + "records an explicit reason and `refresh_policy: retain`, including commits " + "outside the lock-selected scope when applicable. A future refresh must " + "retain these commits until an upstream pairing or equivalence " + "is explicitly established.\n\n" + ) + return ( + "@coderabbitai summary\n\n## Description\n\n" + f"Promote the exact source revision reviewed in {plan.record['consumer_pr']}.\n" + "Only the selected vendor lock URL/branch changes. The source SHA, selected vendor files, " + "compatibility patch, and digests are unchanged.\n\n" + f"{unpaired}" + "## Test Coverage\n\nSource materialization and lock-only diff verified by the promotion tool. " + "No new kernel behavior. The tool posts " + f"`{_CI_SKIP_COMMAND}` to request the bot's no-code-change CI path. " + "Automatic GitHub checks and required reviews still apply.\n\n" + "## PR Checklist\n\n" + "- [x] No runtime/API/dependency changes.\n" + "- [x] Immutable source pin and vendor content preserved.\n" + "- [x] Please check this after reviewing the above items as appropriate for this PR.\n\n" + "## Required squash commit message\n\n" + "Preserve the following message, including the provenance block and DCO sign-off.\n\n" + f"```text\n{message.strip()}\n```\n" + ) + + +def _ensure_ci_skip(gh: GitHub, plan: Promotion, pr: dict) -> dict: + """Request the no-code-change bot path once across sequential retries.""" + current = gh.get(f"repos/{gh.consumer_repo}/pulls/{pr['number']}") + if current.get("merged"): + _verify_remote_commit(gh, plan, gh.consumer_repo, current["merge_commit_sha"]) + _report_merged(current) + return current + if current["state"] != "open" or current["head"]["sha"] != pr["head"]["sha"]: + raise ValueError("Follow-up PR closed or changed before the CI request; inspect it.") + _check_current(gh, plan) + _verify_remote_commit(gh, plan, gh.consumer_repo, current["head"]["sha"]) + actor = gh.get("user")["login"] + endpoint = f"repos/{gh.consumer_repo}/issues/{pr['number']}/comments" + page = 1 + while True: + comments = gh.api(f"{endpoint}?per_page=100&page={page}") + if not isinstance(comments, list): + raise ValueError("Unexpected issue-comment response; refusing a duplicate CI request.") + if any( + comment.get("body", "").strip() == _CI_SKIP_COMMAND + and comment.get("user", {}).get("login") == actor + for comment in comments + ): + print(f"Existing CI skip request found: {current['html_url']}", flush=True) + return current + if len(comments) < 100: + break + page += 1 + comment = gh.api(endpoint, method="POST", payload={"body": _CI_SKIP_COMMAND}) + print(f"Posted CI skip request: {comment['html_url']}", flush=True) + return current + + +def _publish(args: argparse.Namespace, gh: GitHub, plan: Promotion) -> dict: + if plan.completed_pr is not None: + _report_merged(plan.completed_pr) + return plan.completed_pr + existing = _find_followup(gh, plan) + remote = gh.optional(f"repos/{plan.fork}/git/ref/heads/{plan.branch}") + if existing: + existing = gh.get(f"repos/{gh.consumer_repo}/pulls/{existing['number']}") + sha = existing["merge_commit_sha"] if existing.get("merged") else existing["head"]["sha"] + message = _verify_remote_commit(gh, plan, gh.consumer_repo, sha) + if existing.get("merged"): + _report_merged(existing) + return existing + if remote is None or remote["object"]["sha"] != sha: + raise ValueError("Follow-up branch is missing or changed; inspect before resuming.") + elif remote: + sha = remote["object"]["sha"] + message = _verify_remote_commit(gh, plan, plan.fork, sha) + else: + message = "" + if args.auto_merge: + _check_auto_merge(gh) + _check_current(gh, plan) + with _source_repo(args, plan) as source: + if remote is None: + worktree, sha = _prepare_commit(args, plan, source) + message = _git(worktree, "log", "-1", "--format=%B") + _check_current(gh, plan) + # Persist provenance remotely before changing the canonical source branch. + if remote is None: + print(f"Publishing follow-up branch {plan.fork}:{plan.branch}...", flush=True) + _git( + worktree, + "push", + f"https://github.com/{plan.fork}.git", + f"{sha}:refs/heads/{plan.branch}", + ) + _check_current(gh, plan) + # No force push: Git also enforces the fast-forward constraint remotely. + print(f"Ensuring canonical branch points to {plan.reviewed.commit}...", flush=True) + _git( + source, + "push", + plan.previous.url, + f"{plan.reviewed.commit}:refs/heads/{plan.previous.branch}", + ) + print(f"Canonical branch now contains reviewed revision {plan.reviewed.commit}.") + _check_current(gh, plan) + if existing is None: + print("Creating the lock-only follow-up PR...", flush=True) + existing = gh.api( + f"repos/{gh.consumer_repo}/pulls", + method="POST", + payload={ + "title": plan.title, + "body": _pr_body(plan, message), + "base": plan.base_branch, + "head": f"{plan.fork.split('/')[0]}:{plan.branch}", + "head_repo": plan.fork.split("/")[1], + }, + ) + print(f"Lock-only promotion PR: {existing['html_url']}") + existing = _ensure_ci_skip(gh, plan, existing) + if existing.get("merged"): + return existing + if args.auto_merge: + _arm_auto_merge(gh, plan, existing, message) + if args.wait: + deadline = time.monotonic() + args.timeout + while True: + current = gh.get(f"repos/{gh.consumer_repo}/pulls/{existing['number']}") + if current.get("merged"): + _verify_remote_commit(gh, plan, gh.consumer_repo, current["merge_commit_sha"]) + _report_merged(current) + break + if current["state"] == "closed" or current["head"]["sha"] != existing["head"]["sha"]: + if args.auto_merge and current.get("auto_merge"): + _disable_auto_merge(gh, current) + raise ValueError( + "Follow-up PR closed or changed while waiting; inspect before proceeding." + ) + if time.monotonic() >= deadline: + raise TimeoutError( + "Promotion PR is still pending; rerun promote to verify after merge." + ) + time.sleep(min(15, max(0, deadline - time.monotonic()))) + return existing + + +def main(argv: list[str] | None = None) -> int: + """Run the maintainer CLI; dry-run unless publication is explicitly requested.""" + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + promote = commands.add_parser("promote", help="Promote a merged consumer source update.") + promote.add_argument("--source-pr", required=True) + promote.add_argument("--vendor", required=True, help="Vendor key in the lock file.") + promote.add_argument("--consumer-repo", default="NVIDIA/TensorRT-LLM") + promote.add_argument("--base-branch", default="main") + promote.add_argument("--upstream-repo", required=True, help="Upstream OWNER/REPO.") + promote.add_argument("--upstream-branch", default="main") + promote.add_argument( + "--canonical-repo", + required=True, + help="Explicit canonical source fork OWNER/REPO (not the developer's temporary fork).", + ) + promote.add_argument( + "--canonical-branch", + required=True, + help="Expected canonical branch; specify a dated branch when applicable.", + ) + provenance = promote.add_mutually_exclusive_group(required=True) + provenance.add_argument("--upstream-pr", action="append", default=[]) + provenance.add_argument( + "--from-pr-description", + action="store_true", + help="Read contributor metadata and automatically match upstream commits.", + ) + provenance.add_argument( + "--unpaired-reason", + help="Explicit exception for changes with no upstream PR; retain them during refresh.", + ) + promote.add_argument("--map-upstream", action="append", default=[], metavar="COMMIT=PR") + promote.add_argument( + "--auto-match", + action="store_true", + help="Match --upstream-pr commits automatically instead of prompting.", + ) + promote.add_argument("--repo", type=Path, default=Path(__file__).resolve().parents[2]) + promote.add_argument( + "--fork", help="Consumer publishing fork OWNER/REPO; defaults to remote 'fork'." + ) + promote.add_argument( + "--source-repo", + type=Path, + help="Local source repository containing both immutable SHAs.", + ) + promote.add_argument("--worktree", type=Path, help="New/preserved follow-up worktree path.") + promote.add_argument( + "--publish", + action="store_true", + help="Publish the lock-only PR and post the no-code-change /bot skip request.", + ) + promote.add_argument("--auto-merge", action="store_true") + promote.add_argument( + "--wait", action="store_true", help="Wait for merge and verify final commit metadata." + ) + promote.add_argument( + "--timeout", type=int, default=3600, help="Maximum --wait duration in seconds." + ) + args = parser.parse_args(argv) + if (args.auto_merge or args.wait) and not args.publish: + parser.error("--auto-merge and --wait require --publish") + if args.timeout <= 0: + parser.error("--timeout must be positive") + args.repo = args.repo.resolve() + try: + if not vendor._NAME_PATTERN.fullmatch(args.vendor): + raise ValueError("Invalid vendor key.") + _repo_name(f"https://github.com/{args.upstream_repo}") + gh = GitHub( + args.consumer_repo, + args.vendor, + args.upstream_repo, + args.base_branch, + args.upstream_branch, + ) + resolutions = None + if args.from_pr_description: + if __package__: + from . import metadata + else: + import metadata + number = _pr_number(args.source_pr, gh.consumer_repo) + pr = gh.get(f"repos/{gh.consumer_repo}/pulls/{number}") + entry = metadata.parse(pr.get("body") or "", gh.vendor_name, gh.upstream_repo) + args.upstream_pr = [str(number) for number in entry["upstream_prs"]] + args.unpaired_reason = entry["unpaired_reason"] + resolutions = entry["resolutions"] + if args.map_upstream and (args.auto_match or args.from_pr_description): + raise ValueError("Explicit --map-upstream cannot accompany automatic matching.") + plan = _make_plan( + args, + gh, + auto_match=args.auto_match or args.from_pr_description, + resolutions=resolutions, + ) + mode = ( + "ALREADY COMPLETED" if plan.completed_pr else "PUBLISH" if args.publish else "DRY RUN" + ) + print(f"{mode}: {plan.canonical_repo}:{plan.previous.branch}", flush=True) + print(f"Source: {plan.previous.commit} -> {plan.reviewed.commit}") + print(f"Follow-up: {plan.fork}:{plan.branch}") + print(_record_text(plan.record)) + if args.publish: + _publish(args, gh, plan) + elif plan.completed_pr is not None: + _report_merged(plan.completed_pr) + else: + print(f"On publication: post {_CI_SKIP_COMMAND}") + print(f"Squash auto-merge: {'enabled' if args.auto_merge else 'off'}") + print("No remote writes or consumer worktree changes. Use --publish to proceed.") + return 0 + except ( + RuntimeError, + ValueError, + OSError, + KeyError, + subprocess.TimeoutExpired, + vendor.VendorError, + ) as error: + print(f"Promotion stopped: {error}", file=sys.stderr) + print( + "Any completed pushes/PRs are preserved. Inspect the error before retrying.", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/unittest/others/test_maintain_prims_ts.py b/tests/unittest/others/test_maintain_prims_ts.py new file mode 100644 index 000000000000..3751df9bced9 --- /dev/null +++ b/tests/unittest/others/test_maintain_prims_ts.py @@ -0,0 +1,1521 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Promotion lifecycle tests: real local Git, fake GitHub, no network or GPUs.""" + +from __future__ import annotations + +import argparse +import base64 +import contextlib +import copy +import dataclasses +import importlib +import importlib.util +import runpy +import subprocess +import sys +from pathlib import Path + +import pytest + +# tests/unittest/scripts shadows the repository's scripts namespace. Load the +# standalone tool as an isolated package, without editing sys.path. +_SCRIPTS = Path(__file__).resolve().parents[3] / "scripts" / "vendor" +_SPEC = importlib.util.spec_from_file_location( + "prims_ts_maintenance_test", + _SCRIPTS / "promote.py", + submodule_search_locations=[str(_SCRIPTS)], +) +m = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = m +_SPEC.loader.exec_module(m) +sys.modules[f"{_SPEC.name}.promote"] = m +b = importlib.import_module(f"{_SPEC.name}.bot") + +_VENDOR = "example-vendor" +_CONSUMER = "NVIDIA/TensorRT-LLM" +_UPSTREAM = "flashinfer-ai/flashinfer" + +pytestmark = pytest.mark.cpu_only + + +def _git( + repo: Path, *args: str, text: str | None = None, allowed_returncodes: tuple[int, ...] = (0,) +) -> str: + result = subprocess.run( + ["git", "-C", str(repo), *args], input=text, text=True, capture_output=True, timeout=30 + ) + assert result.returncode in allowed_returncodes, result.stderr + return result.stdout.strip() + + +def _init(path: Path) -> Path: + path.mkdir() + _git(path, "init", "-q", "-b", "main") + _git(path, "config", "user.name", "Promotion Test") + _git(path, "config", "user.email", "promotion@example.invalid") + _git(path, "config", "core.hooksPath", str(path / "empty-hooks")) + return path + + +def _write(repo: Path, name: str, content: str) -> None: + file = repo / name + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text(content) + + +def _commit(repo: Path, message: str) -> str: + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-s", "-m", message) + return _git(repo, "rev-parse", "HEAD") + + +class World(m.GitHub): + """GitHub facade whose immutable data comes from temporary Git repositories.""" + + def __init__(self, root: Path) -> None: + super().__init__(_CONSUMER, _VENDOR, _UPSTREAM) + self.root = root + self.source = _init(root / "flashinfer") + _write(self.source, "prims_ts/kernel.py", "VALUE = 0\n") + self.upstream = _commit(self.source, "upstream baseline") + _write(self.source, "prims_ts/kernel.py", "VALUE = 1\n") + self.previous = _commit(self.source, "previous downstream fix") + _write(self.source, "prims_ts/kernel.py", "VALUE = 2\n") + self.first = _commit(self.source, "first fix") + _write(self.source, "prims_ts/kernel.py", "VALUE = 3\n") + self.reviewed = _commit(self.source, "second fix") + self.consumer = _init(root / "trtllm") + assert ( + m.vendor.main( + [ + "--lock", + str(self.consumer / m._LOCK), + "create", + _VENDOR, + "--url", + "https://github.com/maintainer/flashinfer.git", + "--branch", + "trtllm-prims-ts-dev", + "--commit", + self.previous, + "--source", + "prims_ts", + "--destination", + "vendored/prims_ts", + "--include", + "**/*.py", + "--repo", + str(self.source), + ] + ) + == 0 + ) + self.base = _commit(self.consumer, "previous vendor pin") + _write(self.consumer, "vendored/prims_ts/kernel.py", "VALUE = 3\n") + assert ( + m.vendor.main( + [ + "--lock", + str(self.consumer / m._LOCK), + "pin", + _VENDOR, + "--url", + "https://github.com/dev/flashinfer.git", + "--branch", + "tmp-fix", + "--commit", + self.reviewed, + "--repo", + str(self.source), + ] + ) + == 0 + ) + self.merged = _commit(self.consumer, "merged source update") + self.main = self.merged + self.canonical = root / "canonical.git" + self.fork = root / "trtllm-fork.git" + _git(root, "clone", "-q", "--bare", str(self.source), str(self.canonical)) + _git(root, "clone", "-q", "--bare", str(self.consumer), str(self.fork)) + _git(self.canonical, "update-ref", "refs/heads/trtllm-prims-ts-dev", self.previous) + _git( + self.consumer, "remote", "add", "fork", "https://github.com/maintainer/TensorRT-LLM.git" + ) + self.prs: dict[int, dict] = {} + self.calls: list[tuple[str, str, dict | None]] = [] + self.overrides: dict[str, dict | list] = {} + self.state = "BLOCKED" + self.tamper_auto = False + self.fail_create = False + self.comments: list[dict] = [] + self.fail_comment_before = False + self.fail_comment_after = False + self.git_calls: list[tuple[str, ...]] = [] + self.source_comments: list[dict] = [] + self.source_body = b.metadata.template(_VENDOR, _UPSTREAM).replace( + "/pull/123", "/pull/4829" + ) + + def args(self, **overrides: object) -> argparse.Namespace: + values = dict( + source_pr="17", + canonical_repo="maintainer/flashinfer", + canonical_branch="trtllm-prims-ts-dev", + upstream_pr=["4829"], + unpaired_reason=None, + map_upstream=[], + repo=self.consumer, + fork=None, + source_repo=self.source, + worktree=self.root / "followup", + publish=False, + auto_merge=False, + wait=False, + timeout=30, + ) + values.update(overrides) + return argparse.Namespace(**values) + + def commit_data(self, repo: Path, sha: str) -> dict: + return { + "sha": sha, + "parents": [{"sha": s} for s in _git(repo, "show", "-s", "--format=%P", sha).split()], + "commit": {"message": _git(repo, "show", "-s", "--format=%B", sha)}, + "files": [ + {"filename": f} + for f in _git( + repo, "diff-tree", "--no-commit-id", "--name-only", "-r", sha + ).splitlines() + ], + } + + def compare(self, repo: str, base: str, head: str, *, all_commits: bool = False) -> dict: + source = self.consumer if repo == _CONSUMER else self.source + commits = _git(source, "rev-list", "--reverse", f"{base}..{head}").splitlines() + return { + "merge_base_commit": {"sha": _git(source, "merge-base", base, head)}, + "commits": [self.commit_data(source, sha) for sha in commits], + "total_commits": len(commits), + } + + def routed_git( + self, + repo: Path, + *args: str, + text: str | None = None, + allowed_returncodes: tuple[int, ...] = (0,), + ) -> str: + self.git_calls.append(args) + replacements = { + "https://github.com/NVIDIA/TensorRT-LLM.git": str(self.consumer), + "https://github.com/maintainer/TensorRT-LLM.git": str(self.fork), + "https://github.com/maintainer/flashinfer.git": str(self.canonical), + "https://github.com/dev/flashinfer.git": str(self.source), + "https://github.com/flashinfer-ai/flashinfer.git": str(self.source), + } + if args[:3] == ("remote", "get-url", "fork"): + return "https://github.com/maintainer/TensorRT-LLM.git" + return _git( + repo, + *[replacements.get(arg, arg) for arg in args], + text=text, + allowed_returncodes=allowed_returncodes, + ) + + def api( + self, endpoint: str, *, method: str = "GET", payload: dict | None = None + ) -> dict | list: + self.calls.append((method, endpoint, payload)) + if method == "GET" and endpoint in self.overrides: + return copy.deepcopy(self.overrides[endpoint]) + if endpoint == "user": + return {"login": "maintainer"} + if endpoint == "graphql": + query = payload["query"] + pr = self.prs[99] + if query.startswith("query"): + return { + "data": { + "node": {"headRefOid": pr["head"]["sha"], "mergeStateStatus": self.state} + } + } + if "disablePullRequestAutoMerge" in query: + pr["auto_merge"] = None + return {"data": {}} + data = payload["variables"]["input"] + assert data["expectedHeadOid"] == pr["head"]["sha"] + if "enablePullRequestAutoMerge" in query: + pr["auto_merge"] = { + "merge_method": "squash", + "commit_title": data["commitHeadline"], + "commit_message": "lost metadata" if self.tamper_auto else data["commitBody"], + } + else: + message = f"{data['commitHeadline']}\n\n{data['commitBody']}\n" + tree = _git(self.consumer, "rev-parse", f"{pr['head']['sha']}^{{tree}}") + sha = _git(self.consumer, "commit-tree", tree, "-p", self.main, text=message) + self.main = sha + pr.update(merged=True, merged_at="today", state="closed", merge_commit_sha=sha) + return {"data": {}} + prefix = "repos/" + assert endpoint.startswith(prefix), endpoint + owner, name, *rest = endpoint[len(prefix) :].split("/") + repository = f"{owner}/{name}" + route = "/".join(rest) + if route.startswith("issues/17/comments"): + if method == "GET": + return copy.deepcopy(self.source_comments) + comment = { + "id": len(self.source_comments) + 1, + "body": payload["body"], + "user": {"login": "maintainer"}, + } + self.source_comments.append(comment) + return copy.deepcopy(comment) + if route.startswith("issues/99/comments"): + if method == "GET": + page = int(route.rsplit("page=", 1)[1]) + return copy.deepcopy(self.comments[(page - 1) * 100 : page * 100]) + if self.fail_comment_before: + raise RuntimeError("simulated comment failure before posting") + comment = { + "body": payload["body"], + "user": {"login": "maintainer"}, + "html_url": f"https://github.com/NVIDIA/TensorRT-LLM/pull/99#issuecomment-{len(self.comments) + 1}", + } + self.comments.append(comment) + if self.fail_comment_after: + raise RuntimeError("simulated comment failure after posting") + return copy.deepcopy(comment) + if not route: + root = _UPSTREAM if name == "flashinfer" else _CONSUMER + return { + "source": {"full_name": root}, + "allow_auto_merge": True, + "allow_squash_merge": True, + } + if route == "rules/branches/main": + return [{"type": "required_status_checks"}] + if route.startswith("git/ref/heads/"): + branch = route[len("git/ref/heads/") :] + if repository == _CONSUMER: + return {"object": {"sha": self.main}} + if repository == _UPSTREAM: + return {"object": {"sha": self.upstream}} + remote = self.canonical if name == "flashinfer" else self.fork + result = subprocess.run( + ["git", "-C", str(remote), "rev-parse", "--verify", f"refs/heads/{branch}"], + capture_output=True, + text=True, + ) + if result.returncode: + raise RuntimeError("HTTP 404") + return {"object": {"sha": result.stdout.strip()}} + if route.startswith("contents/"): + path, sha = route[len("contents/") :].split("?ref=") + content = _git(self.consumer, "show", f"{sha}:{path}") + "\n" + return {"encoding": "base64", "content": base64.b64encode(content.encode()).decode()} + if route.startswith("commits/"): + return self.commit_data(self.consumer, route.split("/")[1]) + if route.startswith("pulls?"): + if "head=" not in route: + if "state=open" in route: + return copy.deepcopy([pr for pr in self.prs.values() if pr["state"] == "open"]) + return [ + self.get(f"repos/{_CONSUMER}/pulls/17"), + *copy.deepcopy(list(self.prs.values())), + ] + return copy.deepcopy(list(self.prs.values())) + if route.startswith("pulls/") and "/files?" in route: + return [{"filename": m._LOCK}] + if route == "pulls" and method == "POST": + if self.fail_create: + raise RuntimeError("simulated PR creation failure") + branch = payload["head"].split(":")[1] + head = _git(self.fork, "rev-parse", f"refs/heads/{branch}") + self.prs[99] = dict( + number=99, + node_id="PR_test", + state="open", + merged=False, + user={"login": "maintainer"}, + head={"sha": head}, + base={"ref": "main"}, + updated_at="2020-01-01T00:00:00Z", + html_url="https://github.com/NVIDIA/TensorRT-LLM/pull/99", + body=payload["body"], + auto_merge=None, + ) + return copy.deepcopy(self.prs[99]) + if route.startswith("pulls/"): + number = int(route.split("/")[1]) + if repository == _UPSTREAM: + return { + "number": number, + "base": {"ref": "main", "sha": self.upstream}, + "head": {"sha": self.reviewed}, + "state": "open", + "merged": False, + "draft": True, + } + if number == 17: + return { + "number": 17, + "merged": True, + "state": "closed", + "merged_at": "2020-01-01T00:00:00Z", + "updated_at": "2020-01-01T00:00:00Z", + "user": {"login": "author"}, + "body": self.source_body, + "base": {"ref": "main"}, + "merge_commit_sha": self.merged, + "head": {"sha": "a" * 40}, + } + return copy.deepcopy(self.prs[number]) + raise AssertionError((method, endpoint, payload)) + + +@pytest.fixture +def world(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> World: + instance = World(tmp_path) + monkeypatch.setattr(m, "_git", instance.routed_git) + monkeypatch.setattr(m, "GitHub", lambda *args: instance) + return instance + + +def test_dry_run_has_no_writes_and_pins_merged_not_live_head( + world: World, capfd: pytest.CaptureFixture +) -> None: + assert ( + m.main( + [ + "promote", + "--vendor", + _VENDOR, + "--upstream-repo", + _UPSTREAM, + "--canonical-branch", + "trtllm-prims-ts-dev", + "--source-pr", + "17", + "--canonical-repo", + "maintainer/flashinfer", + "--upstream-pr", + "4829", + "--repo", + str(world.consumer), + ] + ) + == 0 + ) + assert "DRY RUN" in capfd.readouterr().out + assert all(method == "GET" for method, _, _ in world.calls) + assert not any(args[0] in ("push", "commit", "worktree") for args in world.git_calls) + plan = m._make_plan(world.args(), world) + assert plan.reviewed.commit == world.reviewed + assert plan.record["changes"][0]["commits"] == [world.first, world.reviewed] + assert plan.record["upstream_base"] == world.upstream + + +def test_publish_signed_lock_only_and_repeat(world: World) -> None: + args = world.args(publish=True) + plan = m._make_plan(args, world) + pr = m._publish(args, world, plan) + assert _git(world.canonical, "rev-parse", "trtllm-prims-ts-dev") == world.reviewed + head = pr["head"]["sha"] + assert _git(world.consumer, "diff", "--name-only", f"{head}^", head) == m._LOCK + assert m._lock_at(world, _CONSUMER, head).to_mapping() == plan.promoted + message = _git(world.consumer, "show", "-s", "--format=%B", head) + assert m._record_from_message(message) == plan.record + assert "Signed-off-by: Promotion Test " in message + assert not any("graphql" == endpoint for _, endpoint, _ in world.calls) + m._publish(args, world, m._make_plan(args, world)) + assert ( + sum(method == "POST" and endpoint.endswith("/pulls") for method, endpoint, _ in world.calls) + == 1 + ) + + +@pytest.mark.parametrize("state", ["BLOCKED", "CLEAN"]) +def test_auto_merge_message_and_immediate_merge(world: World, state: str) -> None: + world.state = state + args = world.args(publish=True, auto_merge=True) + plan = m._make_plan(args, world) + m._publish(args, world, plan) + if state == "CLEAN": + assert world.prs[99]["merged"] + m._verify_remote_commit(world, plan, _CONSUMER, world.main) + m._publish(args, world, m._make_plan(args, world)) + else: + saved = world.prs[99]["auto_merge"] + assert saved["merge_method"] == "squash" + assert m._record_from_message(saved["commit_message"]) == plan.record + assert "Signed-off-by:" in saved["commit_message"] + + +def test_resume_after_upstream_head_moves_retains_snapshot(world: World) -> None: + args = world.args(publish=True) + original = m._make_plan(args, world) + m._publish(args, world, original) + changed = world.get(f"repos/{_UPSTREAM}/pulls/4829") + changed["head"]["sha"] = "b" * 40 + world.overrides[f"repos/{_UPSTREAM}/pulls/4829"] = changed + resumed = m._make_plan(args, world) + assert resumed.record != original.record + m._publish(args, world, resumed) + assert resumed.record == original.record + + +def test_bot_attribution_survives_retry_and_squash_merge(world: World) -> None: + args = world.args(publish=True, auto_merge=True) + original = m._make_plan(args, world) + provenance = { + "attribution": { + world.first: {"method": "same-commit"}, + world.reviewed: { + "method": "author-assertion", + "reason": "downstream adaptation", + "refresh_policy": "retain-until-verified", + }, + }, + "contributor_metadata": { + "source_head": "a" * 40, + "metadata_digest": "b" * 64, + "pr_author": "contributor", + }, + } + original.record.update(provenance) + m._publish(args, world, original) + resumed = m._make_plan(args, world) + resumed.record.update(provenance) + m._publish(args, world, resumed) + # Model GitHub completing the already-armed auto-merge asynchronously. + followup = world.prs[99] + saved = followup["auto_merge"] + tree = _git(world.consumer, "rev-parse", f"{followup['head']['sha']}^{{tree}}") + world.main = _git( + world.consumer, + "commit-tree", + tree, + "-p", + world.main, + text=f"{saved['commit_title']}\n\n{saved['commit_message']}\n", + ) + followup.update(merged=True, merged_at="today", state="closed", merge_commit_sha=world.main) + completed = m._make_plan(args, world) + assert completed.completed_pr["merged"] + assert completed.record == original.record + + +def test_scoped_attribution_preserves_excluded_tail_during_promotion_and_recovery( + world: World, +) -> None: + upstream = world.get(f"repos/{_UPSTREAM}/pulls/4829") + paired_head = world.reviewed + _write(world.source, "tests/test_kernel.py", "# downstream tests only\n") + world.reviewed = _commit(world.source, "excluded test changes") + assert ( + m.vendor.main( + [ + "--lock", + str(world.consumer / m._LOCK), + "pin", + _VENDOR, + "--commit", + world.reviewed, + "--repo", + str(world.source), + ] + ) + == 0 + ) + _git(world.consumer, "add", "-A") + _git(world.consumer, "commit", "-q", "--amend", "-s", "-m", "include tests-only source commit") + world.merged = _git(world.consumer, "rev-parse", "HEAD") + world.main = world.merged + world.overrides[f"repos/{_UPSTREAM}/pulls/4829"] = upstream + args = world.args(publish=True, auto_merge=True) + plan = m._make_plan(args, world, auto_match=True) + assert plan.record["attribution_scope"] == {"source": "prims_ts", "include": ["**/*.py"]} + assert plan.record["attribution"][world.reviewed] == { + "method": "outside-vendor-scope", + "refresh_policy": "retain", + } + unpaired, paired = plan.record["changes"] + assert unpaired["commits"] == [world.reviewed] + assert unpaired["upstream_pr"] is None + assert unpaired["refresh_policy"] == "retain" + assert paired["commits"] == [world.first, paired_head] + assert paired["upstream_pr"] == f"https://github.com/{_UPSTREAM}/pull/4829" + m._validate_snapshot(plan.record, _UPSTREAM) + m._publish(args, world, plan) + saved = world.prs[99]["auto_merge"] + assert m._record_from_message(saved["commit_message"]) == plan.record + assert "outside the lock-selected scope" in world.prs[99]["body"] + assert _git(world.canonical, "rev-parse", "trtllm-prims-ts-dev") == world.reviewed + assert not (args.worktree / "vendored/prims_ts/test_kernel.py").exists() + # Recovery must use the durable, lock-bound scope, not mutable PR metadata. + upstream["head"]["sha"] = "b" * 40 + recovered = m._plan_identity(args, world) + assert m._recover_plan(args, world, recovered) + assert recovered.record == plan.record + m._publish(args, world, recovered) + assert len(world.prs) == 1 + + +def test_similarity_provenance_survives_publication_and_recovery(world: World) -> None: + original_head = world.reviewed + additions = [f"VALUE_{index} = {index}\n" for index in range(16)] + _write(world.source, "prims_ts/kernel.py", "VALUE = 3\n" + "".join(additions)) + world.reviewed = _commit(world.source, "downstream source update") + _git(world.source, "checkout", "-q", "-b", "upstream-adapted", original_head) + _write( + world.source, "prims_ts/kernel.py", "VALUE = 3\n" + "".join(additions[:-1]) + "adapted\n" + ) + upstream_head = _commit(world.source, "upstream equivalent with adaptation") + _git(world.source, "checkout", "-q", "main") + upstream = world.get(f"repos/{_UPSTREAM}/pulls/4829") + upstream["head"]["sha"] = upstream_head + world.overrides[f"repos/{_UPSTREAM}/pulls/4829"] = upstream + _write(world.consumer, "vendored/prims_ts/kernel.py", "VALUE = 3\n" + "".join(additions)) + assert ( + m.vendor.main( + [ + "--lock", + str(world.consumer / m._LOCK), + "pin", + _VENDOR, + "--commit", + world.reviewed, + "--repo", + str(world.source), + ] + ) + == 0 + ) + _git(world.consumer, "add", "-A") + _git(world.consumer, "commit", "-q", "--amend", "-s", "-m", "merged source update") + world.merged = _git(world.consumer, "rev-parse", "HEAD") + world.main = world.merged + args = world.args(publish=True, auto_merge=True) + plan = m._make_plan(args, world, auto_match=True) + evidence = plan.record["attribution"][world.reviewed] + assert evidence["method"] == "similar-edits" + assert evidence["similarity"] == 0.95 + assert evidence["similarity_threshold"] == 0.9 + assert evidence["refresh_policy"] == "retain-until-verified" + assert evidence["comparison"]["upstream_head"] == upstream_head + m._publish(args, world, plan) + saved = world.prs[99]["auto_merge"] + assert m._record_from_message(saved["commit_message"]) == plan.record + upstream["head"]["sha"] = "b" * 40 + recovered = m._plan_identity(args, world) + assert m._recover_plan(args, world, recovered) + assert recovered.record == plan.record + m._publish(args, world, recovered) + assert len(world.prs) == 1 + + +@pytest.mark.parametrize( + "scope", + [ + {"source": "different", "include": ["**/*.py"]}, + {"source": "prims_ts", "include": ["*.py"]}, + {"source": "prims_ts"}, + ], +) +def test_recovery_rejects_mismatched_attribution_scope(world: World, scope: dict) -> None: + args = world.args() + record = m._make_plan(args, world).record + record["attribution_scope"] = scope + with pytest.raises(ValueError, match="attribution scope"): + m._adopt_record(m._plan_identity(args, world), m._record_text(record)) + + +def test_non_main_consumer_and_upstream_branches(world: World) -> None: + world.base_branch = "release/next" + world.upstream_branch = "develop" + source = world.get(f"repos/{_CONSUMER}/pulls/17") + source["base"]["ref"] = world.base_branch + upstream = world.get(f"repos/{_UPSTREAM}/pulls/4829") + upstream["base"]["ref"] = world.upstream_branch + world.overrides[f"repos/{_CONSUMER}/pulls/17"] = source + world.overrides[f"repos/{_UPSTREAM}/pulls/4829"] = upstream + world.overrides[f"repos/{_CONSUMER}/rules/branches/{world.base_branch}"] = [] + args = world.args(publish=True, auto_merge=True) + plan = m._make_plan(args, world) + m._publish(args, world, plan) + assert plan.record["consumer_base"] == "release/next" + assert plan.record["upstream_branch"] == "develop" + creates = [ + payload + for method, endpoint, payload in world.calls + if method == "POST" and endpoint.endswith("/pulls") + ] + assert creates[0]["base"] == "release/next" + + +@pytest.mark.parametrize( + "content", + ["vendors: [", "schema_version: 1\nvendors: &v {toy: *v}", "schema_version: 1\nvendors: []"], +) +def test_malformed_remote_lock_is_actionable(world: World, content: str) -> None: + world.overrides[f"repos/{_CONSUMER}/contents/{m._LOCK}?ref={world.merged}"] = { + "encoding": "base64", + "content": base64.b64encode(content.encode()).decode(), + } + with pytest.raises(ValueError): + m._lock_document_at(world, _CONSUMER, world.merged) + + +def test_remote_lock_accepts_github_wrapped_base64(world: World) -> None: + endpoint = f"repos/{_CONSUMER}/contents/{m._LOCK}?ref={world.merged}" + response = world.get(endpoint) + original = m._lock_document_at(world, _CONSUMER, world.merged) + encoded = response["content"] + response["content"] = "\n".join(encoded[i : i + 60] for i in range(0, len(encoded), 60)) + "\n" + world.overrides[endpoint] = response + assert m._lock_document_at(world, _CONSUMER, world.merged) == original + + +def test_resume_after_push_before_pr_creation(world: World) -> None: + args = world.args(publish=True) + world.fail_create = True + with pytest.raises(RuntimeError, match="creation failure"): + m._publish(args, world, m._make_plan(args, world)) + assert _git(world.canonical, "rev-parse", "trtllm-prims-ts-dev") == world.reviewed + world.fail_create = False + m._publish(args, world, m._make_plan(args, world)) + assert len(world.prs) == 1 + + +def test_auto_merge_tampering_disables_request(world: World) -> None: + world.tamper_auto = True + args = world.args(publish=True, auto_merge=True) + with pytest.raises(ValueError, match="verification failed"): + m._publish(args, world, m._make_plan(args, world)) + assert world.prs[99]["auto_merge"] is None + + +@pytest.mark.parametrize("field,value", [("merged", False), ("base", {"ref": "release"})]) +def test_unmerged_or_wrong_base_rejected(world: World, field: str, value: object) -> None: + endpoint = f"repos/{_CONSUMER}/pulls/17" + changed = world.get(endpoint) + changed[field] = value + world.overrides[endpoint] = changed + with pytest.raises(ValueError, match="already merged"): + m._make_plan(world.args(), world) + + +@pytest.mark.parametrize("target", ["main", "canonical"]) +def test_concurrent_updates_block_before_publication(world: World, target: str) -> None: + plan = m._make_plan(world.args(), world) + if target == "main": + world.main = world.base + else: + _git(world.canonical, "update-ref", "refs/heads/trtllm-prims-ts-dev", world.first) + with pytest.raises(ValueError, match="superseded|moved unexpectedly"): + m._publish(world.args(publish=True), world, plan) + assert not any(method != "GET" for method, _, _ in world.calls) + assert not any(args[0] == "push" for args in world.git_calls) + + +def test_divergent_source_rejected(world: World, monkeypatch: pytest.MonkeyPatch) -> None: + original = world.compare + + def divergent(*args: object, **kwargs: object) -> dict: + result = original(*args, **kwargs) + result["merge_base_commit"]["sha"] = world.upstream + return result + + monkeypatch.setattr(world, "compare", divergent) + with pytest.raises(ValueError, match="not a fast-forward"): + m._make_plan(world.args(), world) + + +def test_upstream_closed_unmerged_rejected(world: World) -> None: + endpoint = f"repos/{_UPSTREAM}/pulls/4829" + changed = world.get(endpoint) + changed.update(state="closed", merged=False) + world.overrides[endpoint] = changed + with pytest.raises(ValueError, match="Upstream PR"): + m._make_plan(world.args(), world) + + +@pytest.mark.parametrize("setting", ["auto", "squash", "queue"]) +def test_unsupported_auto_merge_settings_fail_before_push(world: World, setting: str) -> None: + if setting == "queue": + world.overrides[f"repos/{_CONSUMER}/rules/branches/main"] = [{"type": "merge_queue"}] + else: + data = world.get(f"repos/{_CONSUMER}") + data[f"allow_{'auto_merge' if setting == 'auto' else 'squash_merge'}"] = False + world.overrides[f"repos/{_CONSUMER}"] = data + args = world.args(publish=True, auto_merge=True) + with pytest.raises(ValueError, match="allow|Merge queues"): + m._publish(args, world, m._make_plan(args, world)) + assert not any(args[0] == "push" for args in world.git_calls) + + +def test_multiple_upstream_pr_mapping(world: World, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(m.sys.stdin, "isatty", lambda: False) + args = world.args(upstream_pr=["4829", "4830"]) + with pytest.raises(ValueError, match="--map-upstream"): + m._make_plan(args, world) + args.map_upstream = [f"{world.first}=4829", f"{world.reviewed}=4830"] + plan = m._make_plan(args, world) + assert [group["commits"] for group in plan.record["changes"]] == [ + [world.first], + [world.reviewed], + ] + + +@pytest.mark.parametrize("mapping", ["1234567=4829", "bad=4829", "abcdefg=4829"]) +def test_bad_mapping_rejected(world: World, mapping: str) -> None: + with pytest.raises(ValueError, match="invalid"): + m._make_plan(world.args(map_upstream=[mapping]), world) + + +def test_modified_vendor_files_block_before_push( + world: World, monkeypatch: pytest.MonkeyPatch +) -> None: + original = world.routed_git + + def tamper(repo: Path, *args: str, text: str | None = None) -> str: + result = original(repo, *args, text=text) + if args[:2] == ("worktree", "add"): + _write(world.root / "followup", "vendored/prims_ts/kernel.py", "UNREVIEWED = True\n") + return result + + monkeypatch.setattr(m, "_git", tamper) + args = world.args(publish=True) + with pytest.raises(ValueError, match="outside the vendor lock"): + m._publish(args, world, m._make_plan(args, world)) + assert not any(args[0] == "push" for args in world.git_calls) + + +def test_commit_hook_adds_unrelated_file_rejected( + world: World, monkeypatch: pytest.MonkeyPatch +) -> None: + original = world.routed_git + + def tamper(repo: Path, *args: str, text: str | None = None) -> str: + if args[0] == "commit": + _write(repo, "unrelated.txt", "unexpected hook edit\n") + _git(repo, "add", "unrelated.txt") + return original(repo, *args, text=text) + + monkeypatch.setattr(m, "_git", tamper) + args = world.args(publish=True) + with pytest.raises(ValueError, match="outside the vendor lock"): + m._publish(args, world, m._make_plan(args, world)) + assert not any(args[0] == "push" for args in world.git_calls) + + +@pytest.mark.parametrize( + "value", + [ + "http://github.com/user/repo", + "https://token@github.com/a/b", + "file:///tmp/source", + "https://example.com/a/b", + ], +) +def test_non_github_or_credential_urls_rejected(value: str) -> None: + with pytest.raises(ValueError, match="credential-free"): + m._repo_name(value) + + +@pytest.mark.parametrize("suffix", ["", "/", ".git"]) +def test_github_repo_url_parsing(suffix: str) -> None: + assert m._repo_name(f"https://github.com/owner/repo{suffix}") == "owner/repo" + + +@pytest.mark.parametrize("flags", [["--auto-merge"], ["--wait"], ["--timeout", "0"]]) +def test_cli_invalid_flags(flags: list[str]) -> None: + with pytest.raises(SystemExit) as raised: + m.main( + [ + "promote", + "--vendor", + _VENDOR, + "--upstream-repo", + _UPSTREAM, + "--canonical-branch", + "trtllm-prims-ts-dev", + "--source-pr", + "17", + "--canonical-repo", + "maintainer/flashinfer", + "--upstream-pr", + "4829", + *flags, + ] + ) + assert raised.value.code == 2 + + +def test_command_preserves_identity_and_config(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GH_CONFIG_DIR", "/tmp/test-gh-config") + monkeypatch.setenv("GIT_AUTHOR_NAME", "Real Maintainer") + monkeypatch.setenv("GIT_DIR", "/unrelated/git") + + def run(command: list[str], **kwargs: object) -> subprocess.CompletedProcess: + env = kwargs["env"] + assert env["GH_CONFIG_DIR"] == "/tmp/test-gh-config" + assert env["GIT_AUTHOR_NAME"] == "Real Maintainer" + assert env["GIT_LFS_SKIP_SMUDGE"] == "1" + assert "GIT_DIR" not in env + assert "shell" not in kwargs + return subprocess.CompletedProcess(command, 0, "ok\n", "") + + monkeypatch.setattr(m.subprocess, "run", run) + assert m._run(["git", "status"]) == "ok" + + +def test_metadata_missing_and_duplicate_rejected() -> None: + with pytest.raises(ValueError, match="Missing or ambiguous"): + m._record_from_message("ordinary commit") + record = m._record_text({"version": 1}) + with pytest.raises(ValueError, match="Missing or ambiguous"): + m._record_from_message(record + record) + + +def test_compare_paginates_all_commits(monkeypatch: pytest.MonkeyPatch) -> None: + gh = object.__new__(World) + + def get(endpoint: str) -> dict: + page = int(endpoint.rsplit("=", 1)[1]) + return {"total_commits": 101, "commits": list(range(100)) if page == 1 else [100]} + + monkeypatch.setattr(gh, "get", get) + result = m.GitHub.compare(gh, "a/b", "a" * 40, "b" * 40, all_commits=True) + assert result["commits"] == list(range(101)) + + +def test_existing_auto_merge_is_not_reenabled(world: World) -> None: + args = world.args(publish=True, auto_merge=True) + m._publish(args, world, m._make_plan(args, world)) + m._publish(args, world, m._make_plan(args, world)) + enables = [ + payload + for _, endpoint, payload in world.calls + if endpoint == "graphql" and "enablePullRequestAutoMerge" in payload["query"] + ] + assert len(enables) == 1 + + +def test_remote_lock_cannot_change_other_vendor_entries( + world: World, monkeypatch: pytest.MonkeyPatch +) -> None: + args = world.args(publish=True) + plan = m._make_plan(args, world) + pr = m._publish(args, world, plan) + original = m._lock_document_at + + def tampered(gh: m.GitHub, repo: str, sha: str) -> dict: + data = original(gh, repo, sha) + if sha == pr["head"]["sha"]: + data["vendors"]["unexpected"] = {} + return data + + monkeypatch.setattr(m, "_lock_document_at", tampered) + with pytest.raises(ValueError, match="branch/URL-only"): + m._verify_remote_commit(world, plan, _CONSUMER, pr["head"]["sha"]) + + +def test_unmerged_parent_cannot_hide_unrelated_commits( + world: World, monkeypatch: pytest.MonkeyPatch +) -> None: + args = world.args(publish=True) + plan = m._make_plan(args, world) + pr = m._publish(args, world, plan) + original = world.compare + + def unrelated(repo: str, base: str, head: str, **kwargs: object) -> dict: + result = original(repo, base, head, **kwargs) + if repo == _CONSUMER: + result["merge_base_commit"]["sha"] = world.base + return result + + monkeypatch.setattr(world, "compare", unrelated) + with pytest.raises(ValueError, match="not on the consumer base branch"): + m._verify_remote_commit(world, plan, _CONSUMER, pr["head"]["sha"]) + + +def test_wait_checks_final_commit(world: World) -> None: + world.state = "CLEAN" + args = world.args(publish=True, auto_merge=True, wait=True) + m._publish(args, world, m._make_plan(args, world)) + assert world.prs[99]["merged"] + + +def test_wait_timeout_preserves_enabled_auto_merge( + world: World, monkeypatch: pytest.MonkeyPatch +) -> None: + ticks = iter([0, 2]) + monkeypatch.setattr(m.time, "monotonic", lambda: next(ticks)) + args = world.args(publish=True, auto_merge=True, wait=True, timeout=1) + with pytest.raises(TimeoutError, match="still pending"): + m._publish(args, world, m._make_plan(args, world)) + assert world.prs[99]["auto_merge"] + + +def test_source_fetch_uses_reviewed_sha_when_no_local_checkout(world: World) -> None: + args = world.args(publish=True, source_repo=None) + m._publish(args, world, m._make_plan(args, world)) + assert any(args[0] == "fetch" and args[-1] == world.reviewed for args in world.git_calls) + + +def test_incomplete_local_worktree_is_preserved(world: World) -> None: + args = world.args(publish=True) + args.worktree.mkdir() + marker = args.worktree / "important.txt" + marker.write_text("preserve\n") + with pytest.raises((RuntimeError, AssertionError)): + m._publish(args, world, m._make_plan(args, world)) + assert marker.read_text() == "preserve\n" + + +def test_unpaired_promotion_publishes_and_resumes(world: World) -> None: + args = world.args( + publish=True, upstream_pr=[], unpaired_reason="No upstream PR for the DSL API migration." + ) + plan = m._make_plan(args, world) + change = plan.record["changes"][0] + assert change["upstream_pr"] is None + assert change["upstream_head"] is None + assert change["refresh_policy"] == "retain" + assert change["unpaired_reason"] == args.unpaired_reason + assert change["commits"] == [world.first, world.reviewed] + m._publish(args, world, plan) + m._publish(args, world, m._make_plan(args, world)) + assert len(world.prs) == 1 + assert "Some source commits have no paired upstream PR" in world.prs[99]["body"] + assert not any(f"repos/{_UPSTREAM}/pulls/" in call[1] for call in world.calls) + changed = copy.deepcopy(plan.record) + changed["changes"][0]["refresh_policy"] = "drop" + with pytest.raises(ValueError, match="reason/policy"): + m._adopt_record(plan, m._record_text(changed)) + + +@pytest.mark.parametrize( + "options", + [ + {"upstream_pr": [], "unpaired_reason": None}, + {"upstream_pr": [], "unpaired_reason": " "}, + {"upstream_pr": ["4829"], "unpaired_reason": "No pairing"}, + {"upstream_pr": [], "unpaired_reason": "No pairing", "map_upstream": ["abcdef0=4829"]}, + ], +) +def test_unpaired_options_require_explicit_unambiguous_reason(world: World, options: dict) -> None: + with pytest.raises(ValueError, match="unpaired-reason"): + m._make_plan(world.args(**options), world) + assert not world.calls + + +@pytest.mark.parametrize("same_fork", [False, True]) +def test_canonical_branch_name_on_developer_fork(world: World, same_fork: bool) -> None: + lock = world.consumer / m._LOCK + text = lock.read_text().replace("branch: tmp-fix", "branch: trtllm-prims-ts-dev") + if same_fork: + text = text.replace("github.com/dev/", "github.com/maintainer/") + lock.write_text(text) + _git(world.consumer, "add", m._LOCK) + _git(world.consumer, "commit", "--amend", "--no-edit", "-s") + world.main = world.merged = _git(world.consumer, "rev-parse", "HEAD") + args = world.args(publish=True) + if same_fork: + with pytest.raises(ValueError, match="temporary branch"): + m._make_plan(args, world) + return + plan = m._make_plan(args, world) + m._publish(args, world, plan) + actual = m.vendor._load_lock(args.worktree / m._LOCK).vendors[_VENDOR].to_mapping() + assert actual == {**plan.reviewed.to_mapping(), "url": plan.previous.url} + + +def test_unpaired_cli_dry_run(world: World, capfd: pytest.CaptureFixture) -> None: + assert ( + m.main( + [ + "promote", + "--vendor", + _VENDOR, + "--upstream-repo", + _UPSTREAM, + "--canonical-branch", + "trtllm-prims-ts-dev", + "--source-pr", + "17", + "--canonical-repo", + "maintainer/flashinfer", + "--repo", + str(world.consumer), + "--unpaired-reason", + "No paired upstream PR.", + ] + ) + == 0 + ) + assert '"refresh_policy": "retain"' in capfd.readouterr().out + assert not any(args[0] == "push" for args in world.git_calls) + + +def test_previous_developer_fork_cannot_become_canonical( + world: World, monkeypatch: pytest.MonkeyPatch +) -> None: + original = m._lock_at + + def lock_at(gh: m.GitHub, repo: str, sha: str) -> m.vendor.Vendor: + entry = original(gh, repo, sha) + if sha == world.base: + return dataclasses.replace(entry, url="https://github.com/another-dev/flashinfer.git") + return entry + + monkeypatch.setattr(m, "_lock_at", lock_at) + with pytest.raises(ValueError, match="explicit canonical"): + m._make_plan(world.args(), world) + assert not any(call[0] == "push" for call in world.git_calls) + + +@pytest.mark.parametrize("branch", ["tmp-fix", "trtllm-prims-ts-dev-20260910"]) +def test_canonical_branch_must_be_explicit_and_match(world: World, branch: str) -> None: + with pytest.raises(ValueError, match="canonical"): + m._make_plan(world.args(canonical_branch=branch), world) + + +def test_completed_promotion_ignores_later_live_state( + world: World, monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture +) -> None: + world.state = "CLEAN" + args = world.args(publish=True, auto_merge=True) + plan = m._make_plan(args, world) + m._publish(args, world, plan) + merged = world.main + assert f"Squash commit: {merged}" in capfd.readouterr().out + _git(world.consumer, "switch", "--detach", world.main) + _write(world.consumer, "later.txt", "later main commit\n") + world.main = _commit(world.consumer, "later main update") + world.overrides[f"repos/{_CONSUMER}"] = {"allow_auto_merge": False} + world.overrides["repos/maintainer/flashinfer/git/ref/heads/trtllm-prims-ts-dev"] = { + "object": {"sha": "b" * 40} + } + world.overrides["repos/dev/flashinfer"] = {} + world.overrides[f"repos/{_UPSTREAM}/pulls/4829"] = {"state": "closed", "merged": False} + original_compare = world.compare + + def compare(repo: str, base: str, head: str, **kwargs: object) -> dict: + assert repo == _CONSUMER, "Completed retry must not query old source history" + return original_compare(repo, base, head, **kwargs) + + monkeypatch.setattr(world, "compare", compare) + world.calls.clear() + world.git_calls.clear() + args.source_repo = world.root / "missing-source" + resumed = m._make_plan(args, world) + assert resumed.completed_pr["merge_commit_sha"] == merged + assert resumed.record == plan.record + m._publish(args, world, resumed) + assert all(method == "GET" for method, _, _ in world.calls) + assert not any(call[0] in ("fetch", "push", "commit", "worktree") for call in world.git_calls) + assert len(world.comments) == 1 + + +@pytest.mark.parametrize("phase", ["worktree_created", "before_commit", "after_commit"]) +@pytest.mark.parametrize("main_advances", [False, True]) +def test_interrupted_preparation_resumes_without_extra_commits( + world: World, monkeypatch: pytest.MonkeyPatch, phase: str, main_advances: bool +) -> None: + def interrupted(repo: Path, *args: str, text: str | None = None) -> str: + if phase == "before_commit" and args[0] == "commit": + raise RuntimeError("simulated interruption") + result = world.routed_git(repo, *args, text=text) + if (phase == "worktree_created" and args[:2] == ("worktree", "add")) or ( + phase == "after_commit" and args[0] == "commit" + ): + raise RuntimeError("simulated interruption") + return result + + args = world.args(publish=True) + monkeypatch.setattr(m, "_git", interrupted) + with pytest.raises(RuntimeError, match="simulated interruption"): + m._publish(args, world, m._make_plan(args, world)) + assert not any(call[0] == "push" for call in world.git_calls) + monkeypatch.setattr(m, "_git", world.routed_git) + if main_advances: + _write(world.consumer, "later.txt", "later main commit\n") + world.main = _commit(world.consumer, "later main update") + pr = m._publish(args, world, m._make_plan(args, world)) + head = pr["head"]["sha"] + assert _git(world.consumer, "rev-list", "--count", f"{world.merged}..{head}") == "1" + assert sum(call[:2] == ("worktree", "add") for call in world.git_calls) == 1 + assert len(world.comments) == 1 + + +@pytest.mark.parametrize("edit", ["tracked", "untracked", "index_only", "index_only_other"]) +def test_incomplete_recovery_preserves_unexpected_edits( + world: World, monkeypatch: pytest.MonkeyPatch, edit: str +) -> None: + def interrupted(repo: Path, *args: str, text: str | None = None) -> str: + if args[0] == "commit": + raise RuntimeError("simulated interruption") + return world.routed_git(repo, *args, text=text) + + args = world.args(publish=True) + monkeypatch.setattr(m, "_git", interrupted) + with pytest.raises(RuntimeError, match="interruption"): + m._publish(args, world, m._make_plan(args, world)) + monkeypatch.setattr(m, "_git", world.routed_git) + if edit.startswith("index_only"): + file = m._LOCK if edit == "index_only" else "vendored/prims_ts/kernel.py" + target = args.worktree / file + original = target.read_text() + target.write_text( + original.replace(world.reviewed, world.previous) + if edit == "index_only" + else "preserve this staged edit\n" + ) + _git(args.worktree, "add", file) + target.write_text(original) + preserved = _git(args.worktree, "show", f":{file}") + else: + file = "vendored/prims_ts/kernel.py" if edit == "tracked" else "important.txt" + _write(args.worktree, file, "preserve this edit\n") + with pytest.raises(ValueError): + m._publish(args, world, m._make_plan(args, world)) + if edit.startswith("index_only"): + assert _git(args.worktree, "show", f":{file}") == preserved + else: + assert (args.worktree / file).read_text() == "preserve this edit\n" + assert not any(call[0] == "push" for call in world.git_calls) + assert _git(args.worktree, "rev-parse", "HEAD") == world.merged + + +def test_ci_skip_is_exact_once_and_precedes_auto_merge(world: World) -> None: + args = world.args(publish=True, auto_merge=True) + m._publish(args, world, m._make_plan(args, world)) + m._publish(args, world, m._make_plan(args, world)) + assert [comment["body"] for comment in world.comments] == [m._CI_SKIP_COMMAND] + create_index = next( + i + for i, (verb, route, _) in enumerate(world.calls) + if verb == "POST" and route.endswith("/pulls") + ) + comment_index = next( + i + for i, (verb, route, _) in enumerate(world.calls) + if verb == "POST" and route.endswith("/comments") + ) + auto_index = next( + i + for i, (_, route, payload) in enumerate(world.calls) + if route == "graphql" and "enablePullRequestAutoMerge" in payload["query"] + ) + assert create_index < comment_index < auto_index + + +@pytest.mark.parametrize("after", [False, True]) +def test_ci_comment_failure_is_resumable_without_duplicates(world: World, after: bool) -> None: + world.fail_comment_before = not after + world.fail_comment_after = after + args = world.args(publish=True, auto_merge=True) + with pytest.raises(RuntimeError, match="comment failure"): + m._publish(args, world, m._make_plan(args, world)) + assert len(world.prs) == 1 + assert not world.prs[99]["auto_merge"] + world.fail_comment_before = world.fail_comment_after = False + m._publish(args, world, m._make_plan(args, world)) + assert len(world.comments) == 1 + assert world.prs[99]["auto_merge"] + + +@pytest.mark.parametrize("same_actor", [False, True]) +def test_ci_comment_search_paginates_and_checks_author(world: World, same_actor: bool) -> None: + world.comments = [{"body": "unrelated", "user": {"login": "maintainer"}}] * 100 + world.comments.append( + { + "body": m._CI_SKIP_COMMAND, + "user": {"login": "maintainer" if same_actor else "someone-else"}, + } + ) + args = world.args(publish=True) + m._publish(args, world, m._make_plan(args, world)) + assert len(world.comments) == (101 if same_actor else 102) + assert any("comments?per_page=100&page=2" in route for _, route, _ in world.calls) + + +@pytest.mark.parametrize("change", ["closed", "head"]) +def test_ci_request_rejects_changed_pr(world: World, change: str) -> None: + world.fail_comment_before = True + args = world.args(publish=True) + plan = m._make_plan(args, world) + with pytest.raises(RuntimeError, match="comment failure"): + m._publish(args, world, plan) + verified = copy.deepcopy(world.prs[99]) + if change == "closed": + world.prs[99]["state"] = "closed" + else: + world.prs[99]["head"]["sha"] = "b" * 40 + world.calls.clear() + with pytest.raises(ValueError, match="closed or changed"): + m._ensure_ci_skip(world, plan, verified) + assert all(method == "GET" for method, _, _ in world.calls) + assert not world.comments + + +def test_generated_body_passes_repository_checklist(world: World) -> None: + checker = runpy.run_path(str(_SCRIPTS.parent.parent / ".github/scripts/pr_checklist_check.py")) + plan = m._make_plan(world.args(), world) + body = m._pr_body(plan, f"{plan.title}\n\n{m._record_text(plan.record)}") + assert checker["find_unresolved_tasks"](body) == [] + assert checker["check_pr_checklist_section"](body) == (True, "") + assert "Required owner reviews and checks completed" not in body + assert m._CI_SKIP_COMMAND in body + assert "Automatic GitHub checks" in body + + +def test_mutating_git_commands_stream_output(monkeypatch: pytest.MonkeyPatch) -> None: + def run(command: list[str], **kwargs: object) -> subprocess.CompletedProcess: + assert kwargs["capture_output"] is False + assert kwargs["env"]["GIT_LFS_SKIP_SMUDGE"] == "1" + return subprocess.CompletedProcess(command, 0, None, None) + + monkeypatch.setattr(m.subprocess, "run", run) + assert m._git(Path("/test/repository"), "worktree", "add", "new") == "" + assert m._git(Path("/test/repository"), "commit", "-s", "-F", "-", text="test") == "" + + +@pytest.mark.parametrize("loss", ["sqlite", "workdir"]) +@pytest.mark.parametrize( + "phase", + ["worktree", "stage", "commit", "branch", "canonical", "pr", "comment", "auto_merge", "merged"], +) +def test_bot_recovers_publication_after_cache_loss( + world: World, + monkeypatch: pytest.MonkeyPatch, + phase: str, + loss: str, +) -> None: + workdir = world.root / "bot-state" + b._initialize_workdir(workdir) + args = argparse.Namespace( + migrate_canonical_branch_from=None, + refresh_branch=[], + canonical_repo="maintainer/flashinfer", + canonical_branch="trtllm-prims-ts-dev", + fork="maintainer/TensorRT-LLM", + repo=world.consumer, + workdir=workdir, + publish=True, + since=None, + ) + database = workdir / b._STATE_FILE + store = b.Store(database) + monitor = b.Monitor(args, world, store) + if phase == "merged": + world.state = "CLEAN" + git, api = world.routed_git, world.api + + def crash_git( + repo: Path, + *arguments: str, + text: str | None = None, + allowed_returncodes: tuple[int, ...] = (0,), + ) -> str: + result = git(repo, *arguments, text=text, allowed_returncodes=allowed_returncodes) + boundary = ( + "worktree" + if arguments[:2] == ("worktree", "add") + else "stage" + if arguments[0] == "add" + else "commit" + if arguments[0] == "commit" + else "branch" + if arguments[:2] == ("push", "https://github.com/maintainer/TensorRT-LLM.git") + else "canonical" + if arguments[:2] == ("push", "https://github.com/maintainer/flashinfer.git") + else None + ) + if boundary == phase: + raise RuntimeError("simulated process loss") + return result + + def crash_api(endpoint: str, *, method: str = "GET", payload: dict | None = None): + result = api(endpoint, method=method, payload=payload) + if method == "POST": + boundary = ( + "pr" + if endpoint.endswith("/pulls") + else "comment" + if endpoint.endswith("/issues/99/comments") + else phase + if endpoint == "graphql" + and payload["query"].startswith("mutation") + and phase in ("auto_merge", "merged") + else None + ) + if boundary == phase: + raise RuntimeError("simulated process loss") + return result + + with monkeypatch.context() as crash: + crash.setattr(m, "_git", crash_git) + crash.setattr(world, "api", crash_api) + with pytest.raises(RuntimeError, match="simulated process loss"): + monitor.inspect(17) + # Publication of the provenance branch precedes the first canonical mutation. + if phase == "branch": + assert _git(world.canonical, "rev-parse", "trtllm-prims-ts-dev") == world.previous + committed = phase not in ("worktree", "stage") + expected_head = ( + m._local_branch(world.consumer, m._plan_identity(args=world.args(), gh=world).branch) + if committed + else None + ) + expected_record = ( + m._record_from_message(_git(world.consumer, "show", "-s", "--format=%B", expected_head)) + if committed + else None + ) + source_comments = copy.deepcopy(world.source_comments) + store.close() + if loss == "sqlite": + database.unlink() + else: + # Preserve the lost directory for assertions; Git's registration now + # refers to a missing path, as on workdir loss on the same machine. + workdir.rename(world.root / "lost-bot-state") + b._initialize_workdir(workdir) + + if committed: + world.source_body = "Description changed after the durable promotion commit." + world.overrides[f"repos/{_UPSTREAM}/pulls/4829"] = {"state": "closed", "merged": False} + + def no_rematching(*unused): + raise AssertionError("Durable promotion must not re-match mutable upstream PRs") + + monkeypatch.setattr(b.metadata, "resolve", no_rematching) + + store = b.Store(database) + resumed = b.Monitor(args, world, store) + assert 17 in resumed._discover() # The merged source PR predates startup by years. + resumed.poll() + assert store.get("problem:17") is None + assert store.get("problem:99") is None + assert world.source_comments == source_comments + assert len(world.prs) == len(world.comments) == 1 + assert _git(world.canonical, "rev-parse", "trtllm-prims-ts-dev") == world.reviewed + pr = world.prs[99] + if committed: + assert pr["head"]["sha"] == expected_head + assert sum(call[0] == "commit" for call in world.git_calls) == 1 + if not pr.get("merged"): + saved = copy.deepcopy(pr["auto_merge"]) + assert saved["merge_method"] == "squash" + writes = [ + (method, endpoint, payload) + for method, endpoint, payload in world.calls + if method != "GET" + and (endpoint != "graphql" or payload["query"].startswith("mutation")) + ] + resumed.inspect(17) + repeated = [ + (method, endpoint, payload) + for method, endpoint, payload in world.calls + if method != "GET" + and (endpoint != "graphql" or payload["query"].startswith("mutation")) + ] + assert writes == repeated + tree = _git(world.consumer, "rev-parse", f"{pr['head']['sha']}^{{tree}}") + world.main = _git( + world.consumer, + "commit-tree", + tree, + "-p", + world.main, + text=f"{saved['commit_title']}\n\n{saved['commit_message']}\n", + ) + pr.update(merged=True, merged_at="today", state="closed", merge_commit_sha=world.main) + resumed.inspect(17) + assert store.get("complete:17") == pr["html_url"] + store.close() + + # Even completion markers are only a cache: recover from the squash commit + # with an entirely new database and unavailable live source/metadata. + database.unlink() + world.source_body = "No metadata remains." + world.overrides["repos/dev/flashinfer"] = {} + calls_before = len(world.calls) + store = b.Store(database) + final = b.Monitor(args, world, store) + final.inspect(17) + assert store.get("complete:17") == pr["html_url"] + assert all(method == "GET" for method, _, _ in world.calls[calls_before:]) + assert m._lock_at(world, _CONSUMER, world.main).commit == world.reviewed + if committed: + assert ( + m._record_from_message(_git(world.consumer, "show", "-s", "--format=%B", world.main)) + == expected_record + ) + store.close() + + +@pytest.mark.parametrize("tamper", ["identity", "diff", "dco", "canonical", "closed"]) +def test_recovery_rejects_changed_artifacts_without_writes(world: World, tamper: str) -> None: + args = world.args(publish=True) + original = m._make_plan(args, world) + m._publish(args, world, original) + pr = world.prs[99] + endpoint = f"repos/{_CONSUMER}/commits/{pr['head']['sha']}" + commit = world.get(endpoint) + if tamper == "identity": + record = copy.deepcopy(original.record) + record["promoted"] = "a" * 40 + commit["commit"]["message"] = m._record_text(record) + elif tamper == "diff": + commit["files"].append({"filename": "unreviewed.py"}) + elif tamper == "dco": + commit["commit"]["message"] = m._record_text(original.record) + elif tamper == "canonical": + _git(world.canonical, "update-ref", "refs/heads/trtllm-prims-ts-dev", world.first) + else: + pr["state"] = "closed" + world.overrides[endpoint] = commit + world.calls.clear() + world.git_calls.clear() + with pytest.raises(ValueError): + m._recover_plan(args, world, m._plan_identity(args, world)) + assert all(method == "GET" for method, _, _ in world.calls) + assert not any(call[0] in ("push", "commit", "worktree") for call in world.git_calls) + + +def test_bot_cold_recovery_dry_run_and_superseded_history(world: World) -> None: + args = argparse.Namespace( + migrate_canonical_branch_from=None, + refresh_branch=[], + canonical_repo="maintainer/flashinfer", + canonical_branch="trtllm-prims-ts-dev", + fork="maintainer/TensorRT-LLM", + repo=world.consumer, + workdir=world.root, + publish=False, + since=None, + ) + # No published operation exists and a later pin has superseded this update. + world.main = world.base + world.source_body = "No metadata" + with contextlib.closing(b.Store(":memory:")) as store: + b.Monitor(args, world, store).inspect(17) + assert not world.source_comments + assert all(method == "GET" for method, _, _ in world.calls) + + world.main = world.merged + m._publish(world.args(publish=True), world, m._make_plan(world.args(), world)) + world.calls.clear() + world.git_calls.clear() + with contextlib.closing(b.Store(":memory:")) as store: + b.Monitor(args, world, store).inspect(17) + assert all(method == "GET" for method, _, _ in world.calls) + assert not world.git_calls + assert world.prs[99]["auto_merge"] is None diff --git a/tests/unittest/others/test_vendor_bot.py b/tests/unittest/others/test_vendor_bot.py new file mode 100644 index 000000000000..bb30a565c83b --- /dev/null +++ b/tests/unittest/others/test_vendor_bot.py @@ -0,0 +1,2543 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Hermetic tests for metadata, attribution, local state, and author feedback.""" + +from __future__ import annotations + +import argparse +import contextlib +import copy +import dataclasses +import importlib +import importlib.util +import json +import logging +import os +import subprocess +import sys +import threading +import time +from pathlib import Path +from urllib.parse import parse_qs + +import pytest + +_ROOT = Path(__file__).resolve().parents[3] +_PACKAGE = "vendor_bot_test" +_SPEC = importlib.util.spec_from_file_location( + _PACKAGE, + _ROOT / "scripts/vendor/__init__.py", + submodule_search_locations=[str(_ROOT / "scripts/vendor")], +) +_MODULE = importlib.util.module_from_spec(_SPEC) +sys.modules[_PACKAGE] = _MODULE +_SPEC.loader.exec_module(_MODULE) +b = importlib.import_module(f"{_PACKAGE}.bot") +m = importlib.import_module(f"{_PACKAGE}.metadata") +p = importlib.import_module(f"{_PACKAGE}.promote") + +pytestmark = pytest.mark.cpu_only + + +def _body(content: str) -> str: + return f"{m.BEGIN}\n```yaml\nschema_version: 1\nvendors:\n toy:\n{content}\n```\n{m.END}" + + +def _issue(message: str = "Missing marked block") -> m.Diagnostic: + return m.Diagnostic(message, "Edit the PR description to supply the missing metadata.") + + +@pytest.mark.parametrize("line_ending", ["\n", "\r\n", "\r"], ids=["lf", "crlf", "cr"]) +def test_metadata_only_needs_upstream_urls(line_ending: str) -> None: + entry = m.parse( + _body(" upstream_prs:\n - https://github.com/org/library/pull/42").replace( + "\n", line_ending + ), + "toy", + "org/library", + ) + assert entry == {"upstream_prs": [42], "unpaired_reason": None, "resolutions": {}} + + +def test_metadata_mixed_line_endings() -> None: + body = _body(" upstream_prs:\n - https://github.com/org/library/pull/42") + mixed = body.replace("```yaml\n", "```yaml\r\n").replace("\n```\n", "\r```\r\n") + expected = m.parse(body, "toy", "org/library") + actual = m.parse(mixed, "toy", "org/library") + assert actual == expected + assert m.fingerprint(actual) == m.fingerprint(expected) + + +def test_metadata_size_limit_precedes_line_ending_normalization() -> None: + body = "\r\n" * 32768 + m.template("toy", "org/library") + with pytest.raises(ValueError, match="exceeds the supported metadata size"): + m.parse(body, "toy", "org/library") + + +@pytest.mark.parametrize( + "content", + [ + " upstream_prs: []", + " upstream_prs: https://github.com/org/library/pull/42", + " upstream_prs: [https://github.com/other/library/pull/42]", + " upstream_prs: [https://github.com/org/library/pull/42, https://github.com/org/library/pull/42]", + " unpaired_reason: ''", + " unpaired_reason: test\n upstream_prs: []", + " canonical_repo: attacker/repo", + " upstream_prs: []\n upstream_prs: []", + " upstream_prs: [", + " upstream_prs: [https://github.com/org/library/pull/42]\n resolutions: []", + " upstream_prs: &prs [https://github.com/org/library/pull/42]", + " upstream_prs: &prs [*prs]", + ], +) +def test_invalid_metadata_rejected(content: str) -> None: + with pytest.raises(m.Diagnostic) as failure: + m.parse(_body(content), "toy", "org/library") + assert failure.value.action + assert "**Fix:**" in str(failure.value) + + +@pytest.mark.parametrize( + "body", ["", "missing", m.template("toy", "org/library") * 2, m.END + m.BEGIN] +) +def test_missing_or_duplicate_marked_blocks(body: str) -> None: + with pytest.raises(ValueError): + m.parse(body, "toy", "org/library") + + +def test_author_resolution_requires_explanation() -> None: + content = ( + " upstream_prs: [https://github.com/org/library/pull/42]\n resolutions:\n " + + "a" * 40 + + ":\n upstream_pr: https://github.com/org/library/pull/42\n reason: adapted ABI" + ) + entry = m.parse(_body(content), "toy", "org/library") + assert entry["resolutions"]["a" * 40] == {"upstream_pr": 42, "reason": "adapted ABI"} + with pytest.raises(ValueError): + m.parse(_body(content.replace("adapted ABI", "''")), "toy", "org/library") + + +@pytest.mark.parametrize("line_ending", ["\n", "\r\n", "\r"]) +@pytest.mark.parametrize( + "start_count,end_count", [(0, 0), (0, 1), (1, 0), (2, 1), (1, 2), (2, 2), (2, 0)] +) +def test_marker_diagnostics_name_missing_and_duplicate_markers( + start_count: int, end_count: int, line_ending: str +) -> None: + body = line_ending.join([m.BEGIN] * start_count + [m.END] * end_count) + with pytest.raises(m.Diagnostic) as failure: + m.parse(body, "toy", "org/library") + issue = failure.value + for marker, count in ((m.BEGIN, start_count), (m.END, end_count)): + if count == 0: + assert f"`{marker}`" in issue.problem and "Missing" in issue.problem + elif count > 1: + assert f"`{marker}` appears {count} times" in issue.problem + else: + assert f"`{marker}`" not in issue.problem + assert "PR" in issue.action + if start_count == end_count == 0: + example = issue.example.removeprefix("````markdown\n").removesuffix("\n````") + assert m.parse(example, "toy", "org/library")["upstream_prs"] == [123] + assert "not a comment" in issue.action.replace("—", " ") + elif 0 in (start_count, end_count): + assert "before" in issue.action if start_count == 0 else "after" in issue.action + if max(start_count, end_count) > 1: + assert "preserving" in issue.action and "Consolidate" in issue.action + + +def test_reversed_markers_and_bad_fence_have_different_fixes() -> None: + with pytest.raises(m.Diagnostic) as reversed_markers: + m.parse(m.END + m.BEGIN, "toy", "org/library") + assert "end marker appears before" in reversed_markers.value.problem + assert f"`{m.BEGIN}` before" in reversed_markers.value.action + with pytest.raises(m.Diagnostic) as bad_fence: + m.parse(m.BEGIN + "\nschema_version: 1\n" + m.END, "toy", "org/library") + assert "fenced yaml" in bad_fence.value.problem + assert "three backticks" in bad_fence.value.action + assert m.BEGIN in bad_fence.value.example + + +@pytest.mark.parametrize( + "content,problem,fix", + [ + (" upstream_prs: []", "vendors.toy.upstream_prs", "list every related upstream PR"), + ( + " upstream_prs: [https://github.com/wrong/repo/pull/42]", + "upstream_prs[0]", + "org/library/pull/123", + ), + (" upstream_prs: [42]", "upstream_prs[0]", "full URL"), + ( + " upstream_prs: [https://github.com/org/library/pull/42#discussion]", + "upstream_prs[0]", + "no query string or fragment", + ), + ( + " upstream_prs: [https://github.com/org/library/pull/42, https://github.com/org/library/pull/42]", + "duplicate upstream PR #42", + "Remove the repeated", + ), + (" canonical_repo: wrong", "canonical_repo", "Repository/operator settings"), + (" unpaired_reason: ''", "vendors.toy.unpaired_reason", "Do not use it to bypass"), + (" unpaired_reason: downstream\n resolutions: {}", "cannot accompany", "keep only"), + ( + " upstream_prs: [https://github.com/org/library/pull/42]\n resolutions: []", + "vendors.toy.resolutions", + "full 40-character", + ), + ( + " upstream_prs: &prs [https://github.com/org/library/pull/42]", + "aliases and anchors", + "explicit values", + ), + ], +) +def test_metadata_diagnostics_identify_field_and_fix(content: str, problem: str, fix: str) -> None: + with pytest.raises(m.Diagnostic) as failure: + m.parse(_body(content), "toy", "org/library") + assert problem in failure.value.problem + assert fix in failure.value.action + + +@pytest.mark.parametrize( + "content,problem", + [ + ("schema_version: '1'\nvendors: {}", "schema_version"), + ("schema_version: 1", "vendors"), + ("schema_version: 1\nvendors: {}\nextra: false", "extra"), + ("schema_version: 1\nvendors: {}", "vendors.toy"), + ("[]", "mapping, not a list"), + ], +) +def test_metadata_structure_has_targeted_guidance(content: str, problem: str) -> None: + with pytest.raises(m.Diagnostic) as failure: + m.parse(f"{m.BEGIN}\n```yaml\n{content}\n```\n{m.END}", "toy", "org/library") + assert problem in failure.value.problem + assert "PR description" in failure.value.action or "PR-description" in failure.value.action + + +@pytest.mark.parametrize( + "content,problem", + [ + (" upstream_prs: [secret-unclosed", "YAML line"), + (" private-key: secret\n private-key: secret", "duplicate mapping key"), + (" <<: {private-key: secret}", "merge keys are not allowed"), + ], +) +def test_yaml_errors_show_location_without_echoing_values(content: str, problem: str) -> None: + with pytest.raises(m.Diagnostic) as failure: + m.parse(_body(content), "toy", "org/library") + assert problem in failure.value.problem + assert "YAML line" in failure.value.problem and "column" in failure.value.problem + assert "secret" not in str(failure.value) + assert "line 1 is the first line inside the YAML fence" in failure.value.action + + +@pytest.mark.parametrize( + "sha,resolution,problem,fix", + [ + ( + "abc123", + "upstream_pr: https://github.com/org/library/pull/42\n reason: adapted", + "Invalid source commit key", + "40-character", + ), + ( + "a" * 40, + "upstream_pr: https://github.com/org/library/pull/42", + "exactly", + "missing fields", + ), + ( + "a" * 40, + "upstream_pr: https://github.com/org/library/pull/43\n reason: adapted", + "not listed", + "upstream_prs", + ), + ( + "a" * 40, + "upstream_pr: https://github.com/org/library/pull/42\n reason: ''", + ".reason", + "Explain the downstream adaptation", + ), + ], +) +def test_resolution_errors_identify_commit_and_fix( + sha: str, resolution: str, problem: str, fix: str +) -> None: + content = ( + " upstream_prs: [https://github.com/org/library/pull/42]\n" + f" resolutions:\n {sha}:\n {resolution}" + ) + with pytest.raises(m.Diagnostic) as failure: + m.parse(_body(content), "toy", "org/library") + assert sha in failure.value.problem and problem in failure.value.problem + assert fix in failure.value.action + + +def test_resolution_example_is_nested_under_real_vendor_and_parseable() -> None: + example = m._resolution_example("toy", "org/library") + yaml_text = example.removeprefix("```yaml\n").removesuffix("\n```") + body = (f"{m.BEGIN}\n```yaml\nschema_version: 1\n{yaml_text}\n```\n{m.END}").replace( + "FULL_40_CHARACTER_SOURCE_SHA", "a" * 40 + ) + entry = m.parse(body, "toy", "org/library") + assert entry["resolutions"]["a" * 40]["upstream_pr"] == 123 + + +def test_untrusted_field_names_cannot_inject_markdown() -> None: + body = _body(" '': value") + with pytest.raises(m.Diagnostic) as failure: + m.parse(body, "toy", "org/library") + assert " None: + entry = m.parse( + _body(" unpaired_reason: downstream-only compatibility"), "toy", "org/library" + ) + assert entry["unpaired_reason"] == "downstream-only compatibility" + assert m.parse(m.template("toy", "org/library"), "toy", "org/library")["upstream_prs"] == [123] + + +def _git(repo: Path, *arguments: str) -> str: + result = subprocess.run( + ["git", "-C", str(repo), *arguments], capture_output=True, text=True, check=True + ) + return result.stdout.strip() + + +@pytest.fixture +def source(tmp_path: Path) -> Path: + repo = tmp_path / "source" + repo.mkdir() + _git(repo, "init", "-q", "-b", "main") + _git(repo, "config", "user.name", "Test") + _git(repo, "config", "user.email", "test@example.invalid") + _git(repo, "config", "core.hooksPath", str(tmp_path / "no-hooks")) + return repo + + +def _commit(repo: Path, text: str, filename: str = "code.py") -> str: + (repo / filename).parent.mkdir(parents=True, exist_ok=True) + (repo / filename).write_text(text) + _git(repo, "add", filename) + _git(repo, "commit", "-q", "-m", "test") + return _git(repo, "rev-parse", "HEAD") + + +def _pr(base: str, head: str) -> dict: + return {"base": {"sha": base, "ref": "main"}, "head": {"sha": head}, "state": "open"} + + +def test_match_multiple_upstream_prs_without_manual_map(source: Path) -> None: + base = _commit(source, "VALUE = 0\n") + first = _commit(source, "VALUE = 1\n") + second = _commit(source, "VALUE = 2\n") + assignments, evidence = m.match( + source, base, second, {41: _pr(base, first), 42: _pr(first, second)} + ) + assert assignments == {first: 41, second: 42} + assert all(item["method"] == "same-commit" for item in evidence.values()) + + +def test_match_rebased_patch(source: Path) -> None: + base = _commit(source, "VALUE = 0\n") + downstream = _commit(source, "VALUE = 1\n") + _git(source, "checkout", "-q", "-b", "upstream", base) + context = _commit(source, "context\n", "other.txt") + upstream = _commit(source, "VALUE = 1\n") + assignments, evidence = m.match(source, base, downstream, {42: _pr(context, upstream)}) + assert assignments == {downstream: 42} + assert evidence[downstream]["method"] == "same-edits" + + +def test_match_only_lock_selected_files(source: Path) -> None: + base = _commit(source, "VALUE = 0\n", "library/kernels/code.py") + for name in ( + "tests/test_code.py", + "library/kernels/README.md", + "library/kernels_extra/code.py", + ): + path = source / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("downstream-only content\n") + _git(source, "add", "-A") + downstream = _commit(source, "VALUE = 1\n", "library/kernels/code.py") + _git(source, "checkout", "-q", "-b", "upstream", base) + upstream = _commit(source, "VALUE = 1\n", "library/kernels/code.py") + assignments, evidence = m.match( + source, + base, + downstream, + {42: _pr(base, upstream)}, + source="library/kernels", + include=("**/*.py",), + ) + assert assignments == {downstream: 42} + assert evidence[downstream]["method"] == "same-edits" + with pytest.raises(ValueError, match="no exact match"): + m.match(source, base, downstream, {42: _pr(base, upstream)}) + + +@pytest.mark.parametrize("include", [("*.py",), ("**/*.py",), ("nested/*.py",)]) +def test_patch_uses_lock_globs_and_literal_git_paths( + source: Path, include: tuple[str, ...] +) -> None: + base = _commit(source, "base\n", "outside.txt") + paths = ["root.py", "nested/kernel[1] *.py", "nested/README.md", "nested/deep/kernel.py"] + for name in paths: + path = source / "selected" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"{name}\n") + _git(source, "add", "-A") + head = _commit(source, "not selected\n", "selected-sibling/outside.py") + patch = m._patch(source, base, head, source="selected", include=include) + for name in paths: + assert (f"+{name}\n" in patch) == m.manage._matches(name, include) + assert "not selected" not in patch + assert m._patch(source, base, head, source="absent", include=include) == "" + + +@pytest.mark.parametrize("squash", [False, True]) +def test_excluded_commits_are_not_attributed_to_upstream(source: Path, squash: bool) -> None: + base = _commit(source, "VALUE = 0\n", "src/code.py") + before = _commit(source, "initial tests\n", "tests/test_code.py") + first = _commit(source, "VALUE = 1\n", "src/code.py") + second = _commit(source, "VALUE = 2\n", "src/code.py") + after = _commit(source, "more tests\n", "tests/test_code.py") + upstream = after + if squash: + _git(source, "checkout", "-q", "-b", "upstream", base) + upstream = _commit(source, "VALUE = 2\n", "src/code.py") + assignments, evidence = m.match( + source, + base, + after, + {42: _pr(base, upstream)}, + source="src", + include=("**/*.py",), + ) + assert assignments == {before: None, first: 42, second: 42, after: None} + for sha in (before, after): + assert evidence[sha] == {"method": "outside-vendor-scope", "refresh_policy": "retain"} + assert evidence[first]["method"] == ("source-series-aggregate" if squash else "same-commit") + + +def test_empty_selected_patches_never_match_an_upstream_pr(source: Path) -> None: + base = _commit(source, "VALUE = 0\n", "src/code.py") + head = _commit(source, "tests only\n", "tests/test_code.py") + assignments, _ = m.match(source, base, head, {}, source="src", include=("**/*.py",)) + assert assignments == {head: None} + with pytest.raises(ValueError, match="cover no new lock-selected changes"): + m.match(source, base, head, {42: _pr(base, head)}, source="src", include=("**/*.py",)) + with pytest.raises(ValueError, match="Remove resolutions"): + m.match( + source, + base, + head, + {42: _pr(base, head)}, + {head: {"upstream_pr": 42, "reason": "tests"}}, + source="src", + include=("**/*.py",), + ) + + +def test_selected_code_mismatch_is_not_hidden_by_identical_tests(source: Path) -> None: + base = _commit(source, "VALUE = 0\n", "src/code.py") + tests = _commit(source, "shared tests\n", "tests/test_code.py") + downstream = _commit(source, "VALUE = 1\n", "src/code.py") + _git(source, "checkout", "-q", "-b", "upstream", tests) + upstream = _commit(source, "VALUE = 2\n", "src/code.py") + with pytest.raises(ValueError, match="no exact match"): + m.match( + source, base, downstream, {42: _pr(base, upstream)}, source="src", include=("**/*.py",) + ) + + +@pytest.mark.parametrize("into_scope", [False, True]) +def test_patch_keeps_selected_side_of_move(source: Path, into_scope: bool) -> None: + selected, excluded = "src/code.py", "tests/test_code.py" + old, new = (excluded, selected) if into_scope else (selected, excluded) + base = _commit(source, "VALUE = 0\n", old) + (source / new).parent.mkdir(parents=True, exist_ok=True) + _git(source, "mv", old, new) + _git(source, "commit", "-q", "-m", "move") + head = _git(source, "rev-parse", "HEAD") + patch = m._patch(source, base, head, source="src", include=("**/*.py",)) + assert selected in patch + assert excluded not in patch + assert ("+VALUE = 0" if into_scope else "-VALUE = 0") in patch + assert ("new file mode" if into_scope else "deleted file mode") in patch + + +@pytest.mark.parametrize("change", [{"source": "different"}, {"include": ("*.py",)}]) +def test_resolve_rejects_selection_changes_before_fetch(tmp_path: Path, change: dict) -> None: + previous = _vendor("a" * 40, "https://github.com/operator/library.git", "dev") + reviewed = dataclasses.replace(previous, commit="b" * 40, **change) + cache = tmp_path / "cache" + with pytest.raises(ValueError, match="unchanged lock source/include"): + m.resolve(None, previous, reviewed, {}, cache) + assert not cache.exists() + + +@pytest.mark.parametrize( + ("original", "rebased"), + [ + ( + "def run():\n before = 0\n value = 0\n after = 0\n return value\n", + "def run():\n before = 2\n value = 0\n after = 3\n return value\n", + ), + ( + "def old_helper():\n pass\n\n\n@cache\ndef run():\n value = 0\n", + "def new_helper():\n pass\n\n\n@cache\ndef run():\n value = 0\n", + ), + ( + "def run():\n value = 0\n # gap\n second = 0\n", + "def run():\n value = 0\n" + " # gap\n" * 10 + " second = 0\n", + ), + ], + ids=["neighboring-lines", "neighboring-function", "hunk-spacing"], +) +def test_match_ignores_unchanged_context(source: Path, original: str, rebased: str) -> None: + # Repo-local diff settings must not reintroduce unchanged context. + _git(source, "config", "diff.interHunkContext", "20") + base = _commit(source, original) + downstream = _commit( + source, original.replace("value = 0", "value = 1").replace("second = 0", "second = 1") + ) + _git(source, "checkout", "-q", "-b", "upstream", base) + context = _commit(source, rebased) + upstream = _commit( + source, rebased.replace("value = 0", "value = 1").replace("second = 0", "second = 1") + ) + assignments, evidence = m.match(source, base, downstream, {42: _pr(context, upstream)}) + assert assignments == {downstream: 42} + assert evidence[downstream]["method"] == "same-edits" + + +@pytest.mark.parametrize("different", [" VALUE = 1\n", "VALUE = 1 \n", "VALUE = 1\r\n"]) +def test_matching_preserves_whitespace(source: Path, different: str) -> None: + base = _commit(source, "VALUE = 0\n") + downstream = _commit(source, "VALUE = 1\n") + _git(source, "checkout", "-q", "-b", "upstream", base) + upstream = _commit(source, different) + with pytest.raises(ValueError, match="Upstream attribution is unresolved"): + m.match(source, base, downstream, {42: _pr(base, upstream)}) + + +def test_decorator_insertion_ignores_preceding_function_label(source: Path) -> None: + original = "def old_helper():\n pass\n\n\ndef run():\n return 1\n" + rebased = original.replace("old_helper", "new_helper") + base = _commit(source, original) + downstream = _commit(source, original.replace("def run():", "@cache\ndef run():")) + _git(source, "checkout", "-q", "-b", "upstream", base) + context = _commit(source, rebased) + upstream = _commit(source, rebased.replace("def run():", "@cache\ndef run():")) + assert "@@ def old_helper():" in _git(source, "diff", "--unified=0", base, downstream) + assert "@@ def new_helper():" in _git(source, "diff", "--unified=0", context, upstream) + assignments, evidence = m.match(source, base, downstream, {42: _pr(context, upstream)}) + assert assignments == {downstream: 42} + assert evidence[downstream]["method"] == "same-edits" + + +def test_hunk_labels_are_not_part_of_edit_identity(source: Path) -> None: + original = "def first():\n return False\n\n\ndef second():\n return False\n" + base = _commit(source, original) + downstream = _commit(source, original.replace("return False", "return True", 1)) + _git(source, "checkout", "-q", "-b", "upstream", base) + upstream = _commit( + source, + original.replace("def second():\n return False", "def second():\n return True"), + ) + assignments, evidence = m.match(source, base, downstream, {42: _pr(base, upstream)}) + assert assignments == {downstream: 42} + assert evidence[downstream]["method"] == "same-edits" + with pytest.raises(ValueError, match="#41.*#42"): + m.match(source, base, downstream, {41: _pr(base, downstream), 42: _pr(base, upstream)}) + + +@pytest.mark.parametrize("added", [True, False], ids=["added", "removed"]) +def test_changed_function_definitions_are_not_ignored(source: Path, added: bool) -> None: + original = "def first():\n pass\n\n\ndef second():\n pass\n" + base = _commit(source, "# base\n" if added else original) + downstream = _commit(source, "def first():\n pass\n") + _git(source, "checkout", "-q", "-b", "upstream", base) + upstream = _commit(source, "def second():\n pass\n") + with pytest.raises(ValueError, match="no exact match"): + m.match(source, base, downstream, {42: _pr(base, upstream)}) + + +def test_hunk_like_changed_text_is_not_ignored(source: Path) -> None: + base = _commit(source, "# base\n", "patch.txt") + downstream = _commit(source, "@@ -1 +1 @@ def first():\n", "patch.txt") + _git(source, "checkout", "-q", "-b", "upstream", base) + upstream = _commit(source, "@@ -1 +1 @@ def second():\n", "patch.txt") + with pytest.raises(ValueError, match="no exact match"): + m.match(source, base, downstream, {42: _pr(base, upstream)}) + + +def test_matching_squashed_series_and_ambiguous_duplicates(source: Path) -> None: + base = _commit(source, "VALUE = 0\n") + first = _commit(source, "VALUE = 1\n") + second = _commit(source, "VALUE = 2\n") + _git(source, "checkout", "-q", "-b", "upstream", base) + squash = _commit(source, "VALUE = 2\n") + assignments, evidence = m.match(source, base, second, {42: _pr(base, squash)}) + assert assignments == {first: 42, second: 42} + assert evidence[first]["method"] == "source-series-aggregate" + with pytest.raises(ValueError, match="#42.*#43"): + m.match(source, base, second, {42: _pr(base, squash), 43: _pr(base, squash)}) + + +def test_author_can_resolve_adapted_commit(source: Path) -> None: + base = _commit(source, "VALUE = 0\n") + downstream = _commit(source, "VALUE = 1\n") + _git(source, "checkout", "-q", "-b", "upstream", base) + upstream = _commit(source, "VALUE = 2\n") + assignments, evidence = m.match( + source, + base, + downstream, + {42: _pr(base, upstream)}, + {downstream: {"upstream_pr": 42, "reason": "downstream ABI adaptation"}}, + ) + assert assignments == {downstream: 42} + assert evidence[downstream]["refresh_policy"] == "retain-until-verified" + with pytest.raises(ValueError, match="outside"): + m.match(source, base, downstream, {42: _pr(base, upstream)}, {base: {"upstream_pr": 42}}) + + +@pytest.mark.parametrize("different", ["a \n", "a\r\n", "a\v\n"]) +def test_similarity_preserves_line_bytes(different: str) -> None: + assert m._similarity("a\n", different) == 0 + assert m._similarity("", "") == 0 + + +def test_similarity_keeps_repeated_lines() -> None: + first = "different\n" + "repeat\n" * 299 + second = "another\n" + "repeat\n" * 299 + assert m._similarity(first, second) == pytest.approx(299 / 300) + + +@pytest.mark.parametrize("changed_lines", [1, 2, 3]) +def test_similarity_threshold_and_provenance(source: Path, changed_lines: int) -> None: + base = _commit(source, "") + lines = [f"VALUE_{index} = {index}\n" for index in range(16)] + downstream = _commit(source, "".join(lines)) + _git(source, "checkout", "-q", "-b", "upstream", base) + upstream = _commit(source, "".join(lines[:-changed_lines]) + "adapted\n" * changed_lines) + expected = (20 - changed_lines) / 20 + if expected < 0.9: + with pytest.raises(ValueError, match="no exact match") as failure: + m.match(source, base, downstream, {42: _pr(base, upstream)}) + assert "upstream PR #42: 85.00%" in str(failure.value) + assert "no PR meets the 90.00% similarity threshold" in str(failure.value) + return + assignments, evidence = m.match(source, base, downstream, {42: _pr(base, upstream)}) + assert assignments == {downstream: 42} + assert evidence[downstream] == { + "method": "similar-edits", + "upstream_base": base, + "upstream_head": upstream, + "upstream_pr": 42, + "similarity": expected, + "similarity_threshold": 0.9, + "similarity_metric": "sequence-matcher-lines-v1", + "refresh_policy": "retain-until-verified", + "comparison": { + "source_base": base, + "source_head": downstream, + "upstream_base": base, + "upstream_head": upstream, + }, + } + assert f"**{expected:.2%}** — accepted" in m.similarity_feedback(evidence) + + +def test_multiple_similar_prs_remain_ambiguous(source: Path) -> None: + base = _commit(source, "") + lines = [f"VALUE_{index} = {index}\n" for index in range(16)] + downstream = _commit(source, "".join(lines)) + _git(source, "checkout", "-q", "-b", "upstream", base) + upstream = _commit(source, "".join(lines[:-1]) + "adapted\n") + with pytest.raises(ValueError, match="#41.*#42") as failure: + m.match(source, base, downstream, {41: _pr(base, upstream), 42: _pr(base, upstream)}) + assert "#41: 95.00%" in str(failure.value) + assert "#42: 95.00%" in str(failure.value) + assert "multiple PRs meet the 90.00% similarity threshold" in str(failure.value) + + +def test_exact_matches_take_priority_over_similar_prs(source: Path) -> None: + base = _commit(source, "") + lines = [f"VALUE_{index} = {index}\n" for index in range(16)] + first = _commit(source, "".join(lines)) + second = _commit(source, "second file\n", "second.py") + _git(source, "checkout", "-q", "-b", "upstream", base) + approximate = _commit(source, "".join(lines[:-1]) + "adapted\n") + upstream = _commit(source, "second file\n", "second.py") + assignments, evidence = m.match( + source, base, second, {41: _pr(base, first), 42: _pr(base, upstream)} + ) + assert m._similarity(m._patch(source, base, first), m._patch(source, base, approximate)) == 0.95 + assert assignments == {first: 41, second: 42} + assert all("similarity" not in item for item in evidence.values()) + + +def test_approximate_series_cannot_hide_unmatched_individual_commits(source: Path) -> None: + base = _commit(source, "", "src/code.py") + lines = [f"VALUE_{index} = {index}\n" for index in range(16)] + first = _commit(source, "".join(lines[:8]), "src/code.py") + second = _commit(source, "".join(lines), "src/code.py") + excluded = _commit(source, "tests only\n", "tests/test_code.py") + _git(source, "checkout", "-q", "-b", "upstream", base) + upstream = _commit(source, "".join(lines[:-1]) + "adapted\n", "src/code.py") + assert m._similarity(m._patch(source, base, second), m._patch(source, base, upstream)) == 0.95 + with pytest.raises(ValueError, match="no exact match") as failure: + m.match( + source, base, excluded, {42: _pr(base, upstream)}, source="src", include=("**/*.py",) + ) + assert first in str(failure.value) and second in str(failure.value) + assert excluded not in str(failure.value) + + +def test_similarity_error_also_reports_accepted_matches(source: Path) -> None: + base = _commit(source, "") + lines = [f"VALUE_{index} = {index}\n" for index in range(16)] + first = _commit(source, "".join(lines)) + second = _commit(source, "unrelated\n" * 16, "second.py") + _git(source, "checkout", "-q", "-b", "upstream", base) + upstream = _commit(source, "".join(lines[:-1]) + "adapted\n") + with pytest.raises(ValueError, match="no exact match") as failure: + m.match(source, base, second, {42: _pr(base, upstream)}) + assert f"`{second}`: no exact match" in str(failure.value) + assert f"`{first}`: upstream PR #42, similarity **95.00%** — accepted" in str(failure.value) + + +class FakeGitHub(p.GitHub): + def __init__(self) -> None: + super().__init__("org/consumer", "toy", "org/library") + self.pr = { + "number": 7, + "user": {"login": "author"}, + "state": "open", + "merged": False, + "head": {"sha": "a" * 40}, + "base": {"ref": "main"}, + "body": "", + "updated_at": "2099-01-01T00:00:00Z", + } + self.comments: list[dict] = [] + self.reviews: list[dict] = [] + self.writes: list[tuple[str, str, dict]] = [] + self.merge_base_sha = "c" * 40 + + def api( + self, endpoint: str, *, method: str = "GET", payload: dict | None = None + ) -> dict | list: + if method != "GET": + self.writes.append((endpoint, method, payload)) + if endpoint == "user": + return {"login": "operator"} + if endpoint.startswith("repos/org/consumer/compare/"): + return {"merge_base_commit": {"sha": self.merge_base_sha}} + if endpoint.endswith("/pulls/7"): + return copy.deepcopy(self.pr) + if "/pulls?" in endpoint: + return [copy.deepcopy(self.pr)] + if "/files?" in endpoint: + return [{"filename": p._LOCK}] + if "/reviews/" in endpoint and endpoint.endswith("/dismissals"): + number = int(endpoint.split("/")[-2]) + self.reviews[number - 1]["state"] = "DISMISSED" + return {} + if "/reviews/" in endpoint and method == "PUT": + number = int(endpoint.rsplit("/", 1)[1]) + self.reviews[number - 1]["body"] = payload["body"] + return copy.deepcopy(self.reviews[number - 1]) + if "/reviews" in endpoint: + if method == "GET": + return copy.deepcopy(self.reviews) + self.reviews.append( + { + "id": len(self.reviews) + 1, + "user": {"login": "operator"}, + "body": payload["body"], + "state": "CHANGES_REQUESTED", + } + ) + return self.reviews[-1] + if "/comments/" in endpoint: + number = int(endpoint.rsplit("/", 1)[1]) + self.comments[number - 1]["body"] = payload["body"] + return {} + if "/comments" in endpoint: + if method == "GET": + return copy.deepcopy(self.comments) + self.comments.append( + { + "id": len(self.comments) + 1, + "user": {"login": "operator"}, + "body": payload["body"], + } + ) + return self.comments[-1] + raise AssertionError((endpoint, method, payload)) + + +@pytest.fixture +def monitor(tmp_path: Path): + b._STOP.clear() + args = argparse.Namespace( + canonical_repo="operator/library", + canonical_branch="dev", + refresh_branch=[], + migrate_canonical_branch_from=None, + fork="operator/consumer", + repo=tmp_path, + workdir=tmp_path, + publish=True, + since=None, + ) + store = b.Store(tmp_path / "state.sqlite") + gh = FakeGitHub() + instance = b.Monitor(args, gh, store) + yield instance + store.close() + b._STOP.clear() + + +@pytest.fixture +def migration(monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch): + args = copy.copy(monitor.args) + args.canonical_branch = "dev-new" + args.migrate_canonical_branch_from = "dev" + pin = _vendor("b" * 40, "https://github.com/operator/library.git", "dev-new") + monkeypatch.setattr(p, "_main_sha", lambda gh: "c" * 40) + monkeypatch.setattr(p, "_lock_at", lambda *args: pin) + original_get = monitor.gh.get + monkeypatch.setattr( + monitor.gh, + "get", + lambda endpoint: {"object": {"sha": "b" * 40}} + if "/git/ref/" in endpoint + else original_get(endpoint), + ) + monkeypatch.setattr(monitor.gh, "compare", lambda *args: {"status": "identical"}, raising=False) + monkeypatch.setattr(b, "_pages", lambda *args: iter([])) + monkeypatch.setattr(p, "_git", lambda *args: "") + return args, pin + + +def test_branch_migration_preserves_workdir_and_reconciles_again( + monitor: b.Monitor, + migration, + caplog: pytest.LogCaptureFixture, +) -> None: + caplog.set_level(logging.INFO, logger="vendor-bot") + args, _ = migration + store = monitor.store + for key, value in { + "complete:7": "old-pr", + "promotion:8": 88, + "observation:7": {"state": "completed"}, + "problem:8": "old problem", + "bootstrapped": True, + "cursor": "old cursor", + "last_poll": "old", + "health": {"last_successful_poll": "old", "last_completed_poll": "old"}, + "tracked": [7, 8], + "candidate:7": {"head": "a" * 40, "relevant": True}, + }.items(): + store.put(key, value) + before_since = store.get("since") + (args.workdir / "bot.log").write_text("previous log\n") + restarted = b.Monitor(args, monitor.gh, store) + assert store.get("settings")["canonical_branch"] == "dev-new" + assert store.get("complete:7") is None + assert store.get("promotion:8") is None + assert store.get("problem:8") is None + assert store.get("cursor") is None and store.get("bootstrapped") is None + assert store.get("tracked") == [7, 8] + assert store.get("candidate:7")["relevant"] + assert store.get("since") == before_since + assert store.get("branch_migrations")[0]["previous_results"]["complete:7"] == "old-pr" + assert store.get("migration")["consumer_revision"] == "c" * 40 + assert restarted.progress.state["last_successful_poll"] is None + assert restarted.progress.state["last_completed_poll"] is None + assert (args.workdir / "bot.log").read_text() == "previous log\n" + assert not monitor.gh.writes + assert "dev -> dev-new; rediscovery required" in caplog.text + status = b._status(args.workdir) + assert status["canonical_branch"] == "dev-new" + assert status["migration"]["from"] == "dev" + + +def test_branch_migration_retry_does_not_reset_new_progress(monitor: b.Monitor, migration) -> None: + args, _ = migration + b.Monitor(args, monitor.gh, monitor.store) + monitor.store.put("cursor", "new progress") + b.Monitor(args, monitor.gh, monitor.store) + assert monitor.store.get("cursor") == "new progress" + assert len(monitor.store.get("branch_migrations")) == 1 + args.migrate_canonical_branch_from = None + b.Monitor(args, monitor.gh, monitor.store) + assert monitor.store.get("cursor") == "new progress" + + +@pytest.mark.parametrize("old", [None, "wrong"]) +def test_branch_migration_requires_explicit_old_branch(monitor: b.Monitor, migration, old) -> None: + args, _ = migration + args.migrate_canonical_branch_from = old + with pytest.raises(ValueError, match="--migrate-canonical-branch-from dev"): + b.Monitor(args, monitor.gh, monitor.store) + assert monitor.store.get("settings")["canonical_branch"] == "dev" + + +@pytest.mark.parametrize("key", ["canonical_repo", "fork", "repo"]) +def test_branch_migration_cannot_change_identity(monitor: b.Monitor, migration, key: str) -> None: + args, _ = migration + setattr(args, key, "other/value") + with pytest.raises(ValueError, match="different operator/vendor"): + b.Monitor(args, monitor.gh, monitor.store) + assert monitor.store.get("settings")["canonical_branch"] == "dev" + + +@pytest.mark.parametrize( + "branch,url", + [ + ("dev", "https://github.com/operator/library.git"), + ("dev-new", "https://github.com/other/library.git"), + ], +) +def test_branch_migration_requires_merged_refresh( + monitor: b.Monitor, + migration, + monkeypatch: pytest.MonkeyPatch, + branch: str, + url: str, +) -> None: + args, _ = migration + monkeypatch.setattr(p, "_lock_at", lambda *args: _vendor("b" * 40, url, branch)) + with pytest.raises(ValueError, match="merged base-branch lock"): + b.Monitor(args, monitor.gh, monitor.store) + assert monitor.store.get("settings")["canonical_branch"] == "dev" + + +@pytest.mark.parametrize("status", ["behind", "diverged"]) +def test_branch_migration_rejects_unreachable_pin( + monitor: b.Monitor, + migration, + monkeypatch: pytest.MonkeyPatch, + status: str, +) -> None: + args, _ = migration + monkeypatch.setattr(monitor.gh, "compare", lambda *args: {"status": status}) + with pytest.raises(ValueError, match="does not contain"): + b.Monitor(args, monitor.gh, monitor.store) + + +def test_branch_migration_rejects_lock_race( + monitor: b.Monitor, + migration, + monkeypatch: pytest.MonkeyPatch, +) -> None: + args, pin = migration + locks = iter([pin, dataclasses.replace(pin, commit="d" * 40)]) + monkeypatch.setattr(p, "_lock_at", lambda *args: next(locks)) + with pytest.raises(ValueError, match="changed during migration"): + b.Monitor(args, monitor.gh, monitor.store) + assert monitor.store.get("settings")["canonical_branch"] == "dev" + + +@pytest.mark.parametrize("state", ["open", "absent", "closed"]) +@pytest.mark.parametrize("origin", ["remote", "local", "pr"]) +def test_branch_migration_checks_pending_promotions( + monitor: b.Monitor, + migration, + monkeypatch: pytest.MonkeyPatch, + state: str, + origin: str, +) -> None: + args, _ = migration + branch = "chore/toy-promote-7-aaaaaaaaaaaa" + pr = { + "head": {"ref": branch, "sha": "a" * 40}, + "base": {"ref": "main"}, + "state": state, + "html_url": "https://github.com/org/consumer/pull/88", + } + + def pages(gh, endpoint): + if endpoint.endswith("/branches"): + return iter( + [{"name": branch, "commit": {"sha": "a" * 40}}] if origin == "remote" else [] + ) + if "state=open" in endpoint: + return iter([pr] if origin == "pr" else []) + return iter([pr] if state != "absent" else []) + + monkeypatch.setattr(b, "_pages", pages) + monkeypatch.setattr( + p, "_git", lambda *args: f"{branch} {'a' * 40}" if origin == "local" else "" + ) + monkeypatch.setattr( + monitor.gh, "optional", lambda endpoint: {"commit": {"message": "record"}}, raising=False + ) + monkeypatch.setattr( + p, + "_record_from_message", + lambda message: { + "vendor": "toy", + "canonical_branch": "dev", + "consumer_pr": "https://github.com/org/consumer/pull/7", + "consumer_base": "main", + }, + ) + if state == "closed": + b.Monitor(args, monitor.gh, monitor.store) + else: + with pytest.raises(ValueError, match="Unresolved old-branch promotion"): + b.Monitor(args, monitor.gh, monitor.store) + assert monitor.store.get("settings")["canonical_branch"] == "dev" + assert not monitor.gh.writes + + +@pytest.mark.parametrize("base_branch", ["main", "release/1.3"]) +@pytest.mark.parametrize("other_scope", ["base", "repository"]) +@pytest.mark.parametrize("origin", ["local", "remote"]) +def test_branch_migration_ignores_promotions_for_other_consumers( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, + base_branch: str, + other_scope: str, + origin: str, +) -> None: + monitor.gh.base_branch = base_branch + other_base = "release/1.3" if base_branch == "main" else "main" + branch = "chore/toy-promote-7-aaaaaaaaaaaa" + record = { + "version": 1, + "vendor": "toy", + "canonical_branch": "dev", + "consumer_pr": "https://github.com/other/consumer/pull/7" + if other_scope == "repository" + else "https://github.com/org/consumer/pull/7", + "consumer_base": other_base if other_scope == "base" else base_branch, + } + + def pages(gh, endpoint): + if endpoint.endswith("/branches"): + return iter( + [{"name": branch, "commit": {"sha": "a" * 40}}] if origin == "remote" else [] + ) + assert endpoint.endswith("/pulls?state=open"), endpoint + return iter([]) + + monkeypatch.setattr(b, "_pages", pages) + monkeypatch.setattr( + p, "_git", lambda *args: f"{branch} {'a' * 40}" if origin == "local" else "" + ) + monkeypatch.setattr( + monitor.gh, + "optional", + lambda endpoint: {"commit": {"message": p._record_text(record)}}, + ) + monitor._check_pending_promotions("dev") + assert not monitor.gh.writes + + +@pytest.mark.parametrize("field", ["consumer_pr", "consumer_base"]) +@pytest.mark.parametrize("value", [None, "", 123]) +def test_branch_migration_rejects_ambiguous_consumer_provenance( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, + field: str, + value: object, +) -> None: + branch = "chore/toy-promote-7-aaaaaaaaaaaa" + record = { + "version": 1, + "vendor": "toy", + "canonical_branch": "dev", + "consumer_pr": "https://github.com/org/consumer/pull/7", + "consumer_base": "main", + field: value, + } + monkeypatch.setattr(b, "_pages", lambda *args: iter([])) + monkeypatch.setattr(p, "_git", lambda *args: f"{branch} {'a' * 40}") + monkeypatch.setattr( + monitor.gh, + "optional", + lambda endpoint: {"commit": {"message": p._record_text(record)}}, + ) + with pytest.raises(ValueError, match="Missing or invalid consumer provenance"): + monitor._check_pending_promotions("dev") + assert not monitor.gh.writes + + +def test_branch_migration_sqlite_transaction_rolls_back(monitor: b.Monitor, migration) -> None: + import sqlite3 + + args, _ = migration + monitor.store.put("cursor", "preserved") + monitor.store.put("complete:7", "preserved") + monitor.store.connection.execute( + "CREATE TRIGGER fail_migration BEFORE INSERT ON state WHEN NEW.key = 'migration' " + "BEGIN SELECT RAISE(ABORT, 'injected failure'); END" + ) + with pytest.raises(sqlite3.IntegrityError, match="injected failure"): + b.Monitor(args, monitor.gh, monitor.store) + assert monitor.store.get("settings")["canonical_branch"] == "dev" + assert monitor.store.get("cursor") == "preserved" + assert monitor.store.get("complete:7") == "preserved" + assert monitor.store.get("branch_migrations") is None + monitor.store.connection.execute("DROP TRIGGER fail_migration") + b.Monitor(args, monitor.gh, monitor.store) + + +def test_branch_migration_cache_loss_uses_normal_start( + monitor: b.Monitor, migration, tmp_path: Path +) -> None: + args, _ = migration + with contextlib.closing(b.Store(tmp_path / "fresh.sqlite")) as store: + with pytest.raises(ValueError, match="fresh/cache-loss start"): + b.Monitor(args, monitor.gh, store) + args.migrate_canonical_branch_from = None + b.Monitor(args, monitor.gh, store) + assert store.get("settings")["canonical_branch"] == "dev-new" + assert store.get("bootstrapped") is None + + +def test_feedback_requests_changes_once_and_clears_only_metadata_review(monitor: b.Monitor) -> None: + gh = monitor.gh + monitor._feedback(gh.pr, _issue()) + monitor._feedback(gh.pr, _issue()) + assert len(gh.comments) == len(gh.reviews) == 1 + assert "PR description" in gh.comments[0]["body"] + assert "resolutions:" not in gh.comments[0]["body"] + assert "**Fix:**" in gh.comments[0]["body"] + assert gh.writes[1][2]["event"] == "REQUEST_CHANGES" + monitor._feedback(gh.pr, None) + assert gh.reviews[0]["state"] == "DISMISSED" + assert all(payload.get("event") != "APPROVE" for _, _, payload in gh.writes) + + +def test_bot_does_not_override_operator_manual_review(monitor: b.Monitor) -> None: + monitor.gh.reviews.append( + { + "id": 1, + "user": {"login": "operator"}, + "body": "Manual code review", + "state": "CHANGES_REQUESTED", + } + ) + monitor._feedback(monitor.gh.pr, _issue("Missing metadata")) + monitor._feedback(monitor.gh.pr, None) + assert monitor.gh.reviews[0]["state"] == "CHANGES_REQUESTED" + assert len(monitor.gh.reviews) == 1 + + +def test_dry_run_and_self_review(monitor: b.Monitor) -> None: + monitor.args.publish = False + monitor._feedback(monitor.gh.pr, _issue("Missing metadata")) + assert not monitor.gh.writes + monitor.args.publish = True + monitor.gh.pr["user"]["login"] = "operator" + monitor._feedback(monitor.gh.pr, _issue("Missing metadata")) + assert monitor.gh.comments + assert not monitor.gh.reviews + + +def test_feedback_rechecks_mutable_description(monitor: b.Monitor) -> None: + stale = copy.deepcopy(monitor.gh.pr) + monitor.gh.pr["body"] = "updated while checking" + monitor._feedback(stale, _issue("Missing metadata")) + assert not monitor.gh.writes + + +@pytest.mark.parametrize("missing", [(m.BEGIN, m.END), (m.BEGIN,), (m.END,)]) +def test_posted_marker_feedback_is_specific_and_recovers_without_cache( + monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch, missing: tuple[str, ...] +) -> None: + previous = _vendor("a" * 40, "https://github.com/operator/library.git", "dev") + reviewed = _vendor("b" * 40, "https://github.com/author/library.git", "temporary") + monkeypatch.setattr(monitor, "_inputs", lambda pr: (previous, reviewed)) + valid = m.template("toy", "org/library") + monitor.gh.pr["body"] = valid + for marker in missing: + monitor.gh.pr["body"] = monitor.gh.pr["body"].replace(marker, "") + monitor.inspect(7) + body = monitor.gh.comments[0]["body"] + for marker in missing: + assert f"`{marker}`" in body + assert "**Fix:**" in body and "PR's description—not a comment" in body + assert "resolutions:" not in body and "exactly one" not in body + assert monitor.gh.reviews[0]["state"] == "CHANGES_REQUESTED" + writes = copy.deepcopy(monitor.gh.writes) + with contextlib.closing(b.Store(":memory:")) as store: + restarted = b.Monitor(monitor.args, monitor.gh, store) + monkeypatch.setattr(restarted, "_inputs", lambda pr: (previous, reviewed)) + restarted.inspect(7) + assert monitor.gh.writes == writes + monitor.gh.pr["body"] = valid + monkeypatch.setattr(m, "resolve", lambda *args: ({reviewed.commit: 123}, {})) + restarted.inspect(7) + assert len(monitor.gh.comments) == 1 + assert monitor.gh.reviews[0]["state"] == "DISMISSED" + assert not any(payload.get("event") == "APPROVE" for _, _, payload in monitor.gh.writes) + + +@pytest.mark.parametrize("stage", ["inputs", "resolve"]) +@pytest.mark.parametrize("error_type", [ValueError, RuntimeError, b.manage.VendorError]) +def test_unrecognized_failures_are_operator_errors_not_author_feedback( + monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch, stage: str, error_type: type[Exception] +) -> None: + previous = _vendor("a" * 40, "https://github.com/operator/library.git", "dev") + reviewed = _vendor("b" * 40, "https://github.com/author/library.git", "temporary") + monitor.gh.pr["body"] = m.template("toy", "org/library") + monkeypatch.setattr(monitor, "_inputs", lambda pr: (previous, reviewed)) + + def fail(*args): + raise error_type("Operator failure; do not request author changes") + + if stage == "inputs": + monkeypatch.setattr(monitor, "_inputs", fail) + else: + monkeypatch.setattr(m, "resolve", fail) + monitor.poll() + assert not monitor.gh.writes + health = monitor.store.get("health") + assert health["counts"]["errors"] == 1 + assert "Operator failure" in health["last_error"]["message"] + + +@pytest.mark.parametrize("change", ["other-vendor", "patch-only", "vendor-created"]) +def test_refresh_ignores_prs_without_selected_source_pin_change( + monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch, change: str +) -> None: + current = _vendor("1" * 40, "https://github.com/operator/library.git", "dev") + old = _vendor("2" * 40, "https://github.com/operator/library.git", "dev-old") + original = {"toy": old.to_mapping()} + reviewed = copy.deepcopy(original) + if change == "other-vendor": + reviewed["other"] = current.to_mapping() + elif change == "patch-only": + reviewed["toy"]["digest"] = "sha256-tree-v1:" + "3" * 64 + else: + original = {} + monitor.gh.merge_base_sha = "d" * 40 + locks = {"c" * 40: {"toy": current.to_mapping()}, "d" * 40: original, "a" * 40: reviewed} + monkeypatch.setattr(p, "_main_sha", lambda gh: "c" * 40) + monkeypatch.setattr(p, "_lock_document_at", lambda gh, repo, sha: {"vendors": locks[sha]}) + monitor.inspect(7) + assert not monitor.gh.writes + assert monitor._inputs(monitor.gh.pr) is None + with contextlib.closing(b.Store(monitor.args.workdir / "fresh.sqlite")) as store: + b.Monitor(monitor.args, monitor.gh, store).inspect(7) + assert not monitor.gh.writes + + +def test_refresh_source_update_uses_current_canonical_as_promotion_baseline( + monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch +) -> None: + current = _vendor("1" * 40, "https://github.com/operator/library.git", "dev") + old = _vendor("2" * 40, "https://github.com/operator/library.git", "dev-old") + reviewed = _vendor("3" * 40, "https://github.com/author/library.git", "temporary") + monitor.gh.merge_base_sha = "d" * 40 + locks = {"c" * 40: current, "d" * 40: old, "a" * 40: reviewed} + monkeypatch.setattr(p, "_main_sha", lambda gh: "c" * 40) + monkeypatch.setattr( + p, "_lock_document_at", lambda gh, repo, sha: {"vendors": {"toy": locks[sha].to_mapping()}} + ) + assert monitor._inputs(monitor.gh.pr) == (current, reviewed) + assert not monitor.gh.writes + + +@pytest.fixture +def refresh_candidate(monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch): + previous = _vendor("1" * 40, "https://github.com/operator/library.git", "dev") + reviewed = _vendor("2" * 40, "https://github.com/operator/library.git", "dev-new") + locks = {"c" * 40: previous, "a" * 40: reviewed} + monkeypatch.setattr(p, "_main_sha", lambda gh: "c" * 40) + monkeypatch.setattr( + p, "_lock_document_at", lambda gh, repo, sha: {"vendors": {"toy": locks[sha].to_mapping()}} + ) + original_get = monitor.gh.get + monkeypatch.setattr( + monitor.gh, + "get", + lambda endpoint: {"parents": [{"sha": "c" * 40}]} + if endpoint.endswith("/commits/" + "a" * 40) + else original_get(endpoint), + ) + return locks + + +@pytest.mark.parametrize("base_branch", ["main", "release/1.3"]) +@pytest.mark.parametrize("redirect", ["repository", "branch", "both"]) +@pytest.mark.parametrize("caught_up", [False, True]) +def test_same_commit_reference_redirect_requests_canonical_lock_not_metadata( + monitor: b.Monitor, refresh_candidate, base_branch: str, redirect: str, caught_up: bool +) -> None: + gh = monitor.gh + gh.base_branch = gh.pr["base"]["ref"] = base_branch + previous = refresh_candidate["c" * 40] + reviewed = dataclasses.replace( + previous, + url="https://github.com/author/library.git" if redirect != "branch" else previous.url, + branch="temporary" if redirect != "repository" else previous.branch, + ) + refresh_candidate["a" * 40] = reviewed + if caught_up: + gh.merge_base_sha = "d" * 40 + refresh_candidate["d" * 40] = dataclasses.replace(previous, commit="0" * 40) + gh.pr["body"] = "Invalid/missing promotion metadata must not obscure the lock fix." + with contextlib.closing(b.Store(":memory:")) as store: + restarted = b.Monitor(monitor.args, gh, store) + restarted.inspect(7) + assert store.get("observation:7")["state"] == "blocked" + body = gh.comments[0]["body"] + assert "Temporary source reference adds no new vendor commit" in body + assert previous.commit in body and base_branch in body + assert p._LOCK in body and "vendors.toy" in body + assert previous.url in body and f"{previous.branch}" in body + assert "keep the commit and other lock fields, vendored files, and patch unchanged" in body + assert "No upstream-PR metadata" in body + assert "vendor-promotion:start" not in body + assert gh.reviews[0]["state"] == "CHANGES_REQUESTED" + writes = copy.deepcopy(gh.writes) + with contextlib.closing(b.Store(":memory:")) as store: + b.Monitor(monitor.args, gh, store).inspect(7) + assert gh.writes == writes + + +@pytest.mark.parametrize( + "case", ["canonical", "equivalent-url", "refresh", "inherited", "merged", "promotion"] +) +def test_same_commit_reference_non_redirects_do_not_request_changes( + monitor: b.Monitor, refresh_candidate, case: str +) -> None: + previous = refresh_candidate["c" * 40] + reviewed = previous + if case == "equivalent-url": + reviewed = dataclasses.replace(previous, url="https://github.com/OPERATOR/library") + elif case == "refresh": + reviewed = dataclasses.replace(previous, branch="dev-new") + monitor.args.refresh_branch = ["dev-new"] + elif case in ("inherited", "merged"): + reviewed = dataclasses.replace(previous, url="https://github.com/author/library.git") + if case == "inherited": + monitor.gh.merge_base_sha = "d" * 40 + refresh_candidate["d" * 40] = reviewed + else: + monitor.gh.pr.update(merged=True, state="closed", merge_commit_sha="a" * 40) + elif case == "promotion": + refresh_candidate["c" * 40] = dataclasses.replace( + previous, url="https://github.com/author/library.git", branch="temporary" + ) + refresh_candidate["a" * 40] = reviewed + with contextlib.closing(b.Store(":memory:")) as store: + b.Monitor(monitor.args, monitor.gh, store).inspect(7) + assert store.get("observation:7")["state"] == "ignored" + assert not monitor.gh.writes + + +def test_same_commit_reference_reconciles_stale_feedback_then_clears_after_fix( + monitor: b.Monitor, refresh_candidate +) -> None: + gh = monitor.gh + monitor._feedback(gh.pr, _issue("Source update is not a fast-forward.")) + previous = refresh_candidate["c" * 40] + refresh_candidate["a" * 40] = dataclasses.replace( + previous, url="https://github.com/author/library.git", branch="temporary" + ) + monitor.inspect(7) + assert len(gh.comments) == len(gh.reviews) == 1 + assert all("adds no new vendor commit" in item["body"] for item in (*gh.comments, *gh.reviews)) + assert "fast-forward" not in gh.reviews[0]["body"] + assert gh.reviews[0]["state"] == "CHANGES_REQUESTED" + assert any( + method == "PUT" and endpoint.endswith("/reviews/1") for endpoint, method, _ in gh.writes + ) + refresh_candidate["a" * 40] = previous + for _ in range(2): + with contextlib.closing(b.Store(":memory:")) as store: + b.Monitor(monitor.args, gh, store).inspect(7) + assert store.get("observation:7")["state"] == "ignored" + assert store.get("problem:7") is None + assert len(gh.comments) == len(gh.reviews) == 1 + assert "does not require vendor promotion" in gh.comments[0]["body"] + assert "metadata and source attribution are valid" not in gh.comments[0]["body"] + assert gh.reviews[0]["state"] == "DISMISSED" + assert not any(payload.get("event") == "APPROVE" for _, _, payload in gh.writes) + writes = copy.deepcopy(gh.writes) + monitor.inspect(7) + assert gh.writes == writes + + +@pytest.mark.parametrize("remaining", ["comment", "review", "both", "neither"]) +def test_ignored_pr_cleans_remote_feedback_without_sqlite( + monitor: b.Monitor, refresh_candidate, remaining: str +) -> None: + monitor._feedback(monitor.gh.pr, _issue("Missing block")) + if remaining in ("review", "neither"): + monitor.gh.comments.clear() + if remaining in ("comment", "neither"): + monitor.gh.reviews.clear() + monitor.gh.writes.clear() + monitor.args.refresh_branch = ["dev-new"] + for _ in range(2): + with contextlib.closing(b.Store(":memory:")) as store: + b.Monitor(monitor.args, monitor.gh, store).inspect(7) + assert store.get("observation:7")["state"] == "ignored" + if remaining == "neither": + assert not monitor.gh.writes + else: + assert len(monitor.gh.comments) == 1 + assert "no longer applies" in monitor.gh.comments[0]["body"] + assert "not** a code-review approval" in monitor.gh.comments[0]["body"] + assert all(review["state"] == "DISMISSED" for review in monitor.gh.reviews) + assert len(monitor.gh.writes) == (2 if remaining in ("review", "both") else 1) + + +@pytest.mark.parametrize("existing", ["manual", "other-actor", "other-vendor"]) +def test_ignored_pr_preserves_unrelated_comments_and_reviews( + monitor: b.Monitor, refresh_candidate, existing: str +) -> None: + actor = "human" if existing == "other-actor" else monitor.actor + body = ( + "Manual review" + if existing == "manual" + else "\nOther vendor" + if existing == "other-vendor" + else f"{monitor.marker}\nAnother operator's feedback" + ) + monitor.gh.comments.append({"id": 1, "user": {"login": actor}, "body": body}) + monitor.gh.reviews.append( + {"id": 1, "user": {"login": actor}, "body": body, "state": "CHANGES_REQUESTED"} + ) + monitor.args.refresh_branch = ["dev-new"] + b.Monitor(monitor.args, monitor.gh, monitor.store).inspect(7) + assert not monitor.gh.writes + assert monitor.gh.reviews[0]["state"] == "CHANGES_REQUESTED" + + +@pytest.mark.parametrize("change", ["head", "body", "base", "closed", "dry-run"]) +def test_ignored_feedback_cleanup_respects_concurrent_changes_and_dry_run( + monitor: b.Monitor, change: str +) -> None: + monitor._feedback(monitor.gh.pr, _issue("Missing block")) + snapshot = copy.deepcopy(monitor.gh.pr) + if change == "head": + monitor.gh.pr["head"]["sha"] = "b" * 40 + elif change == "body": + monitor.gh.pr["body"] = "edited" + elif change == "base": + monitor.gh.pr["base"]["ref"] = "release/1.3" + elif change == "closed": + monitor.gh.pr["state"] = "closed" + else: + monitor.args.publish = False + writes = copy.deepcopy(monitor.gh.writes) + monitor._feedback(snapshot, None, no_promotion=True) + assert monitor.gh.writes == writes + assert monitor.gh.reviews[0]["state"] == "CHANGES_REQUESTED" + + +@pytest.mark.parametrize("base_branch", ["main", "release/1.3"]) +@pytest.mark.parametrize("merged", [False, True]) +def test_explicit_refresh_target_never_requests_metadata_or_promotes( + monitor: b.Monitor, refresh_candidate, base_branch: str, merged: bool +) -> None: + monitor.gh.base_branch = base_branch + monitor.gh.pr["base"]["ref"] = base_branch + monitor.gh.pr.update(merged=merged, merge_commit_sha="a" * 40) + monitor.args.refresh_branch = ["dev-new"] + for _ in range(2): + with contextlib.closing(b.Store(":memory:")) as store: + restarted = b.Monitor(monitor.args, monitor.gh, store) + restarted.inspect(7) + assert store.get("observation:7")["state"] == "ignored" + assert store.get("refresh_branches") == ["dev-new"] + assert not monitor.gh.writes + + +@pytest.mark.parametrize("untrusted", ["branch", "repository"]) +def test_refresh_targets_are_explicit_and_repository_scoped( + monitor: b.Monitor, refresh_candidate, untrusted: str +) -> None: + if untrusted == "repository": + monitor.args.refresh_branch = ["dev-new"] + refresh_candidate["a" * 40] = dataclasses.replace( + refresh_candidate["a" * 40], url="https://github.com/author/library.git" + ) + restarted = b.Monitor(monitor.args, monitor.gh, monitor.store) + assert restarted._inputs(monitor.gh.pr) == ( + refresh_candidate["c" * 40], + refresh_candidate["a" * 40], + ) + restarted.inspect(7) + assert "vendor-promotion:start" in monitor.gh.comments[0]["body"] + assert monitor.gh.reviews[0]["state"] == "CHANGES_REQUESTED" + + +def test_refresh_policy_can_change_without_migrating_workdir( + monitor: b.Monitor, refresh_candidate +) -> None: + monitor.store.put("cursor", "preserved") + monitor.store.put("complete:8", "preserved") + for branches in (["dev-new", "dev-new"], []): + monitor.args.refresh_branch = branches + restarted = b.Monitor(monitor.args, monitor.gh, monitor.store) + assert (restarted._inputs(monitor.gh.pr) is None) == bool(branches) + assert monitor.store.get("settings")["canonical_branch"] == "dev" + assert monitor.store.get("cursor") == "preserved" + assert monitor.store.get("complete:8") == "preserved" + assert monitor.store.get("refresh_branches") == sorted(set(branches)) + assert monitor.store.get("migration") is None + assert not monitor.gh.writes + + +def test_updates_after_refresh_require_maintainer_cutover( + monitor: b.Monitor, refresh_candidate +) -> None: + monitor.args.refresh_branch = ["dev-new"] + refresh_candidate["c" * 40] = refresh_candidate["a" * 40] + refresh_candidate["a" * 40] = _vendor( + "3" * 40, "https://github.com/author/library.git", "temporary" + ) + restarted = b.Monitor(monitor.args, monitor.gh, monitor.store) + restarted.inspect(7) + comment = monitor.gh.comments[0]["body"] + assert "maintainer action" in comment + assert "--canonical-branch dev-new --migrate-canonical-branch-from dev" in comment + assert "Do not promote back" in comment + assert not monitor.gh.reviews + + +def test_waiting_for_prior_promotion_is_maintainer_action_without_review( + monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch +) -> None: + previous = _vendor("a" * 40, "https://github.com/previous/library.git", "prior-update") + reviewed = _vendor("b" * 40, "https://github.com/author/library.git", "temporary") + monkeypatch.setattr(p, "_main_sha", lambda gh: "c" * 40) + monkeypatch.setattr( + p, + "_lock_document_at", + lambda gh, repo, sha: { + "vendors": {"toy": (previous if sha == "c" * 40 else reviewed).to_mapping()} + }, + ) + monitor.inspect(7) + body = monitor.gh.comments[0]["body"] + assert "needs maintainer action" in body and "Waiting for the preceding" in body + assert "operator/library" in body and "dev" in body + assert "After it merges" in body and "No metadata edit" in body + assert "```yaml" not in body and "resolutions:" not in body + assert not monitor.gh.reviews + writes = copy.deepcopy(monitor.gh.writes) + monitor.inspect(7) + assert monitor.gh.writes == writes + + +def test_invalid_pr_lock_has_file_fix_not_metadata_template( + monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch +) -> None: + previous = _vendor("a" * 40, "https://github.com/operator/library.git", "dev") + invalid = previous.to_mapping() | {"commit": "short-sha"} + monkeypatch.setattr(p, "_main_sha", lambda gh: "c" * 40) + monkeypatch.setattr( + p, + "_lock_document_at", + lambda gh, repo, sha: { + "vendors": {"toy": previous.to_mapping() if sha == "c" * 40 else invalid} + }, + ) + monitor.inspect(7) + body = monitor.gh.comments[0]["body"] + assert "vendors.toy" in body and "40-hex SHA" in body + assert p._LOCK in body and "Editing the PR description will not fix" in body + assert "```yaml" not in body + assert monitor.gh.reviews[0]["state"] == "CHANGES_REQUESTED" + + +@pytest.mark.parametrize("history", ["diverged", "unrelated", "merge"]) +def test_source_history_diagnostic_has_canonical_pin_and_regeneration_steps( + source: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, history: str +) -> None: + base = _commit(source, "base\n", "src/code.py") + canonical = _commit(source, "canonical\n", "canonical.py") + _git(source, "checkout", "-q", "-b", "temporary", base) + if history == "unrelated": + _git(source, "checkout", "-q", "--orphan", "unrelated") + reviewed_sha = _commit(source, "change\n", "src/code.py") + if history == "merge": + _git(source, "merge", "--no-ff", "-m", "merge canonical", canonical) + reviewed_sha = _git(source, "rev-parse", "HEAD") + previous = _vendor(canonical, "https://github.com/operator/library.git", "dev") + reviewed = _vendor(reviewed_sha, "https://github.com/author/library.git", "temporary") + original_git = p._git + + def local_git(repo, *args, **kwargs): + if args and args[0] == "fetch": + args = tuple(str(source) if arg == reviewed.url else arg for arg in args) + return original_git(repo, *args, **kwargs) + + monkeypatch.setattr(p, "_git", local_git) + monkeypatch.setattr(p, "_check_source_repo", lambda *args: None) + with pytest.raises(m.Diagnostic) as failure: + m.resolve( + FakeGitHub(), + previous, + reviewed, + {"unpaired_reason": "downstream-only", "upstream_prs": [], "resolutions": {}}, + tmp_path / "cache.git", + ) + issue = failure.value + assert { + "diverged": "not a fast-forward", + "unrelated": "no merge base", + "merge": "linear history", + }[history] in issue.problem + assert ( + canonical in issue.action and "operator/library" in issue.action and "dev" in issue.action + ) + assert "temporary" in issue.action and "scripts/vendor/manage.py" in issue.action + assert "vendored snapshot" in issue.action and "push" in issue.action + assert not issue.example + + +@pytest.mark.parametrize("status", ["closed", "wrong-target"]) +def test_upstream_pr_diagnostic_links_affected_pr_and_fix( + source: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, status: str +) -> None: + base = _commit(source, "base\n", "src/code.py") + head = _commit(source, "changed\n", "src/code.py") + previous = _vendor(base, "https://github.com/operator/library.git", "dev") + reviewed = _vendor(head, "https://github.com/author/library.git", "temporary") + gh = FakeGitHub() + pr = _pr(base, head) + if status == "closed": + pr["state"] = "closed" + else: + pr["base"]["ref"] = "release" + monkeypatch.setattr(gh, "get", lambda endpoint: pr) + monkeypatch.setattr(p, "_check_source_repo", lambda *args: None) + original_git = p._git + + def local_git(repo, *args, **kwargs): + if args and args[0] == "fetch": + args = tuple(str(source) if arg == reviewed.url else arg for arg in args) + return original_git(repo, *args, **kwargs) + + monkeypatch.setattr(p, "_git", local_git) + with pytest.raises(m.Diagnostic) as failure: + m.resolve( + gh, + previous, + reviewed, + {"unpaired_reason": None, "upstream_prs": [42], "resolutions": {}}, + tmp_path / "cache.git", + ) + assert "https://github.com/org/library/pull/42" in failure.value.problem + assert ( + "closed without merging" in failure.value.problem + if status == "closed" + else "release" in failure.value.problem + ) + assert "reopen" in failure.value.action and "replace its URL" in failure.value.action + assert not failure.value.example + + +def test_git_expected_absence_does_not_hide_unexpected_failures(source: Path) -> None: + base = _commit(source, "base\n") + with pytest.raises(RuntimeError, match="git failed"): + p._git(source, "merge-base", base, "f" * 40, allowed_returncodes=(0, 1)) + + +def test_discovery_and_persistent_operator_scope(monitor: b.Monitor) -> None: + assert monitor._discover() == [7] + assert monitor._discover() == [7] + assert not monitor.gh.writes + changed = copy.copy(monitor.args) + changed.canonical_repo = "different/library" + with pytest.raises(ValueError, match="different operator"): + b.Monitor(changed, monitor.gh, monitor.store) + + +@pytest.mark.parametrize("base_branch", ["main", "release/1.3"]) +def test_monitors_discover_only_their_consumer_base(monitor: b.Monitor, base_branch: str) -> None: + monitor.gh.base_branch = base_branch + with contextlib.closing(b.Store(":memory:")) as store: + restarted = b.Monitor(monitor.args, monitor.gh, store) + monitor.gh.pr["base"]["ref"] = "release/1.3" if base_branch == "main" else "main" + assert restarted._discover() == [] + monitor.gh.pr["base"]["ref"] = base_branch + assert restarted._discover() == [7] + assert not monitor.gh.writes + + +def test_workdir_cannot_change_consumer_base(monitor: b.Monitor) -> None: + monitor.gh.base_branch = "release/1.3" + with pytest.raises(ValueError, match="different operator/vendor configuration"): + b.Monitor(monitor.args, monitor.gh, monitor.store) + assert monitor.store.get("settings")["base_branch"] == "main" + + +@pytest.mark.parametrize("base_branch", ["main", "release/1.3"]) +def test_promotion_identity_and_followup_keep_consumer_base( + monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch, base_branch: str +) -> None: + gh = monitor.gh + gh.base_branch = base_branch + gh.pr.update(merged=True, merge_commit_sha="b" * 40) + gh.pr["base"]["ref"] = base_branch + previous = _vendor("1" * 40, "https://github.com/operator/library.git", "dev") + reviewed = _vendor("2" * 40, "https://github.com/author/library.git", "temporary") + original_get = gh.get + + def get(endpoint: str) -> dict: + if endpoint.endswith("/commits/" + "b" * 40): + return {"parents": [{"sha": "c" * 40}]} + if "/git/ref/heads/" in endpoint: + assert endpoint.endswith(f"/git/ref/heads/{base_branch}") + return {"object": {"sha": "d" * 40}} + return original_get(endpoint) + + monkeypatch.setattr(gh, "get", get) + monkeypatch.setattr( + p, "_lock_at", lambda gh, repo, sha: previous if sha == "c" * 40 else reviewed + ) + args = monitor._arguments(7, reviewed) + plan = p._plan_identity(args, gh) + assert plan.base_branch == plan.record["consumer_base"] == base_branch + assert plan.main_sha == "d" * 40 + + def followups(endpoint: str): + query = parse_qs(endpoint.split("?", 1)[1]) + assert query["base"] == [base_branch] + assert query["head"] == [f"operator:{plan.branch}"] + return [] + + monkeypatch.setattr(gh, "api", followups) + assert p._find_followup(gh, plan) is None + assert not gh.writes + + +def test_tracked_pr_retargeted_to_another_base_is_ignored(monitor: b.Monitor) -> None: + assert monitor._discover() == [7] + monitor.gh.pr["base"]["ref"] = "release/1.3" + monitor.inspect(7) + assert monitor.store.get("observation:7") == { + "state": "ignored", + "reason": "PR targets a different consumer base branch", + } + assert not monitor.gh.writes + + +def test_feedback_rechecks_consumer_base_before_writing(monitor: b.Monitor) -> None: + inspected = copy.deepcopy(monitor.gh.pr) + monitor.gh.pr["base"]["ref"] = "release/1.3" + monitor._feedback(inspected, _issue()) + assert monitor.store.get("observation:7")["state"] == "ignored" + assert not monitor.gh.writes + + +def test_cold_discovery_backfills_old_merged_prs_and_honors_cutoff( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, +) -> None: + gh = monitor.gh + gh.pr.update( + state="closed", + merged=True, + merged_at="2020-01-01T00:00:00Z", + updated_at="2020-01-01T00:00:00Z", + ) + api = gh.api + + def filtered(endpoint: str, **kwargs): + if "/pulls?state=open" in endpoint: + return [] + return api(endpoint, **kwargs) + + monkeypatch.setattr(gh, "api", filtered) + assert monitor._discover() == [7] + with contextlib.closing(b.Store(":memory:")) as store: + restarted = b.Monitor(monitor.args, gh, store) + assert restarted._discover() == [7] + restricted = copy.copy(monitor.args) + restricted.since = "2021-01-01T00:00:00Z" + with contextlib.closing(b.Store(":memory:")) as store: + assert b.Monitor(restricted, gh, store)._discover() == [] + assert not gh.writes + + +@pytest.mark.parametrize("changed_lines", [1, 3], ids=["accepted", "below-threshold"]) +def test_monitor_posts_similarity_and_deduplicates_feedback( + source: Path, monitor: b.Monitor, monkeypatch: pytest.MonkeyPatch, changed_lines: int +) -> None: + base = _commit(source, "", "src/code.py") + lines = [f"VALUE_{index} = {index}\n" for index in range(16)] + downstream = _commit(source, "".join(lines), "src/code.py") + _git(source, "checkout", "-q", "-b", "upstream", base) + upstream = _commit( + source, "".join(lines[:-changed_lines]) + "adapted\n" * changed_lines, "src/code.py" + ) + previous = _vendor(base, "https://github.com/operator/library.git", "dev") + reviewed = _vendor(downstream, "https://github.com/author/library.git", "temporary") + monkeypatch.setattr(monitor, "_inputs", lambda pr: (previous, reviewed)) + monitor.gh.pr["body"] = _body(" upstream_prs: [https://github.com/org/library/pull/42]") + monkeypatch.setattr( + m, + "resolve", + lambda *args: m.match( + source, + base, + downstream, + {42: _pr(base, upstream)}, + source=reviewed.source, + include=reviewed.include, + upstream_repo="org/library", + ), + ) + monitor.inspect(7) + body = monitor.gh.comments[0]["body"] + assert "https://github.com/org/library/pull/42" in body + assert downstream in body and "90.00%" in body + if changed_lines == 1: + assert "95.00%" in body and "accepted" in body + assert "not** a code-review approval" in body + assert not monitor.gh.reviews + else: + assert "85.00%" in body and "no PR meets" in body + assert monitor.gh.reviews[0]["state"] == "CHANGES_REQUESTED" + writes = copy.deepcopy(monitor.gh.writes) + with contextlib.closing(b.Store(":memory:")) as store: + restarted = b.Monitor(monitor.args, monitor.gh, store) + monkeypatch.setattr(restarted, "_inputs", lambda pr: (previous, reviewed)) + restarted.inspect(7) + assert monitor.gh.writes == writes + + +def test_feedback_deduplicates_with_empty_sqlite(monitor: b.Monitor) -> None: + monitor._feedback(monitor.gh.pr, _issue("Missing block")) + writes = copy.deepcopy(monitor.gh.writes) + with contextlib.closing(b.Store(":memory:")) as store: + restarted = b.Monitor(monitor.args, monitor.gh, store) + restarted._feedback(monitor.gh.pr, _issue("Missing block")) + assert monitor.gh.writes == writes + restarted._feedback(monitor.gh.pr, None) + assert len(monitor.gh.comments) == len(monitor.gh.reviews) == 1 + assert monitor.gh.reviews[0]["state"] == "DISMISSED" + + +def _vendor(commit: str, url: str, branch: str): + return m.manage._validate_vendor( + "toy", + { + "url": url, + "branch": branch, + "commit": commit, + "source": "src", + "destination": "vendored/toy", + "include": ["**/*.py"], + "digest": "sha256-tree-v1:" + "0" * 64, + }, + ) + + +def test_monitor_revalidates_matching_until_committed_and_publishes_once_after_merge( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, +) -> None: + gh = monitor.gh + previous = _vendor("a" * 40, "https://github.com/operator/library.git", "dev") + reviewed = _vendor("b" * 40, "https://github.com/author/library.git", "temporary") + gh.pr["body"] = _body(" upstream_prs: [https://github.com/org/library/pull/42]") + monkeypatch.setattr(monitor, "_inputs", lambda pr: (previous, reviewed)) + matches, publications = [], [] + + def resolve(*args): + matches.append(True) + return {reviewed.commit: 42}, {reviewed.commit: {"method": "same-edits"}} + + def make_plan(args, github, *, auto_match, resolutions): + assert args.auto_merge and not args.wait + assert args.map_upstream == [] + assert auto_match and resolutions == {} + return p.Promotion( + 7, + "c" * 40, + previous, + reviewed, + "operator/library", + "operator/consumer", + {"attribution": {reviewed.commit: {"method": "same-edits"}}}, + consumer_repo="org/consumer", + ) + + monkeypatch.setattr(p, "_plan_identity", lambda args, gh: argparse.Namespace(main_sha="c" * 40)) + monkeypatch.setattr(p, "_recover_plan", lambda *args: False) + monkeypatch.setattr(p, "_lock_at", lambda *args: reviewed) + monkeypatch.setattr(p, "_check_current", lambda *args: None) + + def publish(args, github, plan): + publications.append(plan) + assert plan.record["attribution"][reviewed.commit]["method"] == "same-edits" + return {"number": 8} + + original = gh.get + monkeypatch.setattr( + gh, + "get", + lambda endpoint: { + "number": 8, + "merged": True, + "merge_commit_sha": "d" * 40, + "html_url": "https://github.com/org/consumer/pull/8", + } + if endpoint.endswith("/pulls/8") + else original(endpoint), + ) + monkeypatch.setattr(m, "resolve", resolve) + monkeypatch.setattr(p, "_make_plan", make_plan) + monkeypatch.setattr(p, "_publish", publish) + monkeypatch.setattr(p, "_verify_remote_commit", lambda *args: "verified") + monitor.inspect(7) + monitor.inspect(7) + assert len(matches) == 2 + assert not publications + gh.pr.update(merged=True, state="closed") + monitor.inspect(7) + monitor.inspect(7) + assert len(publications) == 1 + assert monitor.store.get("complete:7").endswith("/8") + + +def test_monitor_reports_attribution_to_author_and_retries_after_correction( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, +) -> None: + previous = _vendor("a" * 40, "https://github.com/operator/library.git", "dev") + reviewed = _vendor("b" * 40, "https://github.com/author/library.git", "temporary") + monkeypatch.setattr(monitor, "_inputs", lambda pr: (previous, reviewed)) + monitor.gh.pr["body"] = _body(" upstream_prs: [https://github.com/org/library/pull/42]") + + def unresolved(*args): + raise _issue("Author action required: no exact match for " + reviewed.commit) + + monkeypatch.setattr(m, "resolve", unresolved) + monitor.inspect(7) + assert "@author" in monitor.gh.comments[0]["body"] + assert reviewed.commit in monitor.gh.comments[0]["body"] + assert monitor.gh.reviews[0]["state"] == "CHANGES_REQUESTED" + monkeypatch.setattr(m, "resolve", lambda *args: ({reviewed.commit: 42}, {})) + monitor.inspect(7) + assert monitor.gh.reviews[0]["state"] == "DISMISSED" + + +@pytest.mark.parametrize("no_promotion", [False, True]) +@pytest.mark.parametrize("comment_deleted", [False, True]) +def test_forbidden_dismissal_informs_author_without_approving( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, + no_promotion: bool, + comment_deleted: bool, +) -> None: + monitor._feedback(monitor.gh.pr, _issue("Missing block")) + if comment_deleted: + monitor.gh.comments.clear() + original = monitor.gh.api + + def forbidden(endpoint, **kwargs): + if endpoint.endswith("/dismissals"): + raise RuntimeError("HTTP 403") + return original(endpoint, **kwargs) + + monkeypatch.setattr(monitor.gh, "api", forbidden) + monitor._feedback(monitor.gh.pr, None, no_promotion=no_promotion) + assert "lacks dismissal permission" in monitor.gh.comments[0]["body"] + assert monitor.gh.reviews[0]["state"] == "CHANGES_REQUESTED" + writes = len(monitor.gh.writes) + monitor._feedback(monitor.gh.pr, None, no_promotion=no_promotion) + assert len(monitor.gh.writes) == writes + monkeypatch.setattr(monitor.gh, "api", original) + monitor._feedback(monitor.gh.pr, None, no_promotion=no_promotion) + assert "lacks dismissal permission" not in monitor.gh.comments[0]["body"] + assert monitor.gh.reviews[0]["state"] == "DISMISSED" + + +def test_old_promotion_messages_remain_readable() -> None: + record = { + "version": 1, + "vendor": "flashinfer-prims-ts", + "trtllm_pr": "https://github.com/NVIDIA/TensorRT-LLM/pull/17", + "trtllm_merge": "a" * 40, + } + message = ( + "----- BEGIN PRIMTS PROMOTION V1 -----\n" + + json.dumps(record) + + "\n----- END PRIMTS PROMOTION V1 -----" + ) + normalized = p._record_from_message(message) + assert normalized["consumer_pr"].endswith("/17") + assert normalized["upstream_repository"] == "flashinfer-ai/flashinfer" + + +def test_workdir_locks_and_refuses_unrelated_data(tmp_path: Path) -> None: + workdir = tmp_path / "state" + b._initialize_workdir(workdir) + with b._locked(workdir): + assert b._running(workdir) + with pytest.raises(ValueError, match="already owns"): + with b._locked(workdir): + pass + assert not b._running(workdir) + unrelated = tmp_path / "unrelated" + unrelated.mkdir() + (unrelated / "data").write_text("preserve") + with pytest.raises(ValueError, match="empty workdir"): + b._initialize_workdir(unrelated) + assert (unrelated / "data").read_text() == "preserve" + + +def test_workdir_refuses_checkout(source: Path) -> None: + with pytest.raises(ValueError, match="outside a Git worktree"): + b._initialize_workdir(source / "bot-state") + + +def test_workdir_recovers_without_sqlite_but_preserves_artifacts(tmp_path: Path) -> None: + workdir = tmp_path / "state" + b._initialize_workdir(workdir) + (workdir / "bot.log").write_text("previous log\n") + (workdir / "bot.lock").touch() + (workdir / "sources.git").mkdir() + (workdir / ("promotion-7-" + "a" * 12)).mkdir() + b._initialize_workdir(workdir) + assert (workdir / "bot.log").read_text() == "previous log\n" + with b._locked(workdir): + with pytest.raises(ValueError, match="already owns"): + with b._locked(workdir): + pass + with contextlib.closing(b.Store(workdir / b._STATE_FILE)) as store: + assert store.get("tracked") is None + + +@pytest.mark.parametrize("name", ["state.sqlite", "bot.log", "bot.lock", "sources.git"]) +def test_workdir_recovery_rejects_symlinks(tmp_path: Path, name: str) -> None: + workdir = tmp_path / "state" + workdir.mkdir() + target = tmp_path / "private-data" + target.write_text("preserve") + (workdir / name).symlink_to(target) + with pytest.raises(ValueError, match="unexpected entry"): + b._initialize_workdir(workdir) + assert target.read_text() == "preserve" + + +def test_daemon_launch_status_stop_without_systemd(tmp_path: Path) -> None: + binary = tmp_path / "bin" + binary.mkdir() + gh = binary / "gh" + gh.write_text( + "#!/usr/bin/env python3\nimport json, sys\n" + "print(json.dumps({'login': 'operator'} if sys.argv[-1] == 'user' else []))\n" + ) + gh.chmod(0o700) + workdir = tmp_path / "state" + env = { + **os.environ, + "PATH": f"{binary}:{os.environ['PATH']}", + "GH_CONFIG_DIR": str(tmp_path / "private-auth"), + } + command = [sys.executable, str(_ROOT / "scripts/vendor/bot.py")] + arguments = [ + "run", + "--workdir", + str(workdir), + "--vendor", + "toy", + "--upstream-repo", + "org/library", + "--canonical-repo", + "operator/library", + "--canonical-branch", + "dev", + "--fork", + "operator/consumer", + "--daemon", + ] + try: + result = subprocess.run( + command + arguments, env=env, capture_output=True, text=True, timeout=15 + ) + assert result.returncode == 0, result.stderr + assert "Started daemon PID" in result.stdout + status = subprocess.run( + command + ["status", "--workdir", str(workdir)], + env=env, + capture_output=True, + text=True, + check=True, + ) + state = json.loads(status.stdout) + assert state["running"] + assert state["pid"] is not None + assert state["publish"] is False + assert state["vendor"] == "toy" + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + state = b._status(workdir) + if state.get("next_poll_at"): + break + time.sleep(0.05) + assert state["last_poll_summary"]["outcome"] == "ok" + assert state["last_successful_poll"] == state["last_completed_poll"] + assert state["operation"]["phase"] == "waiting" + assert state["next_poll_at"] + assert "Poll finished" in (workdir / "bot.log").read_text() + assert "private-auth" not in (workdir / "bot.log").read_text() + finally: + subprocess.run( + command + ["stop", "--workdir", str(workdir)], env=env, capture_output=True, timeout=5 + ) + deadline = time.monotonic() + 10 + while b._running(workdir) and time.monotonic() < deadline: + time.sleep(0.1) + assert not b._running(workdir) + assert b._status(workdir)["pid"] is None + assert b._status(workdir)["next_poll_at"] is None + + +def test_log_level_defaults_to_info_and_accepts_lowercase(tmp_path: Path) -> None: + command = [ + "run", + "--workdir", + str(tmp_path), + "--vendor", + "toy", + "--upstream-repo", + "org/library", + "--canonical-repo", + "operator/library", + "--canonical-branch", + "dev", + "--fork", + "operator/consumer", + ] + assert b._parser().parse_args(command).log_level == "INFO" + assert b._parser().parse_args(command + ["--log-level", "debug"]).log_level == "DEBUG" + + +@pytest.mark.parametrize("publish", [False, True]) +def test_metadata_transitions_are_logged_once_and_clear_problem( + monitor: b.Monitor, + caplog: pytest.LogCaptureFixture, + publish: bool, +) -> None: + caplog.set_level(logging.INFO, logger="vendor-bot") + monitor.args.publish = publish + monitor._feedback(monitor.gh.pr, _issue()) + monitor._feedback(monitor.gh.pr, _issue()) + assert caplog.text.count("PR #7: blocked: Missing marked block") == 1 + assert monitor.store.get("problem:7") == "Missing marked block" + monitor._feedback(monitor.gh.pr, None) + monitor._feedback(monitor.gh.pr, None) + assert caplog.text.count("PR #7: metadata_valid") == 1 + assert monitor.store.get("problem:7") is None + if publish: + assert "posted metadata feedback comment" in caplog.text + assert "requested metadata changes" in caplog.text + assert "dismissed resolved metadata review" in caplog.text + else: + assert not monitor.gh.writes + + +def test_idle_poll_has_summary_and_success_timestamp( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + caplog.set_level(logging.INFO, logger="vendor-bot") + monkeypatch.setattr(monitor, "_discover", lambda: []) + monitor.poll() + health = monitor.store.get("health") + assert health["last_successful_poll"] == health["last_completed_poll"] + assert health["last_completed_poll"] == monitor.store.get("last_poll") + assert health["last_poll_summary"]["outcome"] == "ok" + assert health["last_poll_summary"]["processed"] == 0 + assert health["last_poll_summary"]["errors"] == 0 + assert "Poll started" in caplog.text and "Poll finished" in caplog.text + + +def test_pr_failure_is_not_a_successful_poll_and_recovers( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(monitor, "_discover", lambda: [7]) + + def fail(number: int) -> None: + raise RuntimeError("HTTP 429: rate limit") + + monkeypatch.setattr(monitor, "inspect", fail) + monitor.poll() + failed = monitor.store.get("health") + assert failed["last_completed_poll"] + assert failed["last_successful_poll"] is None + assert failed["last_poll_summary"]["outcome"] == "needs_attention" + assert failed["last_poll_summary"]["errors"] == 1 + assert failed["last_poll_summary"]["processed"] == 1 + assert failed["last_error"]["pr"] == 7 + assert "HTTP 429" in failed["last_error"]["message"] + monkeypatch.setattr( + monitor, "inspect", lambda number: monitor._transition(number, "metadata_valid") + ) + monitor.poll() + recovered = monitor.store.get("health") + assert recovered["last_successful_poll"] == recovered["last_completed_poll"] + assert recovered["last_poll_summary"]["outcome"] == "ok" + assert recovered["last_error"] == failed["last_error"] # Historical, not an active error. + assert monitor.store.get("problem:7") is None + + +def test_author_blocked_poll_is_not_reported_healthy( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, +) -> None: + previous = _vendor("a" * 40, "https://github.com/operator/library.git", "dev") + reviewed = _vendor("b" * 40, "https://github.com/author/library.git", "temporary") + monkeypatch.setattr(monitor, "_inputs", lambda pr: (previous, reviewed)) + monitor.poll() + health = monitor.store.get("health") + assert health["last_poll_summary"]["outcome"] == "needs_attention" + assert health["counts"]["blocked"] == 1 + assert health["counts"]["errors"] == 0 + assert health["counts"]["relevant"] == 1 + assert health["last_successful_poll"] is None + + +@pytest.mark.parametrize("error_type", [RuntimeError, KeyError]) +def test_discovery_failure_never_advances_completion( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, + error_type: type[Exception], +) -> None: + monitor.progress.update(last_completed_poll="old", last_successful_poll="old") + + def fail() -> list[int]: + raise error_type("discovery failed") + + monkeypatch.setattr(monitor, "_discover", fail) + with pytest.raises(error_type): + monitor.poll() + health = monitor.store.get("health") + assert health["last_completed_poll"] == health["last_successful_poll"] == "old" + assert health["last_poll_summary"]["outcome"] == "failed" + + +def test_stopping_discovery_does_not_advance_cursor_or_success( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, +) -> None: + original = monitor.gh.api + + def stop(endpoint: str, **kwargs): + if "/pulls?" in endpoint: + b._STOP.set() + return original(endpoint, **kwargs) + + monkeypatch.setattr(monitor.gh, "api", stop) + monitor.poll() + assert monitor.store.get("cursor") is None + assert monitor.store.get("bootstrapped") is None + health = monitor.store.get("health") + assert health["last_completed_poll"] is None + assert health["last_successful_poll"] is None + assert health["last_poll_summary"]["outcome"] == "interrupted" + + +def test_heartbeat_during_blocked_discovery_does_not_fake_progress( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + caplog.set_level(logging.INFO, logger="vendor-bot") + monkeypatch.setattr(b, "_HEARTBEAT_SECONDS", 0.01) + reported = threading.Event() + original = monitor.progress.heartbeat + + def heartbeat() -> None: + original() + reported.set() + + def wait_for_heartbeat(endpoint: str, **kwargs): + before = monitor.store.get("health") + assert reported.wait(2), "No heartbeat during blocked API call" + assert monitor.store.get("health") == before + return [] + + monkeypatch.setattr(monitor.progress, "heartbeat", heartbeat) + monkeypatch.setattr(monitor.gh, "api", wait_for_heartbeat) + with monitor.progress.heartbeats(): + monitor.poll() + assert "Heartbeat: phase=discover_open" in caplog.text + assert "operation completion not yet confirmed" in caplog.text + assert "Discovery completed" in caplog.text + assert not any(t.name == "vendor-bot-heartbeat" for t in threading.enumerate()) + + +def test_progress_checkpoint_and_restart_keep_history(monitor: b.Monitor) -> None: + progress = monitor.progress + progress.operation("discover_history") + progress.last_checkpoint = float("-inf") + progress.count("scanned", 123) + assert monitor.store.get("health")["counts"]["scanned"] == 123 + progress.error("previous error", 7) + progress.update(last_successful_poll="old", next_poll_at="stale") + restarted = b.Progress(monitor.store) + assert restarted.state["last_successful_poll"] == "old" + assert restarted.state["last_error"]["pr"] == 7 + assert restarted.state["next_poll_at"] is None + assert restarted.state["operation"] is None + + +def test_completed_promotion_clears_stale_problem_without_remote_writes(monitor: b.Monitor) -> None: + monitor.store.put("problem:7", "interrupted after completion checkpoint") + monitor.store.put("complete:7", "https://github.com/org/consumer/pull/8") + monitor.inspect(7) + assert monitor.store.get("problem:7") is None + assert monitor.store.get("observation:7")["state"] == "completed" + assert not monitor.gh.writes + + +@pytest.mark.parametrize("status", ["added", "removed", "modified"]) +def test_lock_creation_removal_is_not_a_promotion_but_other_404s_fail( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, + status: str, +) -> None: + original = monitor.gh.api + + def files(endpoint: str, **kwargs): + if "/files?" in endpoint: + return [{"filename": p._LOCK, "status": status}] + return original(endpoint, **kwargs) + + def missing(*args): + raise RuntimeError("gh failed: gh: Not Found (HTTP 404)") + + monkeypatch.setattr(monitor.gh, "api", files) + monkeypatch.setattr(p, "_main_sha", lambda gh: "a" * 40) + monkeypatch.setattr(p, "_lock_document_at", missing) + if status == "modified": + with pytest.raises(RuntimeError, match="HTTP 404"): + monitor._inputs(monitor.gh.pr) + else: + monitor.inspect(7) + assert monitor.store.get("observation:7")["state"] == "ignored" + assert monitor.store.get("problem:7") is None + assert not monitor.gh.writes + + +def test_status_is_read_only_and_hides_stale_process_data(tmp_path: Path) -> None: + workdir = tmp_path / "missing" + assert b._status(workdir)["running"] is False + assert not workdir.exists() + workdir.mkdir() + with contextlib.closing(b.Store(workdir / b._STATE_FILE)) as store: + store.put("process", {"pid": 123, "publish": True}) + store.put("settings", {"vendor": "toy", "actor": "operator"}) + progress = b.Progress(store) + progress.operation("publishing_promotion", 7) + progress.update(next_poll_at="future", last_completed_poll="previous") + progress.update(running=False, unexpected="do not expose unrelated cached fields") + before = (workdir / b._STATE_FILE).read_bytes() + with b._locked(workdir): + state = b._status(workdir) + assert state["running"] and state["pid"] == 123 + assert state["publish"] and state["vendor"] == "toy" + assert state["operation"]["pr"] == 7 + assert "unexpected" not in state + (workdir / "stop").touch() + assert b._status(workdir)["stop_requested"] + state = b._status(workdir) + assert state["running"] is False and state["pid"] is None + assert state["operation"] is None and state["next_poll_at"] is None + assert state["last_completed_poll"] == "previous" + assert (workdir / b._STATE_FILE).read_bytes() == before + + +def test_status_exposes_consumer_base_and_refresh_policy(monitor: b.Monitor) -> None: + monitor.args.refresh_branch = ["dev-new"] + b.Monitor(monitor.args, monitor.gh, monitor.store) + state = b._status(monitor.args.workdir) + assert state["base_branch"] == "main" + assert state["canonical_branch"] == "dev" + assert state["refresh_branches"] == ["dev-new"] + assert not state["running"] + + +@pytest.mark.parametrize("policy", [{}, [123]]) +def test_status_rejects_malformed_refresh_policy(monitor: b.Monitor, policy: object) -> None: + monitor.store.put("refresh_branches", policy) + state = b._status(monitor.args.workdir) + assert not state["state_available"] + assert "Malformed refresh policy" in state["state_error"] + + +def test_refresh_branch_cli_is_repeatable_and_rejects_invalid_refs(tmp_path: Path) -> None: + workdir = tmp_path / "unused-bot-state" + argv = [ + "run", + "--workdir", + str(workdir), + "--vendor", + "toy", + "--upstream-repo", + "org/library", + "--canonical-repo", + "operator/library", + "--canonical-branch", + "dev", + "--fork", + "operator/consumer", + "--refresh-branch", + "dev-new", + "--refresh-branch", + "../invalid", + "--once", + ] + parsed = b._parser().parse_args(argv) + assert parsed.refresh_branch == ["dev-new", "../invalid"] + assert not parsed.publish and not parsed.daemon + assert b.main(argv) == 1 + assert not workdir.exists() + + +@pytest.mark.parametrize("malformed", [False, True]) +def test_status_reports_unreadable_cache_without_losing_liveness( + tmp_path: Path, malformed: bool +) -> None: + database = tmp_path / b._STATE_FILE + if malformed: + with contextlib.closing(b.Store(database)) as store: + store.put("health", []) + else: + database.write_text("not SQLite") + before = database.read_bytes() + with b._locked(tmp_path): + result = b._status(tmp_path) + assert result["running"] and not result["state_available"] + assert result["state_error"] + assert database.read_bytes() == before + + +def test_log_and_status_errors_redact_secrets_and_yaml_excerpts( + monitor: b.Monitor, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + caplog.set_level(logging.INFO, logger="vendor-bot") + monkeypatch.setenv("GH_TOKEN", "private-environment-token") + detail = ( + "fetch failed https://user:password@example.invalid/repo " + "private-environment-token ghp_123456789 github_pat_abcdef\n" + "PR description text must not be logged" + ) + monitor.progress.error(detail, 7) + monitor._transition(7, "blocked", detail) + diagnostic = json.dumps(monitor.store.get("health")) + caplog.text + for secret in ( + "user:password", + "private-environment-token", + "ghp_123456789", + "github_pat_abcdef", + "PR description text", + ): + assert secret not in diagnostic + assert "fetch failed" in diagnostic + assert b._summary("Authorization: Bearer secret") == "Authorization: [REDACTED]" + + +def test_serve_reports_backoff_and_clears_process_on_exit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + caplog.set_level(logging.INFO, logger="vendor-bot") + b._STOP.clear() + args = argparse.Namespace( + workdir=tmp_path, + publish=False, + consumer_repo="org/consumer", + vendor="toy", + upstream_repo="org/library", + base_branch="main", + upstream_branch="main", + interval=30, + once=False, + ) + + class BrokenMonitor: + actor = "operator" + + def __init__(self, args, gh, store, progress): + self.progress = progress + + def poll(self): + self.progress.error("HTTP 503") + raise RuntimeError("HTTP 503") + + def finish_wait(seconds: float) -> bool: + state = b._status(tmp_path) + assert state["running"] + assert state["operation"]["phase"] == "waiting" + assert state["next_poll_at"] + assert state["last_error"]["message"] == "HTTP 503" + b._STOP.set() + return True + + monkeypatch.setattr(b, "Monitor", BrokenMonitor) + monkeypatch.setattr(b._STOP, "wait", finish_wait) + try: + b._serve(args) + finally: + b._STOP.clear() + assert "in 60s; consecutive_poll_failures=1" in caplog.text + assert b._status(tmp_path)["pid"] is None + assert b._status(tmp_path)["next_poll_at"] is None + + +def test_startup_error_is_visible_after_process_exits( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + args = argparse.Namespace( + workdir=tmp_path, + publish=False, + consumer_repo="org/consumer", + vendor="toy", + upstream_repo="org/library", + base_branch="main", + upstream_branch="main", + interval=30, + once=True, + ) + + def unauthorized(self, endpoint: str) -> dict: + raise RuntimeError("HTTP 401: authentication required") + + monkeypatch.setattr(p.GitHub, "get", unauthorized) + with pytest.raises(RuntimeError, match="HTTP 401"): + b._serve(args) + state = b._status(tmp_path) + assert not state["running"] and state["pid"] is None + assert state["last_error"]["message"] == "HTTP 401: authentication required" + assert state["last_completed_poll"] is None + assert state["next_poll_at"] is None + assert not any(t.name == "vendor-bot-heartbeat" for t in threading.enumerate()) diff --git a/tests/unittest/others/test_vendor_sources.py b/tests/unittest/others/test_vendor_sources.py index 3872728cd895..87346a9bb07e 100644 --- a/tests/unittest/others/test_vendor_sources.py +++ b/tests/unittest/others/test_vendor_sources.py @@ -34,7 +34,7 @@ _REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent -_VENDOR_SOURCES = _REPO_ROOT / "scripts" / "vendor_sources.py" +_VENDOR_SOURCES = _REPO_ROOT / "scripts" / "vendor" / "manage.py" _VENDOR_NAME = "example" _SOURCE = "python/example" _DESTINATION = "src/example"