diff --git a/.github/labeler.yml b/.github/labeler.yml index a4745cc6a..9a0d02fa9 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -16,10 +16,401 @@ # under the License. # # GENERATED by tools/dev/generate-labeler-config.py from the -# `**Capability:**` line of every tools//README.md. Do not edit by -# hand; change the README and let the prek hook regenerate this file. +# `**Capability:**` line of every tools//README.md and the `family:` / +# `capability:` frontmatter of every skill. Do not edit by hand; change the +# README or the skill and let the prek hook regenerate this file. --- -changed-files-labels-limit: 8 +changed-files-labels-limit: 20 + +capability:authoring: + - changed-files: + - any-glob-to-any-file: + - 'plugins/magpie-security/skills/model-prepare/**' + - 'plugins/magpie-security/skills/model-update/**' + - 'plugins/magpie-utilities/skills/optimize-skill/**' + - 'plugins/magpie-utilities/skills/write-skill/**' + - 'tools/skill-evals/evals/optimize-skill/**' + - 'tools/skill-evals/evals/security-model-prepare/**' + - 'tools/skill-evals/evals/security-model-update/**' + - 'tools/skill-evals/evals/write-skill/**' + +capability:fix: + - changed-files: + - any-glob-to-any-file: + - 'plugins/magpie-issue/skills/fix-workflow/**' + - 'plugins/magpie-repo-health/skills/audit-finding-fix/**' + - 'plugins/magpie-security/skills/issue-fix/**' + - 'tools/skill-evals/evals/audit-finding-fix/**' + - 'tools/skill-evals/evals/issue-fix-workflow/**' + - 'tools/skill-evals/evals/security-issue-fix/**' + +capability:intake: + - changed-files: + - any-glob-to-any-file: + - 'plugins/magpie-contributor-growth/skills/identity-map/**' + - 'plugins/magpie-security/skills/issue-import-from-md/**' + - 'plugins/magpie-security/skills/issue-import-from-pr/**' + - 'plugins/magpie-security/skills/issue-import-from-scan/**' + - 'plugins/magpie-security/skills/issue-import-via-forwarder/**' + - 'plugins/magpie-security/skills/issue-import/**' + - 'plugins/magpie-security/skills/issue-sync/**' + - 'plugins/magpie-setup/skills/shared-config-sync/**' + - 'tools/skill-evals/evals/contributor-identity-map/**' + - 'tools/skill-evals/evals/security-issue-import-from-md/**' + - 'tools/skill-evals/evals/security-issue-import-from-pr/**' + - 'tools/skill-evals/evals/security-issue-import-from-scan/**' + - 'tools/skill-evals/evals/security-issue-import-via-forwarder/**' + - 'tools/skill-evals/evals/security-issue-import/**' + - 'tools/skill-evals/evals/security-issue-sync/**' + - 'tools/skill-evals/evals/setup-shared-config-sync/**' + +capability:platform: + - changed-files: + - any-glob-to-any-file: + - 'plugins/magpie-setup/skills/isolated-setup-doctor/**' + - 'plugins/magpie-setup/skills/isolated-setup-install/**' + - 'plugins/magpie-setup/skills/isolated-setup-update/**' + - 'plugins/magpie-setup/skills/isolated-setup-verify/**' + - 'plugins/magpie-setup/skills/override-upstream/**' + - 'plugins/magpie-setup/skills/privacy-llm/**' + - 'plugins/magpie-setup/skills/setup/**' + - 'plugins/magpie-setup/skills/shared-config-sync/**' + - 'plugins/magpie-setup/skills/status/**' + - 'plugins/magpie-setup/skills/upstream-fix/**' + - 'plugins/magpie-utilities/skills/report-framework-issue/**' + - 'tools/skill-evals/evals/report-framework-issue/**' + - 'tools/skill-evals/evals/setup-isolated-setup-doctor/**' + - 'tools/skill-evals/evals/setup-isolated-setup-install/**' + - 'tools/skill-evals/evals/setup-isolated-setup-update/**' + - 'tools/skill-evals/evals/setup-isolated-setup-verify/**' + - 'tools/skill-evals/evals/setup-override-upstream/**' + - 'tools/skill-evals/evals/setup-privacy-llm/**' + - 'tools/skill-evals/evals/setup-shared-config-sync/**' + - 'tools/skill-evals/evals/setup-status/**' + - 'tools/skill-evals/evals/setup-upstream-fix/**' + - 'tools/skill-evals/evals/setup/**' + +capability:reassess: + - changed-files: + - any-glob-to-any-file: + - 'plugins/magpie-issue/skills/reassess/**' + - 'plugins/magpie-issue/skills/reproducer/**' + - 'plugins/magpie-security/skills/model-update/**' + - 'plugins/magpie-setup/skills/isolated-setup-doctor/**' + - 'tools/skill-evals/evals/issue-reassess/**' + - 'tools/skill-evals/evals/issue-reproducer/**' + - 'tools/skill-evals/evals/security-model-update/**' + - 'tools/skill-evals/evals/setup-isolated-setup-doctor/**' + +capability:reconciliation: + - changed-files: + - any-glob-to-any-file: + - 'plugins/magpie-utilities/skills/skill-reconciler/**' + - 'tools/skill-evals/evals/skill-reconciler/**' + +capability:resolve: + - changed-files: + - any-glob-to-any-file: + - 'plugins/magpie-contributor-growth/skills/committer-onboarding/**' + - 'plugins/magpie-issue/skills/deduplicate/**' + - 'plugins/magpie-release-management/skills/announce-draft/**' + - 'plugins/magpie-release-management/skills/archive-sweep/**' + - 'plugins/magpie-release-management/skills/keys-sync/**' + - 'plugins/magpie-release-management/skills/prepare/**' + - 'plugins/magpie-release-management/skills/promote/**' + - 'plugins/magpie-release-management/skills/rc-cut/**' + - 'plugins/magpie-release-management/skills/vote-draft/**' + - 'plugins/magpie-release-management/skills/vote-tally/**' + - 'plugins/magpie-security/skills/cve-allocate/**' + - 'plugins/magpie-security/skills/issue-deduplicate/**' + - 'plugins/magpie-security/skills/issue-fix/**' + - 'plugins/magpie-security/skills/issue-invalidate/**' + - 'tools/skill-evals/evals/committer-onboarding/**' + - 'tools/skill-evals/evals/issue-deduplicate/**' + - 'tools/skill-evals/evals/release-announce-draft/**' + - 'tools/skill-evals/evals/release-archive-sweep/**' + - 'tools/skill-evals/evals/release-keys-sync/**' + - 'tools/skill-evals/evals/release-prepare/**' + - 'tools/skill-evals/evals/release-promote/**' + - 'tools/skill-evals/evals/release-rc-cut/**' + - 'tools/skill-evals/evals/release-vote-draft/**' + - 'tools/skill-evals/evals/release-vote-tally/**' + - 'tools/skill-evals/evals/security-cve-allocate/**' + - 'tools/skill-evals/evals/security-issue-deduplicate/**' + - 'tools/skill-evals/evals/security-issue-fix/**' + - 'tools/skill-evals/evals/security-issue-invalidate/**' + +capability:review: + - changed-files: + - any-glob-to-any-file: + - 'plugins/magpie-contributor-growth/skills/onboarding-concierge/**' + - 'plugins/magpie-mentoring/skills/good-first-issue-author/**' + - 'plugins/magpie-mentoring/skills/good-first-issue-sweep/**' + - 'plugins/magpie-mentoring/skills/newcomer-issue-explainer/**' + - 'plugins/magpie-mentoring/skills/welcome/**' + - 'plugins/magpie-pairing/skills/multi-agent-review/**' + - 'plugins/magpie-pairing/skills/self-review/**' + - 'plugins/magpie-pr-management/skills/code-review/**' + - 'plugins/magpie-pr-management/skills/mentor/**' + - 'plugins/magpie-pr-management/skills/pre-first-pr-check/**' + - 'plugins/magpie-pr-management/skills/quick-merge/**' + - 'plugins/magpie-security/skills/model-verify/**' + - 'tools/skill-evals/evals/good-first-issue-author/**' + - 'tools/skill-evals/evals/good-first-issue-sweep/**' + - 'tools/skill-evals/evals/mentoring-welcome/**' + - 'tools/skill-evals/evals/newcomer-issue-explainer/**' + - 'tools/skill-evals/evals/onboarding-concierge/**' + - 'tools/skill-evals/evals/pairing-multi-agent-review/**' + - 'tools/skill-evals/evals/pairing-self-review/**' + - 'tools/skill-evals/evals/pr-management-code-review/**' + - 'tools/skill-evals/evals/pr-management-mentor/**' + - 'tools/skill-evals/evals/pr-management-quick-merge/**' + - 'tools/skill-evals/evals/pre-first-pr-check/**' + - 'tools/skill-evals/evals/security-model-verify/**' + +capability:stats: + - changed-files: + - any-glob-to-any-file: + - 'plugins/magpie-contributor-growth/skills/activity-sweep/**' + - 'plugins/magpie-contributor-growth/skills/calibrate/**' + - 'plugins/magpie-contributor-growth/skills/candidate-screen/**' + - 'plugins/magpie-contributor-growth/skills/contributor-to-committer/**' + - 'plugins/magpie-contributor-growth/skills/nomination/**' + - 'plugins/magpie-contributor-growth/skills/sentiment/**' + - 'plugins/magpie-issue/skills/backlog-stats/**' + - 'plugins/magpie-issue/skills/reassess-stats/**' + - 'plugins/magpie-pr-management/skills/stats/**' + - 'plugins/magpie-release-management/skills/audit-report/**' + - 'plugins/magpie-security/skills/tracker-stats-dashboard/**' + - 'plugins/magpie-setup/skills/status/**' + - 'plugins/magpie-utilities/skills/list-skills/**' + - 'tools/skill-evals/evals/contributor-activity-sweep/**' + - 'tools/skill-evals/evals/contributor-calibrate/**' + - 'tools/skill-evals/evals/contributor-candidate-screen/**' + - 'tools/skill-evals/evals/contributor-nomination/**' + - 'tools/skill-evals/evals/contributor-sentiment/**' + - 'tools/skill-evals/evals/contributor-to-committer/**' + - 'tools/skill-evals/evals/issue-backlog-stats/**' + - 'tools/skill-evals/evals/issue-reassess-stats/**' + - 'tools/skill-evals/evals/list-skills/**' + - 'tools/skill-evals/evals/pr-management-stats/**' + - 'tools/skill-evals/evals/release-audit-report/**' + - 'tools/skill-evals/evals/security-tracker-stats-dashboard/**' + - 'tools/skill-evals/evals/setup-status/**' + +capability:triage: + - changed-files: + - any-glob-to-any-file: + - 'plugins/magpie-contributor-growth/skills/committer-onboarding/**' + - 'plugins/magpie-issue/skills/stale-sweep/**' + - 'plugins/magpie-issue/skills/triage/**' + - 'plugins/magpie-mentoring/skills/good-first-issue-sweep/**' + - 'plugins/magpie-pr-management/skills/pr-stale-sweep/**' + - 'plugins/magpie-pr-management/skills/pr-triage/**' + - 'plugins/magpie-pr-management/skills/quick-merge/**' + - 'plugins/magpie-pr-management/skills/reviewer-routing/**' + - 'plugins/magpie-release-management/skills/archive-sweep/**' + - 'plugins/magpie-release-management/skills/verify-rc/**' + - 'plugins/magpie-release-management/skills/vote-tally/**' + - 'plugins/magpie-repo-health/skills/ci-runner-audit/**' + - 'plugins/magpie-repo-health/skills/dependency-audit/**' + - 'plugins/magpie-repo-health/skills/dependency-license-audit/**' + - 'plugins/magpie-repo-health/skills/flaky-test-triage/**' + - 'plugins/magpie-repo-health/skills/license-compliance-audit/**' + - 'plugins/magpie-repo-health/skills/workflow-security-audit/**' + - 'plugins/magpie-security/skills/issue-triage/**' + - 'tools/skill-evals/evals/ci-runner-audit/**' + - 'tools/skill-evals/evals/committer-onboarding/**' + - 'tools/skill-evals/evals/dependency-audit/**' + - 'tools/skill-evals/evals/dependency-license-audit/**' + - 'tools/skill-evals/evals/flaky-test-triage/**' + - 'tools/skill-evals/evals/good-first-issue-sweep/**' + - 'tools/skill-evals/evals/issue-stale-sweep/**' + - 'tools/skill-evals/evals/issue-triage/**' + - 'tools/skill-evals/evals/license-compliance-audit/**' + - 'tools/skill-evals/evals/pr-management-quick-merge/**' + - 'tools/skill-evals/evals/pr-management-triage/**' + - 'tools/skill-evals/evals/pr-stale-sweep/**' + - 'tools/skill-evals/evals/release-archive-sweep/**' + - 'tools/skill-evals/evals/release-verify-rc/**' + - 'tools/skill-evals/evals/release-vote-tally/**' + - 'tools/skill-evals/evals/reviewer-routing/**' + - 'tools/skill-evals/evals/security-issue-triage/**' + - 'tools/skill-evals/evals/workflow-security-audit/**' + +family:ci: + - changed-files: + - any-glob-to-any-file: + - '.gitattributes' + - '.github/**' + - '.gitignore' + - '.lychee.toml' + - '.pre-commit-config.yaml' + - '.rat-excludes' + - '.typos.toml' + - 'pyproject.toml' + - 'tools/dev/**' + - 'uv.lock' + +family:contributor-growth: + - changed-files: + - any-glob-to-any-file: + - 'docs/contributor-growth/**' + - 'plugins/magpie-contributor-growth/**' + - 'tools/skill-evals/evals/committer-onboarding/**' + - 'tools/skill-evals/evals/contributor-activity-sweep/**' + - 'tools/skill-evals/evals/contributor-calibrate/**' + - 'tools/skill-evals/evals/contributor-candidate-screen/**' + - 'tools/skill-evals/evals/contributor-identity-map/**' + - 'tools/skill-evals/evals/contributor-nomination/**' + - 'tools/skill-evals/evals/contributor-sentiment/**' + - 'tools/skill-evals/evals/contributor-to-committer/**' + - 'tools/skill-evals/evals/onboarding-concierge/**' + +family:docs: + - changed-files: + - any-glob-to-any-file: + - '**/README.md' + - '*.md' + - 'MISSION.md' + - 'docs/**' + +family:issue: + - changed-files: + - any-glob-to-any-file: + - 'plugins/magpie-issue/**' + - 'tools/skill-evals/evals/issue-backlog-stats/**' + - 'tools/skill-evals/evals/issue-deduplicate/**' + - 'tools/skill-evals/evals/issue-fix-workflow/**' + - 'tools/skill-evals/evals/issue-reassess-stats/**' + - 'tools/skill-evals/evals/issue-reassess/**' + - 'tools/skill-evals/evals/issue-reproducer/**' + - 'tools/skill-evals/evals/issue-stale-sweep/**' + - 'tools/skill-evals/evals/issue-triage/**' + +family:mentoring: + - changed-files: + - any-glob-to-any-file: + - 'docs/mentoring/**' + - 'plugins/magpie-mentoring/**' + - 'tools/skill-evals/evals/good-first-issue-author/**' + - 'tools/skill-evals/evals/good-first-issue-sweep/**' + - 'tools/skill-evals/evals/mentoring-welcome/**' + - 'tools/skill-evals/evals/newcomer-issue-explainer/**' + +family:pairing: + - changed-files: + - any-glob-to-any-file: + - 'docs/pairing/**' + - 'plugins/magpie-pairing/**' + - 'tools/skill-evals/evals/pairing-multi-agent-review/**' + - 'tools/skill-evals/evals/pairing-self-review/**' + +family:pr-management: + - changed-files: + - any-glob-to-any-file: + - 'docs/pr-management/**' + - 'plugins/magpie-pr-management/**' + - 'tools/skill-evals/evals/pr-management-code-review/**' + - 'tools/skill-evals/evals/pr-management-mentor/**' + - 'tools/skill-evals/evals/pr-management-quick-merge/**' + - 'tools/skill-evals/evals/pr-management-stats/**' + - 'tools/skill-evals/evals/pr-management-triage/**' + - 'tools/skill-evals/evals/pr-stale-sweep/**' + - 'tools/skill-evals/evals/pre-first-pr-check/**' + - 'tools/skill-evals/evals/reviewer-routing/**' + +family:release-management: + - changed-files: + - any-glob-to-any-file: + - 'docs/release-management/**' + - 'plugins/magpie-release-management/**' + - 'tools/skill-evals/evals/release-announce-draft/**' + - 'tools/skill-evals/evals/release-archive-sweep/**' + - 'tools/skill-evals/evals/release-audit-report/**' + - 'tools/skill-evals/evals/release-keys-sync/**' + - 'tools/skill-evals/evals/release-prepare/**' + - 'tools/skill-evals/evals/release-promote/**' + - 'tools/skill-evals/evals/release-rc-cut/**' + - 'tools/skill-evals/evals/release-verify-rc/**' + - 'tools/skill-evals/evals/release-vote-draft/**' + - 'tools/skill-evals/evals/release-vote-tally/**' + +family:repo-health: + - changed-files: + - any-glob-to-any-file: + - 'docs/repo-health/**' + - 'plugins/magpie-repo-health/**' + - 'tools/skill-evals/evals/audit-finding-fix/**' + - 'tools/skill-evals/evals/ci-runner-audit/**' + - 'tools/skill-evals/evals/dependency-audit/**' + - 'tools/skill-evals/evals/dependency-license-audit/**' + - 'tools/skill-evals/evals/flaky-test-triage/**' + - 'tools/skill-evals/evals/license-compliance-audit/**' + - 'tools/skill-evals/evals/workflow-security-audit/**' + +family:security: + - changed-files: + - any-glob-to-any-file: + - 'docs/security/**' + - 'plugins/magpie-security/**' + - 'tools/skill-evals/evals/security-cve-allocate/**' + - 'tools/skill-evals/evals/security-issue-deduplicate/**' + - 'tools/skill-evals/evals/security-issue-fix/**' + - 'tools/skill-evals/evals/security-issue-import-from-md/**' + - 'tools/skill-evals/evals/security-issue-import-from-pr/**' + - 'tools/skill-evals/evals/security-issue-import-from-scan/**' + - 'tools/skill-evals/evals/security-issue-import-via-forwarder/**' + - 'tools/skill-evals/evals/security-issue-import/**' + - 'tools/skill-evals/evals/security-issue-invalidate/**' + - 'tools/skill-evals/evals/security-issue-sync/**' + - 'tools/skill-evals/evals/security-issue-triage/**' + - 'tools/skill-evals/evals/security-model-prepare/**' + - 'tools/skill-evals/evals/security-model-update/**' + - 'tools/skill-evals/evals/security-model-verify/**' + - 'tools/skill-evals/evals/security-tracker-stats-dashboard/**' + +family:setup: + - changed-files: + - any-glob-to-any-file: + - '.apache-magpie-overrides/**' + - '.apache-magpie.lock' + - 'docs/setup/**' + - 'plugins/magpie-setup/**' + - 'tools/skill-evals/evals/setup-isolated-setup-doctor/**' + - 'tools/skill-evals/evals/setup-isolated-setup-install/**' + - 'tools/skill-evals/evals/setup-isolated-setup-update/**' + - 'tools/skill-evals/evals/setup-isolated-setup-verify/**' + - 'tools/skill-evals/evals/setup-override-upstream/**' + - 'tools/skill-evals/evals/setup-privacy-llm/**' + - 'tools/skill-evals/evals/setup-shared-config-sync/**' + - 'tools/skill-evals/evals/setup-status/**' + - 'tools/skill-evals/evals/setup-upstream-fix/**' + - 'tools/skill-evals/evals/setup/**' + +family:tools: + - any: + - changed-files: + - any-glob-to-any-file: + - 'plugins/magpie-adversarial-review/**' + - 'plugins/magpie-agent-guard/**' + - 'plugins/magpie-vetted-ops/**' + - changed-files: + - all-globs-to-any-file: + - 'tools/**' + - '!tools/skill-evals/evals/**' + - '!tools/spec-loop/specs/**' + +family:utilities: + - changed-files: + - any-glob-to-any-file: + - 'docs/utilities/**' + - 'plugins/magpie-utilities/**' + - 'tools/skill-evals/evals/list-skills/**' + - 'tools/skill-evals/evals/optimize-skill/**' + - 'tools/skill-evals/evals/report-framework-issue/**' + - 'tools/skill-evals/evals/skill-reconciler/**' + - 'tools/skill-evals/evals/write-skill/**' contract:change-request: - any: diff --git a/.github/workflows/labeler-signal.yml b/.github/workflows/labeler-signal.yml new file mode 100644 index 000000000..afcc66c18 --- /dev/null +++ b/.github/workflows/labeler-signal.yml @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +# A doorbell for labeler.yml, and nothing more. +# +# A `pull_request` run holds no privileges, so this one does nothing at all: +# it has no permissions, checks nothing out and runs no code. Its only effect +# is that it completes, which fires labeler.yml's `workflow_run` trigger in the +# default branch's context, where the labeler reads the pull request through +# the API and labels it. `edited` is included so a pull request that starts +# referring to an issue passes its labels on to that issue, and `closed` so a +# merge passes an outside contributor's labels on (see labeler.yml). +name: "Labeler signal" +"on": + pull_request: + types: [opened, reopened, synchronize, ready_for_review, edited, closed] + +permissions: {} + +jobs: + signal: + runs-on: ubuntu-slim + timeout-minutes: 2 + steps: + - name: Signal the labeler + run: echo "labeler.yml runs on this workflow's completion" diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 359058260..f957765ea 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -15,65 +15,178 @@ # specific language governing permissions and limitations # under the License. # -# Applies the tool-capability labels (`contract:*` / `substrate:*`) to new -# pull requests from the tool directories they touch. The mapping is -# `.github/labeler.yml`, generated from each tool README's `**Capability:**` -# line by tools/dev/generate-labeler-config.py. +# Labels pull requests from the files they touch, and passes those labels on +# to the issues each pull request closes or refers to. The mapping is +# `.github/labeler.yml`, generated by tools/dev/generate-labeler-config.py +# from tool READMEs (`contract:*` / `substrate:*`) and skill frontmatter +# (`family:*` / `capability:*`); see docs/labels-and-capabilities.md. # -# It runs on a schedule rather than on a pull-request event: a scheduled run -# executes in this repository's context, so its token can label fork PRs -# without `pull_request_target`, and no PR event ever starts it. Nothing from -# a PR is checked out or run; the labeler reads the changed-file list and the -# config (from the default branch) through the API. +# Privilege boundary — read this before changing any trigger. # -# Each run labels the open PRs created since the previous successful run -# started, so a PR is labelled once, shortly after it is opened. Path-based -# labels are a starting point (docs/labels-and-capabilities.md asks for the -# capability the change *implements*), and a label a maintainer removes -# afterwards is not re-added. +# This workflow holds a token that can label pull requests and issues, so it +# never checks out, builds or runs anything from a pull request. It does not +# use pull_request_target. Its triggers are signals only: +# - workflow_run fires when labeler-signal.yml (an unprivileged +# `pull_request` run that does nothing) completes. It always runs this file +# from the default branch, which is the property that makes it safe, and +# labels the pull request the moment it is opened or pushed to. +# - schedule is a daily safety net: it labels any open pull request that +# still has no `family:*` label (a run that failed or was dropped). +# - workflow_dispatch labels one pull request, or runs the safety net. +# The only value taken from the triggering event is the head SHA, checked to be +# 40 hex characters and used solely to find the pull request among the open +# ones. A pull request's body is read only to extract issue numbers, which are +# checked to be issues before any label is added; no text from it reaches a +# command. The labeler action reads the changed-file list and the config (from +# the default branch) through the API. +# +# Who decides which issues get labels: a pull request's body names them, and +# its author can edit the body at any time, even after the merge. So the body +# is trusted only when the author is an OWNER, MEMBER or COLLABORATOR. For +# anyone else the body is never read: their pull request labels only the issues +# its merge actually closed, which GitHub records as the issue's closer and +# nobody can edit afterwards. +# +# Labels are only ever added. A label a maintainer removes is re-added only +# when the pull request is pushed to again and still matches the rule. --- -name: "Tool capability labels" +name: "Pull request labels" "on": + workflow_run: # zizmor: ignore[dangerous-triggers] -- default-branch code, no PR input; see header + workflows: ["Labeler signal"] + types: [completed] schedule: - - cron: "17 * * * *" + - cron: "17 3 * * *" workflow_dispatch: + inputs: + pr: + description: "Label this pull request number (blank: every open pull request without a family label)" + required: false + type: string permissions: {} concurrency: - group: tool-capability-labels + group: pull-request-labels cancel-in-progress: false jobs: label: - name: Label new pull requests + name: Label pull requests and their issues + if: >- + github.event_name != 'workflow_run' || + github.event.workflow_run.event == 'pull_request' runs-on: ubuntu-slim - timeout-minutes: 5 + timeout-minutes: 10 permissions: - actions: read # find the previous successful run - contents: read # read .github/labeler.yml and the PRs' changed files - pull-requests: write # add the labels + contents: read # read .github/labeler.yml and the pull requests' changed files + pull-requests: write # add labels to pull requests + issues: write # add the same labels to the issues they close or refer to steps: - - name: Select pull requests opened since the last run + - name: Select pull requests id: select env: GH_TOKEN: ${{ github.token }} REPO: ${{ github.repository }} + EVENT: ${{ github.event_name }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + PR_INPUT: ${{ inputs.pr }} run: | set -euo pipefail - since=$(gh api "repos/$REPO/actions/workflows/labeler.yml/runs?status=success&per_page=1" \ - --jq '.workflow_runs[0].run_started_at // empty') - if [ -z "$since" ]; then - since=$(date -u -d '24 hours ago' +%Y-%m-%dT%H:%M:%SZ) - fi # Dependency and version bumps implement no capability: skip bot authors. - prs=$(gh pr list --repo "$REPO" --state open --limit 100 \ - --search "created:>=$since" \ - --json number,author --jq '.[] | select(.author.is_bot | not) | .number') - echo "since $since: ${prs:-none}" | tr '\n' ' ' + open_prs() { + gh pr list --repo "$REPO" --state open --limit 200 \ + --json number,headRefOid,author,labels --jq "$1" + } + case "$EVENT" in + workflow_run) + if ! [[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::unexpected head SHA"; exit 1 + fi + prs=$(open_prs ".[] | select(.headRefOid == \"$HEAD_SHA\" and (.author.is_bot | not)) | .number") + if [ -z "$prs" ]; then # closed: the merged pull request this commit belongs to + prs=$(gh api "repos/$REPO/commits/$HEAD_SHA/pulls" \ + --jq '.[] | select(.merged_at != null and (.user.type != "Bot")) | .number') + fi + ;; + workflow_dispatch) + if [ -n "$PR_INPUT" ]; then + if ! [[ "$PR_INPUT" =~ ^[0-9]+$ ]]; then + echo "::error::pr must be a pull request number"; exit 1 + fi + prs=$PR_INPUT + else + prs=$(open_prs '.[] | select((.author.is_bot | not) and ([.labels[].name | startswith("family:")] | any | not)) | .number') + fi + ;; + *) + prs=$(open_prs '.[] | select((.author.is_bot | not) and ([.labels[].name | startswith("family:")] | any | not)) | .number') + ;; + esac + echo "pull requests: ${prs:-none}" | tr '\n' ' ' { echo "prs<> "$GITHUB_OUTPUT" + - if: steps.select.outputs.prs != '' uses: actions/labeler@bf12e9b00b37c5c0ca2b87b79b2daf7891dbda13 # v7.0.0 with: pr-number: ${{ steps.select.outputs.prs }} + + - name: Pass the labels on to linked issues + if: steps.select.outputs.prs != '' + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PRS: ${{ steps.select.outputs.prs }} + run: | + set -euo pipefail + for pr in $PRS; do + [[ "$pr" =~ ^[0-9]+$ ]] || continue + read -r assoc merged < <(gh api "repos/$REPO/pulls/$pr" --jq '"\(.author_association) \(.merged)"') + labels=$(gh pr view "$pr" --repo "$REPO" --json labels \ + --jq '[.labels[].name | select(test("^(family|capability|contract|substrate):"))] | join(",")') + [ -n "$labels" ] || continue + closing=$(gh pr view "$pr" --repo "$REPO" --json closingIssuesReferences \ + --jq '.closingIssuesReferences[].number') + case "$assoc" in + OWNER|MEMBER|COLLABORATOR) + # A project member's body is trusted: the issues it closes, and the + # issues it refers to (#N, or this repository's issue URL). + mentioned=$(gh pr view "$pr" --repo "$REPO" --json body --jq '.body // ""' \ + | grep -oE "(^|[^&0-9A-Za-z_/])#[0-9]+|github\.com/${REPO}/issues/[0-9]+" \ + | grep -oE '[0-9]+$' || true) + ;; + *) + # Anyone else: never the body, and only once merged. Keep just the + # issues whose recorded closer is this pull request. + if [ "$merged" != "true" ]; then + echo "#$pr: author is $assoc and it is not merged; its closed issues are labelled on merge" + continue + fi + mentioned="" + verified="" + for issue in $closing; do + [[ "$issue" =~ ^[0-9]+$ ]] || continue + closer=$(gh api graphql -F owner="${REPO%/*}" -F name="${REPO#*/}" -F number="$issue" -f query=' + query($owner: String!, $name: String!, $number: Int!) { + repository(owner: $owner, name: $name) { + issue(number: $number) { + timelineItems(itemTypes: [CLOSED_EVENT], last: 10) { + nodes { ... on ClosedEvent { closer { ... on PullRequest { number } } } } + } + } + } + }' --jq '[.data.repository.issue.timelineItems.nodes[].closer.number // empty] | map(tostring) | join(" ")' 2>/dev/null || true) + if [[ " $closer " == *" $pr "* ]]; then verified=$(printf '%s\n%s' "$verified" "$issue"); fi + done + closing=$verified + ;; + esac + for issue in $(printf '%s\n%s\n' "$closing" "$mentioned" | grep -E '^[0-9]+$' | sort -un | head -20); do + [ "$issue" = "$pr" ] && continue + kind=$(gh api "repos/$REPO/issues/$issue" --jq 'if .pull_request then "pull" else "issue" end' 2>/dev/null || true) + [ "$kind" = "issue" ] || continue + echo "#$pr -> issue #$issue: $labels" + gh issue edit "$issue" --repo "$REPO" --add-label "$labels" + done + done diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 07641528a..04baf5dc8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -533,10 +533,20 @@ repos: - repo: local hooks: - id: generate-labeler-config - name: generate-labeler-config (tool READMEs -> .github/labeler.yml) + name: generate-labeler-config (tool READMEs + skill frontmatter -> .github/labeler.yml) language: system entry: tools/dev/generate-labeler-config.py - files: ^(tools/[^/]+/README\.md|\.github/labeler\.yml|tools/dev/generate-labeler-config\.py)$ + files: ^(tools/[^/]+/README\.md|\.github/labeler\.yml|tools/dev/generate-labeler-config\.py|plugins/magpie-[^/]+/skills/[^/]+/SKILL\.md|skills/[^/]+|docs/labels-and-capabilities\.md)$ + pass_filenames: false + # Every label docs/labels-and-capabilities.md defines must have a rule in + # .github/labeler.yml (or an UNMAPPED entry with a reason), and no rule + # may name an undefined label: a label nobody can apply automatically is + # how pull requests ended up unlabelled. + - id: check-labeler-coverage + name: check-labeler-coverage (every taxonomy label has a labeler rule) + language: system + entry: tools/dev/generate-labeler-config.py --check-coverage + files: ^(tools/[^/]+/README\.md|\.github/labeler\.yml|tools/dev/generate-labeler-config\.py|plugins/magpie-[^/]+/skills/[^/]+/SKILL\.md|skills/[^/]+|docs/labels-and-capabilities\.md)$ pass_filenames: false # Workspace-level static checks. Iterate over every uv-workspace # member declared in the root `pyproject.toml`'s diff --git a/docs/labels-and-capabilities.md b/docs/labels-and-capabilities.md index 7abfbda56..5e99e9748 100644 --- a/docs/labels-and-capabilities.md +++ b/docs/labels-and-capabilities.md @@ -445,16 +445,21 @@ that adjusts the validator config to support a new triage rule is `capability:triage` (the change's purpose), not `substrate:framework-dev` (the file it edited). -The tool-capability labels are pre-applied within an hour of a PR being opened: -the scheduled [`.github/workflows/labeler.yml`](../.github/workflows/labeler.yml) -labels each new PR once, with the `**Capability:**` of every tool directory it -touches, from -[`.github/labeler.yml`](../.github/labeler.yml), which +Labels are pre-applied the moment a PR is opened or pushed to: +[`.github/workflows/labeler.yml`](../.github/workflows/labeler.yml) +runs on the completion of the unprivileged +[`labeler-signal.yml`](../.github/workflows/labeler-signal.yml), from the default branch, +and applies the rules in [`.github/labeler.yml`](../.github/labeler.yml), which [`tools/dev/generate-labeler-config.py`](../tools/dev/generate-labeler-config.py) -generates from the tool READMEs. +generates from the repository's own declarations: +the `family:` and `capability:` frontmatter of every skill (covering its directory and its eval suite), +the `**Capability:**` line of every tool README, +and the paths of the non-skill families (`family:tools`, `family:ci`, `family:docs`). +The same labels are passed on to the issues the PR closes or refers to. +A daily run labels any open PR still without a `family:*` label. That is a starting point, not the answer: remove a label the change does not -implement, and add the skill capability yourself. -Bot PRs and sweeps that would gain more than eight labels are left unlabelled. +implement, and add the capability it does implement when the paths do not show it. +Bot PRs are left unlabelled. ### A new tool under `tools/` diff --git a/tools/dev/README.md b/tools/dev/README.md index f1d3c7168..e721286b4 100644 --- a/tools/dev/README.md +++ b/tools/dev/README.md @@ -65,7 +65,7 @@ installable for other members to depend on it. | [`gh-signed-commit.py`](gh-signed-commit.py) | Commits the working tree through GitHub's `createCommitOnBranch` mutation instead of `git commit` + `git push`, so the commit is signed by GitHub and shows as **Verified** with no key material in CI. Collects changed and deleted paths from `git status --porcelain -z`, decomposes renames (the mutation has no rename concept), and pins `expectedHeadOid` so a concurrent push fails the call rather than being overwritten. Used by [`bump-dev-version.yml`](../../.github/workflows/bump-dev-version.yml). | | [`check-placeholders.sh`](check-placeholders.sh) | Fails the build on hardcoded project references in skill and tool docs, which must use `` / `` / `` / `` instead. Carries both casings and matches spaced variants. | | [`check-workspace-members.py`](check-workspace-members.py) | Catches a new `tools//pyproject.toml` that was never added to `[tool.uv.workspace] members` — an omission that silently drops the tool from both the pre-commit hooks and the CI pytest matrix. Also verifies each member's tests actually run: both surfaces key off `[tool.pytest.ini_options]`, so a project can carry a full `tests/` directory and be executed by nothing. Reports tests-without-config, config-without-tests, and neither; `[tool.magpie.checks] skip = ["pytest"]` is the declared exemption. | -| [`generate-labeler-config.py`](generate-labeler-config.py) | Generates `.github/labeler.yml` — the path → label map the [`labeler.yml`](../../.github/workflows/labeler.yml) workflow uses to pre-apply `contract:*` / `substrate:*` labels to a new PR — from the `**Capability:**` line of every `tools//README.md`, so a new tool or a changed capability needs no hand-edit. A skill's eval fixtures under `tools/skill-evals/evals/` and the spec-loop specs do not count as touching their tool. Rewrites in place and exits 1 on change, which is how the prek hook runs it; `--check` only reports. | +| [`generate-labeler-config.py`](generate-labeler-config.py) | Generates `.github/labeler.yml` — the path → label map the [`labeler.yml`](../../.github/workflows/labeler.yml) workflow uses to pre-apply labels to a PR and its linked issues — from the `**Capability:**` line of every `tools//README.md` (`contract:*` / `substrate:*`), the `family:` / `capability:` frontmatter of every skill (its directory and its eval suite), and the paths of the non-skill families (`family:tools`, `family:ci`, `family:docs`), so a new tool or skill needs no hand-edit. Only labels `docs/labels-and-capabilities.md` defines are emitted. A skill's eval fixtures under `tools/skill-evals/evals/` and the spec-loop specs do not count as touching their tool. Rewrites in place and exits 1 on change, which is how the prek hook runs it; `--check` only reports. `--check-coverage` (the `check-labeler-coverage` hook) fails when a label the taxonomy defines has no rule, unless it is listed in `UNMAPPED` with a reason, or when a rule names an undefined label. | | [`run-workspace-check.sh`](run-workspace-check.sh) | Runs one static-check or test command across every workspace member, auto-discovering which members a given check applies to. The four `workspace-*` hooks call it, so adding a tool needs no edit to the pre-commit config. | | [`run-skill-script-tests.sh`](run-skill-script-tests.sh) | Runs the stdlib `unittest` suites under `plugins/*/skills/*/tests`, which cover skills' sibling scripts. Those scripts are not workspace members, so the `workspace-pytest` hook never reaches them. Runs as the `skill-script-tests` pre-commit hook. | | [`add-license-headers.py`](add-license-headers.py) | Stamps the SPDX licence header into Markdown files that lack one. | diff --git a/tools/dev/generate-labeler-config.py b/tools/dev/generate-labeler-config.py index 849fb9062..c45befce4 100755 --- a/tools/dev/generate-labeler-config.py +++ b/tools/dev/generate-labeler-config.py @@ -15,17 +15,31 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Generate `.github/labeler.yml` from the tool READMEs' `**Capability:**` lines. +"""Generate `.github/labeler.yml` from the repository's own declarations. -Every `tools//README.md` declares the tool capabilities it provides -(`contract:*` / `substrate:*`, see docs/labels-and-capabilities.md). The -`labeler` workflow applies those labels to a pull request that touches the -tool, so the README line is the single source of truth: a new tool, or a -changed capability, needs no hand-edit of the labeler config. +The `labeler` workflow applies these labels to a pull request from the files +it touches, so each label's source of truth stays where it is declared and a +new tool or skill needs no hand-edit of the labeler config: -Files that live under a tool directory but belong to something else are -excluded: a skill's eval fixtures under `tools/skill-evals/evals/`, and the -spec-loop specs and sync marker under `tools/spec-loop/`. +- `contract:*` / `substrate:*` come from the `**Capability:**` line of every + `tools//README.md`. +- `family:*` and `capability:*` come from each skill's `family:` and + `capability:` frontmatter, applied to the skill's directory and to its eval + suite under `tools/skill-evals/evals/` (found through the `skills/` + symlink that names it). +- The three non-skill families of docs/labels-and-capabilities.md map to their + paths: `family:tools` (tools, and plugins that only package a tool), + `family:ci` (`.github/`, the pre-commit config, `tools/dev/`, root tooling + files such as `pyproject.toml` and `.gitignore`), `family:setup` (Magpie's + own `.apache-magpie-overrides/` and pin) and + `family:docs` (`docs/`, `MISSION.md`, every `README.md` and root `*.md`). + A family's guide under `docs//` also gets that family. + +Only labels the taxonomy doc defines are emitted, so a typo in frontmatter +cannot create a label. Files that live under a tool directory but belong to +something else are excluded from the tool labels: a skill's eval fixtures +under `tools/skill-evals/evals/`, and the spec-loop specs and sync marker +under `tools/spec-loop/`. Run as a prek hook. Rewrites the file in place and exits 1 if it changed (re-stage and commit again); `--check` reports drift without writing. @@ -53,7 +67,38 @@ # A pull request that would gain more path-based labels than this is a # cross-cutting sweep (licence headers, dependency bumps across every tool); # the labeler then applies none and leaves the capability to the maintainer. -LABELS_LIMIT = 8 +# actions/labeler applies *no* changed-files label when more new ones match +# than this, so the limit is a cliff, not a cap. Keep it well above what a +# normal pull request touches (a family, a capability or two, a few tools). +LABELS_LIMIT = 20 + +TAXONOMY_RELPATH = Path("docs") / "labels-and-capabilities.md" +_ALL_TAXONOMY_RE = re.compile(r"^\| `((?:family|capability|contract|substrate):[a-z0-9-]+)` \|", re.MULTILINE) +_CONFIG_LABEL_RE = re.compile(r"^([a-z]+:[a-z0-9-]+):$", re.MULTILINE) + +# Taxonomy labels that are deliberately applied by hand, never from paths. +# Each needs a reason; an entry here is the only way a label may lack a rule. +UNMAPPED: dict[str, str] = {} +_TAXONOMY_RE = re.compile(r"^\| `((?:family|capability):[a-z0-9-]+)` \|", re.MULTILINE) + +# The non-skill families and the paths they cover. +PATH_FAMILIES: dict[str, tuple[str, ...]] = { + "family:ci": ( + ".github/**", + ".pre-commit-config.yaml", + "tools/dev/**", + ".gitignore", + ".gitattributes", + "pyproject.toml", + "uv.lock", + ".typos.toml", + ".rat-excludes", + ".lychee.toml", + ), + # Magpie's own adoption of the framework: the committed overrides and pin. + "family:setup": (".apache-magpie-overrides/**", ".apache-magpie.lock"), + "family:docs": ("docs/**", "MISSION.md", "*.md", "**/README.md"), +} HEADER = """\ # Licensed to the Apache Software Foundation (ASF) under one @@ -74,8 +119,9 @@ # under the License. # # GENERATED by tools/dev/generate-labeler-config.py from the -# `**Capability:**` line of every tools//README.md. Do not edit by -# hand; change the README and let the prek hook regenerate this file. +# `**Capability:**` line of every tools//README.md and the `family:` / +# `capability:` frontmatter of every skill. Do not edit by hand; change the +# README or the skill and let the prek hook regenerate this file. --- """ @@ -94,8 +140,113 @@ def load_capabilities(root: Path) -> dict[str, list[str]]: return by_label -def render(by_label: dict[str, list[str]]) -> str: +def taxonomy_labels(root: Path) -> set[str]: + """The `family:*` and `capability:*` labels docs/labels-and-capabilities.md defines.""" + path = root / TAXONOMY_RELPATH + return set(_TAXONOMY_RE.findall(path.read_text(encoding="utf-8"))) if path.is_file() else set() + + +def _frontmatter(skill_md: Path) -> dict[str, list[str]]: + """`family` and `capability` from a SKILL.md frontmatter, as lists of values.""" + lines = skill_md.read_text(encoding="utf-8").split("\n") + if not lines or lines[0].strip() != "---": + return {} + out: dict[str, list[str]] = {} + key = None + for line in lines[1:]: + if line.strip() == "---": + break + top = re.match(r"^([a-z_]+):\s*(.*)$", line) + if top: + key, value = top.group(1), top.group(2).strip() + if key in ("family", "capability") and value: + out[key] = [value] + elif key in ("family", "capability"): + out[key] = [] + continue + item = re.match(r"^\s+-\s+(.+)$", line) + if item and key in ("family", "capability"): + out[key].append(item.group(1).strip()) + return out + + +def load_path_rules(root: Path) -> dict[str, list[str]]: + """Map each `family:*` / `capability:*` label to the sorted globs it covers.""" + known = taxonomy_labels(root) + rules: dict[str, set[str]] = {} + excluded: dict[str, tuple[str, ...]] = {} + + def add(label: str, *globs: str) -> None: + if label in known: + rules.setdefault(label, set()).update(globs) + + evals_for: dict[Path, str] = {} + for link in sorted((root / "skills").glob("*")): + if link.is_symlink(): + evals_for[link.resolve()] = link.name + for skill_md in sorted((root / "plugins").glob("magpie-*/skills/*/SKILL.md")): + skill_dir = skill_md.parent + globs = [f"{skill_dir.relative_to(root).as_posix()}/**"] + suite = evals_for.get(skill_dir.resolve()) + if suite and (root / "tools" / "skill-evals" / "evals" / suite).is_dir(): + globs.append(f"tools/skill-evals/evals/{suite}/**") + meta = _frontmatter(skill_md) + for family in meta.get("family", []): + add(f"family:{family}", *globs) + for capability in meta.get("capability", []): + add(capability, *globs) + for plugin in sorted((root / "plugins").glob("magpie-*")): + if not plugin.is_dir(): + continue + family = f"family:{plugin.name.removeprefix('magpie-')}" + if (plugin / "skills").is_dir(): + add(family, f"plugins/{plugin.name}/**") + else: # a plugin that only packages a tool + add("family:tools", f"plugins/{plugin.name}/**") + if (root / "docs" / plugin.name.removeprefix("magpie-")).is_dir(): + add(family, f"docs/{plugin.name.removeprefix('magpie-')}/**") + if (root / "tools").is_dir(): + add("family:tools", "tools/**") + if "family:tools" in rules: # skill evals and specs are not tool code + excluded["family:tools"] = ("tools/skill-evals/evals/**", "tools/spec-loop/specs/**") + for label, path_globs in PATH_FAMILIES.items(): + add(label, *path_globs) + + def prune(globs: set[str]) -> list[str]: + """Drop a glob that a broader `/**` in the same rule already covers.""" + trees = [g[:-2] for g in globs if g.endswith("/**")] + return sorted(g for g in globs if not any(g != t + "**" and g.startswith(t) for t in trees)) + + out = {label: prune(globs) for label, globs in rules.items()} + for label, negations in excluded.items(): + out[label] = out[label] + [f"!{glob}" for glob in negations] + return out + + +def render(by_label: dict[str, list[str]], path_rules: dict[str, list[str]] | None = None) -> str: lines = [HEADER.rstrip("\n"), f"changed-files-labels-limit: {LABELS_LIMIT}", ""] + for label in sorted(path_rules or {}): + globs = (path_rules or {})[label] + negations = [g for g in globs if g.startswith("!")] + if not negations: + lines += [f"{label}:", " - changed-files:", " - any-glob-to-any-file:"] + lines += [f" - '{glob}'" for glob in globs] + else: + # A negation only works inside all-globs-to-any-file, so the excluded + # tree gets its own group, OR-ed with the plain globs. + tree = "tools/**" + plain = [g for g in globs if not g.startswith("!") and g != tree] + lines += [f"{label}:", " - any:"] + if plain: + lines += [" - changed-files:", " - any-glob-to-any-file:"] + lines += [f" - '{glob}'" for glob in plain] + lines += [ + " - changed-files:", + " - all-globs-to-any-file:", + f" - '{tree}'", + ] + lines += [f" - '{glob}'" for glob in negations] + lines.append("") for label in sorted(by_label): tools = by_label[label] plain = [t for t in tools if t not in EXCLUDES] @@ -111,14 +262,48 @@ def render(by_label: dict[str, list[str]]) -> str: return "\n".join(lines).rstrip("\n") + "\n" +def coverage_problems(root: Path) -> list[str]: + """Taxonomy labels with no labeler rule, and labeler rules for undefined labels.""" + doc = root / TAXONOMY_RELPATH + defined = set(_ALL_TAXONOMY_RE.findall(doc.read_text(encoding="utf-8"))) if doc.is_file() else set() + config = root / CONFIG_RELPATH + configured = ( + set(_CONFIG_LABEL_RE.findall(config.read_text(encoding="utf-8"))) if config.is_file() else set() + ) + problems = [ + f"{label} is defined in {TAXONOMY_RELPATH.as_posix()} but no rule applies it: declare it in a skill's " + "frontmatter or a tool README's **Capability:** line, or list it in UNMAPPED with a reason" + for label in sorted(defined - configured - set(UNMAPPED)) + ] + problems += [ + f"{label} has a labeler rule but is not defined in {TAXONOMY_RELPATH.as_posix()}" + for label in sorted(configured - defined) + ] + problems += [ + f"UNMAPPED lists {label}, which is not a taxonomy label" for label in sorted(set(UNMAPPED) - defined) + ] + return problems + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--check", action="store_true", help="report drift without rewriting the file") + parser.add_argument( + "--check-coverage", + action="store_true", + help="fail when a taxonomy label has no labeler rule, or a rule names an undefined label", + ) parser.add_argument("--root", type=Path, default=REPO_ROOT, help=argparse.SUPPRESS) args = parser.parse_args(argv) + if args.check_coverage: + problems = coverage_problems(args.root) + for problem in problems: + print(problem, file=sys.stderr) + return 1 if problems else 0 + path = args.root / CONFIG_RELPATH - expected = render(load_capabilities(args.root)) + expected = render(load_capabilities(args.root), load_path_rules(args.root)) current = path.read_text(encoding="utf-8") if path.is_file() else "" if current == expected: return 0 diff --git a/tools/dev/tests/test_generate_labeler_config.py b/tools/dev/tests/test_generate_labeler_config.py index abce64775..a4f568e02 100644 --- a/tools/dev/tests/test_generate_labeler_config.py +++ b/tools/dev/tests/test_generate_labeler_config.py @@ -80,3 +80,113 @@ def test_main_rewrites_then_reports_in_sync(tmp_path: Path) -> None: def test_committed_config_is_in_sync() -> None: assert mod.main(["--check"]) == 0 + + +_TAXONOMY = """ +| `family:release-management` | opt-in | release skills | +| `family:tools` | Substrate tools | +| `family:ci` | workflows | +| `family:docs` | docs | +| `capability:resolve` | Resolve. | +| `capability:triage` | Triage. | +""" + + +def _skill(root: Path, plugin: str, name: str, link: str, frontmatter: str) -> None: + d = root / "plugins" / plugin / "skills" / name + d.mkdir(parents=True) + (d / "SKILL.md").write_text(f"---\nname: {name}\n{frontmatter}---\n# {name}\n", encoding="utf-8") + (root / "skills").mkdir(exist_ok=True) + (root / "skills" / link).symlink_to(Path("..") / "plugins" / plugin / "skills" / name) + (root / "tools" / "skill-evals" / "evals" / link).mkdir(parents=True, exist_ok=True) + + +def _taxonomy(root: Path) -> None: + (root / "docs").mkdir(exist_ok=True) + (root / "docs" / "labels-and-capabilities.md").write_text(_TAXONOMY, encoding="utf-8") + + +def test_skill_family_and_capabilities_cover_skill_and_eval_suite(tmp_path: Path) -> None: + _taxonomy(tmp_path) + _skill( + tmp_path, + "magpie-release-management", + "rc-cut", + "release-rc-cut", + "family: release-management\ncapability:\n - capability:resolve\n - capability:triage\n", + ) + rules = mod.load_path_rules(tmp_path) + skill = "plugins/magpie-release-management/skills/rc-cut/**" + suite = "tools/skill-evals/evals/release-rc-cut/**" + assert rules["capability:resolve"] == [skill, suite] + assert rules["capability:triage"] == [skill, suite] + # the plugin-wide glob already covers the skill directory, so it is pruned + assert rules["family:release-management"] == ["plugins/magpie-release-management/**", suite] + + +def test_unknown_labels_are_never_emitted(tmp_path: Path) -> None: + _taxonomy(tmp_path) + _skill( + tmp_path, + "magpie-release-management", + "rc-cut", + "release-rc-cut", + "family: releases\ncapability: capability:resolving\n", + ) + rules = mod.load_path_rules(tmp_path) + assert "family:releases" not in rules and "capability:resolving" not in rules + + +def test_tool_only_plugin_and_tools_tree_are_family_tools_without_evals(tmp_path: Path) -> None: + _taxonomy(tmp_path) + (tmp_path / "plugins" / "magpie-agent-guard" / "tools").mkdir(parents=True) + _tool(tmp_path, "osv", "contract:security-cross-ref") + out = mod.render(mod.load_capabilities(tmp_path), mod.load_path_rules(tmp_path)) + block = out.split("family:tools:\n", 1)[1].split("\n\n", 1)[0] + assert "'plugins/magpie-agent-guard/**'" in block + assert ( + "- all-globs-to-any-file:\n - 'tools/**'\n - '!tools/skill-evals/evals/**'" + in block + ) + + +def test_limit_is_not_a_low_cliff() -> None: + # actions/labeler drops every changed-files label when more than the limit match + assert mod.LABELS_LIMIT >= 20 + + +def test_coverage_flags_a_taxonomy_label_without_a_rule(tmp_path: Path) -> None: + _taxonomy(tmp_path) + (tmp_path / ".github").mkdir() + mod.main(["--root", str(tmp_path)]) # nothing declares capability:resolve or :triage + problems = mod.coverage_problems(tmp_path) + assert any(p.startswith("capability:resolve is defined") for p in problems) + assert mod.main(["--root", str(tmp_path), "--check-coverage"]) == 1 + + +def test_coverage_passes_once_every_label_has_a_rule(tmp_path: Path) -> None: + _taxonomy(tmp_path) + _skill( + tmp_path, + "magpie-release-management", + "rc-cut", + "release-rc-cut", + "family: release-management\ncapability:\n - capability:resolve\n - capability:triage\n", + ) + (tmp_path / "plugins" / "magpie-agent-guard" / "tools").mkdir(parents=True) + (tmp_path / ".github").mkdir() + mod.main(["--root", str(tmp_path)]) + assert mod.coverage_problems(tmp_path) == [] + + +def test_coverage_flags_a_rule_for_an_undefined_label(tmp_path: Path) -> None: + _taxonomy(tmp_path) + (tmp_path / ".github").mkdir() + (tmp_path / ".github" / "labeler.yml").write_text( + "family:nonsense:\n - changed-files: []\n", encoding="utf-8" + ) + assert any("family:nonsense has a labeler rule" in p for p in mod.coverage_problems(tmp_path)) + + +def test_committed_config_covers_every_taxonomy_label() -> None: + assert mod.coverage_problems(mod.REPO_ROOT) == []