From 1b8f67d0b4e1b1432b1381756bf8f99132b7844f Mon Sep 17 00:00:00 2001 From: Sal Date: Wed, 19 Aug 2026 01:59:37 +0100 Subject: [PATCH] ci(labels): add a labeler config audit against the canonical label set A dangling labeler reference never fails. actions/labeler applies labels through the issues API, which creates a label that does not exist instead of erroring, so a labeler.yml naming a deleted legacy label silently recreates it on the next matching pull request. An org-wide label cleanup then partially undoes itself with nothing in any log explaining why. That is how 'enhancement*' returned to z-shell/zi after the #467 cleanup deleted it. Add a read-only audit reporting labeler.yml keys absent from lib/labels.yml, with the canonical replacement where legacy_migrations records one. A live run across 93 repositories finds 162 non-canonical references in 23 repositories. Read-only by design: which labels land on a repository's pull requests is a per-repository editorial call, not a uniform substitution, so the script reports and the owning repository edits. Covers both actions/labeler schemas, since v4 and v5 are both in use across the org and only top-level keys are label names in either. Treats a missing config as absence rather than a finding, and lets any non-404 API error surface rather than reporting the repository as clean. Add 11 unit tests running against an injected fixture client with no live API calls, plus a workflow that runs them and self-audits this repository. --- .../workflows/labeler-config-audit-test.yml | 54 +++++ scripts/labeler-config-audit.rb | 225 ++++++++++++++++++ scripts/test-labeler-config-audit.rb | 173 ++++++++++++++ 3 files changed, 452 insertions(+) create mode 100644 .github/workflows/labeler-config-audit-test.yml create mode 100644 scripts/labeler-config-audit.rb create mode 100644 scripts/test-labeler-config-audit.rb diff --git a/.github/workflows/labeler-config-audit-test.yml b/.github/workflows/labeler-config-audit-test.yml new file mode 100644 index 000000000..70b3d7fc7 --- /dev/null +++ b/.github/workflows/labeler-config-audit-test.yml @@ -0,0 +1,54 @@ +--- +name: Labeler Config Audit Tests +on: + workflow_call: {} + push: + paths: + - "scripts/labeler-config-audit.rb" + - "scripts/test-labeler-config-audit.rb" + - "lib/labels.yml" + - ".github/workflows/labeler-config-audit-test.yml" + pull_request: + paths: + - "scripts/labeler-config-audit.rb" + - "scripts/test-labeler-config-audit.rb" + - "lib/labels.yml" + - ".github/workflows/labeler-config-audit-test.yml" + workflow_dispatch: {} + +# The suite runs entirely against an injected fixture client (see +# scripts/test-labeler-config-audit.rb); it makes no live `gh api` calls, so it +# needs no repository permissions beyond checkout. +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + labeler-config-audit-test: + name: Labeler Config Audit Tests + runs-on: ubuntu-latest + steps: + - name: "⤵️ Check out code from GitHub" + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + - name: "⚙️ Report Ruby version" + run: ruby --version + + - name: "🧹 Syntax check" + run: | + ruby -c scripts/labeler-config-audit.rb + ruby -c scripts/test-labeler-config-audit.rb + + - name: "🧪 Run labeler-config-audit unit tests" + run: ruby scripts/test-labeler-config-audit.rb + + # This repository's own labeler config must satisfy the canonical set. + # Auditing it here makes the gate self-applying rather than advisory, + # without reaching across repository boundaries from CI. + - name: "🔎 Audit this repository's labeler config" + run: ruby scripts/labeler-config-audit.rb --repo ${{ github.repository }} + env: + GH_TOKEN: ${{ github.token }} diff --git a/scripts/labeler-config-audit.rb b/scripts/labeler-config-audit.rb new file mode 100644 index 000000000..7fb2a94fc --- /dev/null +++ b/scripts/labeler-config-audit.rb @@ -0,0 +1,225 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "json" +require "open3" +require "optparse" +require "set" +require "yaml" + +# Read-only audit that reports `.github/labeler.yml` label keys which are not in +# the canonical set in lib/labels.yml. Tracked on z-shell/.github#527. +# +# Why this needs its own check rather than showing up in labels-sync.rb: a +# dangling labeler reference never fails. `actions/labeler` applies labels +# through the issues API, which *creates* a label that does not exist instead of +# erroring. So a labeler.yml naming a deleted legacy label silently recreates it +# on the next matching pull request, and an org-wide label cleanup partially +# undoes itself with nothing in any log to show why. That is exactly how +# `enhancement ✨` returned to z-shell/zi after the #467 cleanup deleted it. +# +# Read-only by design: rewriting a labeler.yml changes which labels land on +# future pull requests in that repository, which is a per-repository editorial +# call rather than a uniform mechanical substitution. This script reports; the +# repository owner edits. +module LabelerConfigAudit + SCHEMA = "z-shell/labeler-config-audit/v1" + CONFIG_PATHS = [".github/labeler.yml", ".github/labeler.yaml"].freeze + + class GitHubError < StandardError + attr_reader :status + + def initialize(message, status: nil) + super(message) + @status = status + end + end + + # Thin `gh api` wrapper, matching scripts/repo-settings-audit.rb's + # GitHubClient shape so tests can inject a fixture stand-in instead of + # shelling out. + class GitHubClient + def initialize(runner: nil) + @runner = runner || lambda { |command| Open3.capture3(*command) } + end + + # Returns nil for a missing file rather than raising. A repository without a + # labeler config is not a finding, and must not be reported as one. + def file(repo, path) + command = ["gh", "api", "--method", "GET", "repos/#{repo}/contents/#{path}"] + stdout, stderr, status = @runner.call(command) + + unless successful?(status) + error = parse_error(stderr) + return nil if error["status"] == 404 + + raise GitHubError.new(error.fetch("message", "GitHub API request failed"), status: error["status"]) + end + + decode(stdout) + end + + private + + def successful?(status) + status.respond_to?(:success?) ? status.success? : status == true + end + + def decode(body) + parsed = JSON.parse(body) + raise GitHubError, "contents response must be an object" unless parsed.is_a?(Hash) + return nil unless parsed["content"] + + parsed.fetch("content").unpack1("m").force_encoding(Encoding::UTF_8) + rescue JSON::ParserError => error + raise GitHubError, "GitHub API returned invalid JSON: #{error.message}" + end + + def parse_error(body) + parsed = JSON.parse(body) + return parsed.merge("status" => parsed["status"] || status_from(body)) if parsed.is_a?(Hash) + + { "message" => body.to_s.strip, "status" => status_from(body) } + rescue JSON::ParserError + { "message" => body.to_s.strip, "status" => status_from(body) } + end + + def status_from(body) + body.to_s[/HTTP\s+(\d{3})/, 1]&.to_i + end + end + + # The label keys of a labeler config. + # + # Both actions/labeler schemas are in use across the org: v4 maps a label to a + # list of globs, v5 nests them under `any`/`all`. Only the top-level keys are + # label names in either, so read those and ignore the values entirely. + def self.label_keys(source) + parsed = YAML.safe_load(source, aliases: true) + return [] unless parsed.is_a?(Hash) + + parsed.keys.map(&:to_s).reject(&:empty?) + rescue Psych::Exception => error + raise GitHubError, "labeler config is not valid YAML: #{error.message}" + end + + def self.canonical_labels(labels_file) + data = YAML.safe_load_file(labels_file) + Array(data.is_a?(Hash) ? data["labels"] : nil).filter_map { |entry| entry["name"] if entry.is_a?(Hash) }.to_set + end + + def self.legacy_map(labels_file) + data = YAML.safe_load_file(labels_file) + map = data.is_a?(Hash) ? data["legacy_migrations"] : nil + map.is_a?(Hash) ? map : {} + end + + # One repository's findings. `config` is nil when the repository has no + # labeler config at all, which is distinct from having one with no drift. + def self.audit_repo(repo, client:, canonical:, legacy:) + source = CONFIG_PATHS.filter_map { |path| client.file(repo, path) }.first + return { "repo" => repo, "config" => nil, "unknown" => [], "summary" => { "unknown" => 0 } } if source.nil? + + unknown = label_keys(source).reject { |key| canonical.include?(key) }.map do |key| + { "label" => key, "replacement" => legacy[key] } + end + + { + "repo" => repo, + "config" => true, + "unknown" => unknown, + "summary" => { "unknown" => unknown.length } + } + end + + def self.render_markdown(results, io) + drifted = results.select { |result| result.fetch("summary").fetch("unknown").positive? } + + io.puts "# Labeler config audit" + io.puts + io.puts "Repos scanned: #{results.length}" + io.puts "Repos with a labeler config: #{results.count { |result| result['config'] }}" + io.puts "Repos referencing non-canonical labels: #{drifted.length}" + io.puts + io.puts "A labeler key that is not in `lib/labels.yml` is recreated as a real" + io.puts "label the next time the config matches a pull request. This audit is" + io.puts "read-only; fixes belong in the owning repository." + + drifted.each do |result| + io.puts + io.puts "## #{result.fetch('repo')}" + io.puts + result.fetch("unknown").each do |entry| + replacement = entry["replacement"] + suffix = replacement ? " -> `#{replacement}`" : " (no canonical replacement recorded)" + io.puts "- `#{entry.fetch('label')}`#{suffix}" + end + end + end + + def self.run(argv, io: $stdout, client: nil) + options = { + labels_file: File.join(File.expand_path("..", __dir__), "lib", "labels.yml"), + org: "z-shell", + repos: [], + all_repos: false, + json: false + } + + parser = OptionParser.new do |opts| + opts.banner = "Usage: #{$PROGRAM_NAME} [options]" + opts.on("--labels-file PATH", "Canonical labels file") { |value| options[:labels_file] = value } + opts.on("--org ORG", "Organization for --all-repos") { |value| options[:org] = value } + opts.on("--repo OWNER/REPO", "Repository to audit; may be repeated") { |value| options[:repos] << value } + opts.on("--all-repos", "Audit every repository in --org") { options[:all_repos] = true } + opts.on("--json", "Emit JSON instead of Markdown") { options[:json] = true } + opts.on("-h", "--help", "Show this help") do + io.puts opts + return 0 + end + end + parser.parse!(argv) + + if !options[:all_repos] && options[:repos].empty? + warn parser + warn "\nerror: pass at least one --repo OWNER/REPO or --all-repos" + return 2 + end + + if options[:all_repos] && !options[:repos].empty? + warn parser + warn "\nerror: use either --all-repos or --repo values, not both" + return 2 + end + + client ||= GitHubClient.new + canonical = canonical_labels(options[:labels_file]) + legacy = legacy_map(options[:labels_file]) + + repos = options[:repos] + if options[:all_repos] + stdout, _stderr, _status = Open3.capture3("gh", "repo", "list", options[:org], "--limit", "1000", + "--json", "nameWithOwner") + repos = JSON.parse(stdout).map { |entry| entry.fetch("nameWithOwner") } + end + + results = repos.sort.map { |repo| audit_repo(repo, client: client, canonical: canonical, legacy: legacy) } + drifted = results.count { |result| result.fetch("summary").fetch("unknown").positive? } + + if options[:json] + io.puts JSON.pretty_generate( + "schema" => SCHEMA, + "labels_file" => options[:labels_file], + "repos_scanned" => results.length, + "repos_with_drift" => drifted, + "results" => results + ) + else + render_markdown(results, io) + end + + drifted.zero? ? 0 : 1 + end +end + +exit(LabelerConfigAudit.run(ARGV)) if $PROGRAM_NAME == __FILE__ diff --git a/scripts/test-labeler-config-audit.rb b/scripts/test-labeler-config-audit.rb new file mode 100644 index 000000000..1fa6edf58 --- /dev/null +++ b/scripts/test-labeler-config-audit.rb @@ -0,0 +1,173 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "json" +require "stringio" + +require_relative "labeler-config-audit" + +# A stand-in for LabelerConfigAudit::GitHubClient. Maps "repo:path" to either a +# config body or :missing, and mimics `gh api` well enough to exercise the +# 404-is-not-a-finding path. +class FixtureClient + def initialize(files) + @files = files + end + + def file(repo, path) + entry = @files["#{repo}:#{path}"] + return nil if entry.nil? || entry == :missing + + raise LabelerConfigAudit::GitHubError.new(entry.fetch(:message), status: entry[:status]) if entry.is_a?(Hash) + + entry + end +end + +class LabelerConfigAuditTest + LABELS_FILE = File.expand_path("../lib/labels.yml", __dir__) + V4_CONFIG = <<~YAML + "documentation 📝": + - "docs/*.md" + area:ci: + - ".github/workflows/*.yml" + YAML + V5_CONFIG = <<~YAML + "enhancement ✨": + - any: + - changed-files: + - any-glob-to-any-file: "lib/**" + type:bug: + - any: + - changed-files: + - any-glob-to-any-file: "src/**" + YAML + + def assert_equal(expected, actual) + raise "expected #{expected.inspect}, got #{actual.inspect}" unless expected == actual + end + + def assert(value, message = "expected truthy value") + raise message unless value + end + + def canonical + LabelerConfigAudit.canonical_labels(LABELS_FILE) + end + + def legacy + LabelerConfigAudit.legacy_map(LABELS_FILE) + end + + def audit(repo, files) + LabelerConfigAudit.audit_repo(repo, client: FixtureClient.new(files), canonical: canonical, legacy: legacy) + end + + # A repository with no labeler config is not a finding. Reporting it as one + # would make the audit noisy enough to be ignored. + def test_missing_config_is_not_a_finding + result = audit("z-shell/none", {}) + assert_equal(nil, result.fetch("config")) + assert_equal(0, result.fetch("summary").fetch("unknown")) + end + + def test_yaml_extension_is_also_read + files = { "z-shell/alt:.github/labeler.yaml" => "area:docs:\n - \"docs/**\"\n" } + result = audit("z-shell/alt", files) + assert_equal(true, result.fetch("config")) + assert_equal(0, result.fetch("summary").fetch("unknown")) + end + + def test_canonical_only_config_is_clean + files = { "z-shell/clean:.github/labeler.yml" => "area:ci:\n - \".github/**\"\ntype:docs:\n - \"docs/**\"\n" } + assert_equal(0, audit("z-shell/clean", files).fetch("summary").fetch("unknown")) + end + + # The reason this audit exists: a legacy key is reported with the canonical + # replacement so the fix is mechanical. + def test_legacy_key_reports_its_replacement + files = { "z-shell/drift:.github/labeler.yml" => V4_CONFIG } + result = audit("z-shell/drift", files) + assert_equal(1, result.fetch("summary").fetch("unknown")) + entry = result.fetch("unknown").first + assert_equal("documentation 📝", entry.fetch("label")) + assert_equal("type:docs", entry.fetch("replacement")) + end + + # An unknown key with no recorded migration still has to be reported, with a + # null replacement rather than a guess. + def test_unmapped_key_reports_nil_replacement + files = { "z-shell/odd:.github/labeler.yml" => "plugin 🧿:\n - \"*.zsh\"\n" } + entry = audit("z-shell/odd", files).fetch("unknown").first + assert_equal("plugin 🧿", entry.fetch("label")) + assert_equal(nil, entry.fetch("replacement")) + end + + # actions/labeler v5 nests globs under any/all. Only top-level keys are label + # names, so the nested structure must not leak into the findings. + def test_v5_schema_reads_only_top_level_keys + files = { "z-shell/v5:.github/labeler.yml" => V5_CONFIG } + result = audit("z-shell/v5", files) + assert_equal(1, result.fetch("summary").fetch("unknown")) + assert_equal("enhancement ✨", result.fetch("unknown").first.fetch("label")) + end + + def test_invalid_yaml_raises + files = { "z-shell/bad:.github/labeler.yml" => "a:\n - b\n :\t- oops\n" } + audit("z-shell/bad", files) + raise "expected GitHubError" + rescue LabelerConfigAudit::GitHubError => error + assert(error.message.include?("not valid YAML"), "unexpected message: #{error.message}") + end + + def test_json_output_and_exit_code_on_drift + io = StringIO.new + code = LabelerConfigAudit.run( + ["--repo", "z-shell/drift", "--json", "--labels-file", LABELS_FILE], + io: io, + client: FixtureClient.new("z-shell/drift:.github/labeler.yml" => V4_CONFIG) + ) + assert_equal(1, code) + payload = JSON.parse(io.string) + assert_equal(LabelerConfigAudit::SCHEMA, payload.fetch("schema")) + assert_equal(1, payload.fetch("repos_with_drift")) + end + + def test_clean_repo_exits_zero + io = StringIO.new + code = LabelerConfigAudit.run( + ["--repo", "z-shell/clean", "--labels-file", LABELS_FILE], + io: io, + client: FixtureClient.new("z-shell/clean:.github/labeler.yml" => "area:ci:\n - \".github/**\"\n") + ) + assert_equal(0, code) + assert(io.string.include?("Repos referencing non-canonical labels: 0"), io.string) + end + + def test_missing_target_argument_is_a_usage_error + assert_equal(2, LabelerConfigAudit.run([], io: StringIO.new, client: FixtureClient.new({}))) + end + + # A 404 means no config; any other API failure must surface rather than be + # silently reported as a clean repository. + def test_non_404_api_error_propagates + files = { "z-shell/boom:.github/labeler.yml" => { message: "server error", status: 500 } } + audit("z-shell/boom", files) + raise "expected GitHubError" + rescue LabelerConfigAudit::GitHubError => error + assert_equal(500, error.status) + end +end + +tests = LabelerConfigAuditTest.new +methods = LabelerConfigAuditTest.instance_methods(false).grep(/^test_/).sort +failures = methods.filter_map do |method| + tests.public_send(method) + puts "PASS #{method}" + nil +rescue StandardError => error + warn "FAIL #{method}: #{error.message}" + error +end + +exit(failures.empty? ? 0 : 1)