lint.sh

#!/usr/bin/env bash
#
# The list of linters, in one place, for all three callers.
#
# It lived in three: the lint job in .github/workflows/ci.yml, the lint section
# of scripts/validate.sh, and the Makefile's `lint` target — and all three
# disagreed. CI ran `yamllint --strict`; the other two left warnings as
# warnings, so a warning passed `make validate` and failed the pull request that
# followed. actionlint and editorconfig-checker were GitHub Actions, which meant
# they could not be run here at all: two linters whose first and only feedback
# was a red build. That is #68.
#
# Same fix as scripts/seed-validation-env.sh — one script, every caller calls
# it. The self-check at the bottom asserts the callers really do call it,
# because one definition only helps for as long as nothing routes around it.
#
# Every linter here runs at a version this repository pins (#848). actionlint,
# editorconfig-checker, zizmor, markdownlint-cli2, shellcheck, packer and tofu
# run from images pinned in stacks/observability/compose.yaml behind the `lint`
# profile, for the same reason `archiver` and `gitleaks` are there: an image
# this repository runs must be one Dependabot bumps and `make pin-digests` can
# re-digest (#65). markdownlint-cli2 and shellcheck were the last two to move:
# they ran from `npx --yes` and the runner's apt package, so the version that
# judged a pull request was whatever npm or Ubuntu served that day.
#
# yamllint publishes no image of its own, so it runs from `pipx run --spec`,
# with the version read from scripts/requirements-lint.txt, which Dependabot's
# pip entry for /scripts bumps.
#
# ansible-lint is on the pipx side too, and not by preference. The only images of it
# on Docker Hub are third-party rebuilds without upstream's version numbers, and
# scripts/pin-digests.sh resolves Docker Hub alone, so the upstream ghcr.io image
# could not be digest-pinned here. Its version is pinned instead in
# ansible/requirements-lint.txt, which Dependabot's pip entry for /ansible bumps
# — the same "one place, and the bot edits it" rule, kept by a different file.
#
# Usage:
#   scripts/lint.sh                       every linter; an unreachable one SKIPs
#   scripts/lint.sh --require-all         an unreachable linter FAILs (CI)
#   scripts/lint.sh --skips-file <path>   append one line per skip
#
# --require-all is the asymmetry that matters: a local run may honestly skip a
# linter it cannot reach, CI may not, because a linter skipped there is one
# nobody will ever run.
#
# WHAT EACH LINTER LOOKS AT, AND WHY THAT IS NOT OBVIOUS
#
# Three of these decide their own file list, and none of them reads .gitignore.
# That only matters because a workstation has trees a CI checkout does not —
# rendered config, certificates, backups, and another session's git worktree
# under .claude/worktrees/, which is an entire second copy of this repository.
# A linter that walks the tree finds all of it and fails locally for a change
# that is clean, while CI stays green. Green where nothing runs and red where
# everything does is the wrong way round, and it is how a check stops being
# read.
#
# So, measured rather than assumed:
#
#   - yamllint walks `.`, so its exclusions live in .yamllint.yaml
#   - markdownlint-cli2 globs `**/*.md`, which DOES match dot directories, so
#     its exclusions live in .markdownlint-cli2.yaml
#   - shellcheck is handed scripts/*.sh below — one literal glob, cannot wander
#   - ruff walks `.` but honours .gitignore, which already names every one of
#     those trees. Verified with --show-files: the tracked .py files and
#     nothing else, with worktrees present
#   - packer is handed packer/ — one directory, cannot wander
#   - tofu is handed tofu/, the same way
#   - ansible-lint is handed ansible/ and told it is the project directory, so
#     it reads ansible/requirements.yml and installs the collections into
#     ansible/.ansible/ (gitignored, and excluded from yamllint above). Its
#     rule set is ansible/.ansible-lint's profile, not the version's default
#   - actionlint reads only <repo root>/.github/workflows. Verified with
#     -verbose: it lints 2 files with a worktree present, not 4
#   - editorconfig-checker is handed a `git ls-files` list built below, so it
#     sees tracked files and nothing else
#
# The last of those is the shape to copy when this comes up again: a list built
# from git cannot drift, whereas the first two carry a list of exclusions that
# has to be extended by hand every time a new ignored directory appears.
#
# --skips-file follows seed-validation-env.sh's contract: the caller owns the
# path and its lifetime. scripts/validate.sh passes an mktemp it removes on exit
# and adds the line count to its SKIPPED counter. Without it a skip here would
# be invisible there, and validate.sh would sign off with an unqualified "all
# checks passed" over a linter that never ran.

set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "${REPO_ROOT}" || exit 1

REQUIRE_ALL=0
SKIPS_FILE=""
while (($#)); do
  case "$1" in
    --require-all) REQUIRE_ALL=1 ;;
    --skips-file)  SKIPS_FILE="${2:?--skips-file needs a path}"; shift ;;
    *) printf 'unknown argument: %s\n' "$1" >&2; exit 2 ;;
  esac
  shift
done

FAILED=0
SKIPPED=0
pass() { printf '\033[0;32m  PASS\033[0m %s\n' "$*"; }
fail() { printf '\033[0;31m  FAIL\033[0m %s\n' "$*"; FAILED=1; }
skip() {
  printf '\033[0;33m  SKIP\033[0m %s\n' "$*"
  SKIPPED=$((SKIPPED + 1))
  [[ -n "${SKIPS_FILE}" ]] && printf '%s\n' "$*" >> "${SKIPS_FILE}"
  return 0
}

have() { command -v "$1" >/dev/null 2>&1; }
have_docker() { have docker && docker info >/dev/null 2>&1; }

# Resolve a linter to the command that runs it.
#
# yamllint, markdownlint-cli2 and shellcheck were all reported as "not
# installed" on a host where CI ran them successfully every push — because CI
# invokes them through `pipx run` and `npx --yes`, and validate.sh only probed
# for a globally installed binary and gave up. Two red CI runs were spent on
# defects a local run could have caught, and reported as passing instead.
#
# That fix preferred a binary on PATH and fell back to the runner CI used. #848
# turned the order round: the pinned runner comes first, everywhere. A binary
# on PATH is whatever version that machine happens to have, and on a GitHub
# runner the preinstalled yamllint and shellcheck would have outranked every
# pin. A PATH binary is now only the last resort of a local run, and is
# reported as unpinned when it is used. --require-all (CI) never takes it.
#
# Sets an array rather than returning a string to word-split: the docker forms
# embed ${REPO_ROOT}, and word-splitting a path is how that breaks on the first
# directory with a space in it.
RUNNER=()
UNPINNED=0
runner_for() {
  RUNNER=()
  UNPINNED=0
  pinned_runner_for "$1" && return 0
  if ! ((REQUIRE_ALL)) && have "$1"; then
    RUNNER=("$1")
    UNPINNED=1
    return 0
  fi
  return 1
}

# Every container below runs with --network none except tofu's. A linter reads
# files and prints, and on a fork PR the files are the author's. A linter with
# a parsing bug should then have nothing to reach. Triage of the linter images'
# CVEs had been saying "no network" while every runner had a network (#1001,
# #1013). tofu keeps one because `init` fetches the providers that `validate`
# needs, and the three tofu calls share this runner.
pinned_runner_for() {
  local tool="$1" img
  case "${tool}" in
    yamllint)
      # Same shape as ansible-lint below: the pin file is the only place the
      # version is written.
      if have pipx; then
        RUNNER=(pipx run --spec "$(grep -E '^yamllint==' scripts/requirements-lint.txt)" yamllint)
        return 0
      fi ;;
    markdownlint-cli2)
      # The entrypoint IS markdownlint-cli2, and with no arguments it reads its
      # globs from .markdownlint-cli2.yaml at the mount root. --user only so the
      # uid matches the checkout's owner; it writes nothing.
      if have_docker; then
        img="$(./scripts/image-for.sh markdownlint-cli2)"
        RUNNER=(docker run --rm --network none --user "$(id -u):$(id -g)" -v "${REPO_ROOT}:/workdir:ro" -w /workdir "${img}")
        return 0
      fi ;;
    shellcheck)
      # The entrypoint IS shellcheck. Handed relative paths from the caller,
      # which is why the working directory is the mount.
      if have_docker; then
        img="$(./scripts/image-for.sh shellcheck)"
        RUNNER=(docker run --rm --network none -v "${REPO_ROOT}:/mnt:ro" -w /mnt "${img}")
        return 0
      fi ;;
    ansible-lint)
      # The spec is read from the pin file rather than written here, so that
      # file stays the only place the version appears. requirements.txt goes in
      # as a constraint, so the syntax check runs on the ansible-core phoenix
      # runs, not on whichever one ansible-lint's own range would resolve to.
      if have pipx; then
        RUNNER=(pipx run --pip-args="-c ${REPO_ROOT}/ansible/requirements.txt"
                --spec "$(grep -E '^ansible-lint==' ansible/requirements-lint.txt)" ansible-lint)
        return 0
      fi ;;
    ruff)
      # Same route as ansible-lint, and the pin file is again the only place
      # the version appears (#853).
      if have pipx; then
        RUNNER=(pipx run --spec "$(grep -E '^ruff==' scripts/requirements-lint.txt)" ruff)
        return 0
      fi ;;
    actionlint)
      # No --user: this image's entrypoint IS actionlint and it already drops to
      # USER guest. It locates .github/workflows by walking up to the .git
      # directory, which is why the whole repository is mounted.
      if have_docker; then
        img="$(./scripts/image-for.sh actionlint)"
        RUNNER=(docker run --rm --network none -v "${REPO_ROOT}:/repo" -w /repo "${img}")
        return 0
      fi ;;
    zizmor)
      # Entrypoint is zizmor. --offline is passed by the caller below, so the
      # audits that would query GitHub (and want a token) are skipped and the
      # result does not depend on the network (#839).
      if have_docker; then
        img="$(./scripts/image-for.sh zizmor)"
        RUNNER=(docker run --rm --network none -v "${REPO_ROOT}:/repo:ro" -w /repo "${img}")
        return 0
      fi ;;
    editorconfig-checker)
      # The binary is named after the image reference because this image sets
      # Cmd and not Entrypoint: anything appended would replace the binary
      # rather than be passed to it.
      #
      # No --user, deliberately: running as another uid steps outside the path
      # upstream tests, and the checker writes nothing.
      #
      # This image installs git and marks /check a safe.directory so that the
      # checker can list files itself. That is no longer relied on — the list is
      # built on the host and passed in at the call site below, for the reason
      # recorded there. The mount is still the whole repository, because
      # .editorconfig, .editorconfig-checker.json and every file being checked
      # all have to be visible from inside it.
      if have_docker; then
        img="$(./scripts/image-for.sh editorconfig-checker)"
        RUNNER=(docker run --rm --network none -v "${REPO_ROOT}:/check" -w /check "${img}" editorconfig-checker)
        return 0
      fi ;;
    packer)
      # The entrypoint IS packer. --user so that `fmt` on a workstation could
      # never leave a root-owned file behind, although -check writes nothing.
      # Neither call below needs a plugin or a Proxmox: `fmt` is pure syntax,
      # and `validate -syntax-only` stops before plugins are loaded (ADR-0074).
      if have_docker; then
        img="$(./scripts/image-for.sh packer)"
        RUNNER=(docker run --rm --network none --user "$(id -u):$(id -g)" -e HOME=/tmp -v "${REPO_ROOT}:/repo" -w /repo "${img}")
        return 0
      fi ;;
    tofu)
      # Same shape as packer, but with a network: `init` below downloads the
      # providers. The entrypoint IS tofu, and --user so that the provider cache
      # `init` writes under tofu/.terraform/ (gitignored) is not root-owned on a
      # workstation. That cache outlives the run, so the linters that walk the
      # tree exclude it (.markdownlint-cli2.yaml, .yamllint.yaml).
      if have_docker; then
        img="$(./scripts/image-for.sh tofu)"
        RUNNER=(docker run --rm --user "$(id -u):$(id -g)" -e HOME=/tmp -e TF_IN_AUTOMATION=1 -v "${REPO_ROOT}:/repo" -w /repo "${img}")
        return 0
      fi ;;
  esac
  return 1
}

# "markdownlint-cli2 unavailable" with no hint sends the reader to a search
# engine. Say what to install.
hint_for() {
  case "$1" in
    yamllint|ansible-lint|ruff) printf 'install pipx' ;;
    *)                          printf 'needs a docker daemon' ;;
  esac
}

run_linter() {
  local tool="$1"; shift
  if ! runner_for "${tool}"; then
    if ((REQUIRE_ALL)); then
      fail "${tool} is unreachable, and this run requires every linter"
    else
      skip "${tool} unavailable ($(hint_for "${tool}"))"
    fi
    return 0
  fi
  # Output is captured and printed only on failure, rather than validate.sh's
  # run-quietly-then-re-run-verbosely pattern: two of these pull an image, and
  # paying that twice just to see the message is a poor trade.
  local out rc=0
  out="$("${RUNNER[@]}" "$@" 2>&1)" || rc=$?
  # LINT_FAIL_ON: a pattern that fails a run exiting 0. For a tool that only
  # warns about the one thing it was invoked to refuse; see tofu init below.
  if ((rc == 0)) && [[ -n "${LINT_FAIL_ON:-}" ]] && grep -qF -- "${LINT_FAIL_ON}" <<<"${out}"; then
    rc=1
  fi
  if ((rc == 0)); then
    if ((UNPINNED)); then
      pass "${tool} (a local binary, NOT the pinned version: $(hint_for "${tool}") to run that)"
    else
      pass "${tool}"
    fi
  else
    printf '%s\n' "${out}"
    fail "${tool}"
  fi
}

# ---------------------------------------------------------------------------
# The list. This is the only copy of it.
# ---------------------------------------------------------------------------
# --strict is the point of the exercise: without it yamllint exits 0 on a
# warning, which is how a clean `make validate` and a red CI run came to
# describe the same tree (#68).
run_linter yamllint --strict .
# No arguments: the globs live in .markdownlint-cli2.yaml.
run_linter markdownlint-cli2
run_linter shellcheck scripts/*.sh
# Two calls, one tool, as with packer: check is what the code does (#853), and
# format is how it is laid out (#938). Both read ruff.toml. --diff so a failure
# shows the fix, and `ruff format scripts/` applies it.
run_linter ruff check .
run_linter ruff format --check --diff .
# No arguments: actionlint finds the workflows from the repository root.
run_linter actionlint
run_linter zizmor --offline .github/workflows
# packer/ (ADR-0074). Two calls, one tool: fmt is the layout, validate is
# whether the HCL means anything. A real build is proved on phoenix, not here.
run_linter packer fmt -check -diff -recursive packer/
run_linter packer validate -syntax-only packer/
# tofu/ (ADR-0076). Unlike packer, validate needs the providers' schemas, so
# init runs first and downloads them: -backend=false so no state and no
# passphrase is involved, and -lockfile=readonly so a provider that does not
# match the committed .terraform.lock.hcl fails here instead of being quietly
# re-pinned. Whether the state is encrypted is not a lint question; that is
# scripts/check-tofu-state-encryption.sh --self-test, in self-tests.sh.
#
# -lockfile=readonly does not fail on a lock that disagrees with versions.tf: it
# prints "Changes to the provider selections were detected, but not saved" and
# exits 0. Dependabot's first provider bump (#961) left the lock that way, with
# constraints still on the old version and one zh: hash missing, and merged
# green. So that warning is a failure here. The fix for a red bump is
# `tofu -chdir=tofu init -upgrade -backend=false`, committed onto its PR.
run_linter tofu fmt -check -diff -recursive tofu/
LINT_FAIL_ON='Changes to the provider selections were detected' \
  run_linter tofu -chdir=tofu init -backend=false -input=false -lockfile=readonly
run_linter tofu -chdir=tofu validate
# ansible/ (ADR-0077). Includes ansible-playbook's own syntax check, which needs
# the pinned collections — hence --project-dir, which makes ansible-lint install
# ansible/requirements.yml before it looks. Proved against the guests on
# phoenix, not here: CI has no route to VLAN 30.
run_linter ansible-lint --project-dir ansible ansible/
# The file list is built here rather than left to the checker, which is the one
# linter in this list that decides for itself what to look at.
#
# Given no arguments it lists files with `git ls-files`, and inside the
# container that call fails silently. A git worktree's .git is a *file* holding
# a path to a gitdir outside the bind mount, so git answers "not a git
# repository" and the checker falls back to walking the filesystem — picking up
# the gitignored backups/, certificates/, .venv/ and .rendered/ trees that exist
# on a workstation and not in a CI checkout. `make validate` then failed on the
# deploy host, where `make up` had rendered the Alertmanager URLs, and passed in
# CI, which never renders. Green where nothing runs and red where everything
# does is the wrong way round, and a validate that is red for a reason the
# operator learns to dismiss is worse than one that does not run.
#
# Listing on the host fixes every caller at once: the host's git resolves a
# worktree, and the checker is handed the same tracked files whether it runs
# from the image or from a local binary. Nothing can substitute a different set
# without saying so. -z because a tracked path may contain a space, and the
# existence test because a tracked file deleted from the working tree makes the
# checker panic with a Go stack trace instead of reporting anything.
ec_files=()
if have git; then
  while IFS= read -r -d '' f; do
    [[ -e "${f}" ]] && ec_files+=("${f}")
  done < <(git ls-files -z)
fi
if ((${#ec_files[@]})); then
  run_linter editorconfig-checker "${ec_files[@]}"
else
  # Not a skip: a skip says the linter could not be reached. This says it could,
  # and there is no trustworthy answer to what it should check — which is the
  # state the checker used to paper over by walking the tree.
  fail "editorconfig-checker: git listed no files to check"
fi

# ---------------------------------------------------------------------------
# Self-check: the three callers must actually be callers
# ---------------------------------------------------------------------------
# One list only removes the drift for as long as nothing routes around it, and
# routing around it is easy and looks perfectly reasonable: adding
# `- name: mylinter` / `run: npx mylinter` to the lint job is precisely how the
# five-step list this replaced came to exist. So the lint job may contain a
# checkout and this script and nothing else, the Makefile target may contain
# this script and nothing else, and validate.sh must call it.
# seed-validation-env.sh asserts against compose.yaml for the same reason; this
# asserts against its callers.
#
# Three greps rather than a Python checker: this must never become the check
# that skips because python3 is missing, and asserting the *absence* of anything
# but one exact line needs no YAML semantics. If it ever needs them, promote it
# to scripts/check_lint_callers.py.
callers_ok=1

job="$(awk '/^  lint:/ {i=1; next} i && /^  [^[:space:]]/ {i=0} i' \
       .github/workflows/ci.yml)"
if [[ -z "${job}" ]]; then
  fail "no lint job in .github/workflows/ci.yml — CI is not running these linters"
  callers_ok=0
else
  stray="$(printf '%s\n' "${job}" \
    | grep -E '^[[:space:]]*(-[[:space:]]*)?(run|uses):' \
    | grep -vE '^[[:space:]]*-[[:space:]]*uses:[[:space:]]*actions/checkout@' \
    | grep -vE '^[[:space:]]*run:[[:space:]]*\./scripts/lint\.sh --require-all[[:space:]]*$' \
    || true)"
  if [[ -n "${stray}" ]]; then
    fail "ci.yml's lint job runs something this script does not know about:"
    printf '%s\n' "${stray}" | sed 's/^[[:space:]]*/        /'
    printf '        Add the linter to %s instead — CI has to call it.\n' \
      "${BASH_SOURCE[0]}"
    callers_ok=0
  fi
fi

recipe="$(awk '/^lint:/ {i=1; next} i && /^[^\t]/ {i=0} i' Makefile \
          | grep -v '^[[:space:]]*$' || true)"
if [[ "${recipe}" != $'\t./scripts/lint.sh' ]]; then
  fail "the Makefile's lint target does not delegate to this script:"
  printf '%s\n' "${recipe}" | sed 's/^/        /'
  callers_ok=0
fi

if ! grep -q 'scripts/lint\.sh' scripts/validate.sh; then
  fail "scripts/validate.sh does not call this script — make validate would lint nothing"
  callers_ok=0
fi

((callers_ok)) && pass "ci.yml, the Makefile and validate.sh all delegate here"

# ---------------------------------------------------------------------------
# When a caller passed --skips-file it owns the summary: validate.sh folds these
# skips into its own count and prints one, and two summaries is one too many.
if [[ -z "${SKIPS_FILE}" ]]; then
  printf '\n'
  if ((FAILED)); then
    printf '\033[0;31mlint failed\033[0m\n'
  elif ((SKIPPED)); then
    printf '\033[0;32mlint passed\033[0m \033[0;33m(%d skipped — NOT run)\033[0m\n' \
      "${SKIPPED}"
  else
    printf '\033[0;32mlint passed\033[0m\n'
  fi
fi

exit "${FAILED}"