Makefile

SHELL := /bin/bash
# Every recipe runs strict, rather than each one opting in by hand (#849). The
# by-hand version missed the case it most needed: `for sd in $(cmd)` carries on
# when cmd fails, because a failing substitution in a `for` list is ignored even
# under set -e. Under these flags a failing `x="$(cmd)"` does exit, so a recipe
# that loops over a command's output captures it first.
.SHELLFLAGS := -eu -o pipefail -c
.DEFAULT_GOAL := help

STACK       ?= observability
# The Python the checks run under: the first python3 ON PATH, in order, that
# can import yaml, else /usr/bin/python3. A bare `python3` used to win by PATH
# order alone, and on alexander (2026-10-08) that was mise's 3.14, ahead of the
# system python3 that has PyYAML. `make up STACK=lab` brought the containers up
# and then stopped at check_mounted_config.py, so the new rules were never
# reloaded. Every PATH entry is tried, so a yaml-less venv ahead of a working
# interpreter does not hide it. Override with `make ... PYTHON=/path/to/python3`.
FIND_PYTHON = IFS=:; for d in $$PATH; do [ -x "$$d/python3" ] && "$$d/python3" -c 'import yaml' >/dev/null 2>&1 && { echo "$$d/python3"; exit 0; }; done; [ -x /usr/bin/python3 ] && echo /usr/bin/python3 || echo python3
ifeq ($(origin PYTHON),undefined)
PYTHON := $(shell $(FIND_PYTHON))
endif
# Exported, so `up`'s sub-make inherits this choice rather than finding the shim
# below first on PATH and choosing that.
export PYTHON
# The recipes call $(PYTHON), but the scripts they run (validate.sh,
# check_loki_rules.sh and the rest) call `python3` themselves. So a one-line
# python3 that execs $(PYTHON) goes first on PATH for every recipe. It is a
# script, not a symlink: a symlinked venv python3 looks for pyvenv.cfg beside
# the link and loses the venv's packages. It lives under ~/.cache, keyed on the
# chosen path, so worktrees that choose differently do not collide.
ifneq ($(filter /%,$(PYTHON)),)
PYTHON_SHIM := $(HOME)/.cache/homelab-make/$(shell printf '%s' '$(PYTHON)' | cksum | cut -d' ' -f1)
$(shell mkdir -p '$(PYTHON_SHIM)' && printf '#!/bin/sh\nexec %s "$$@"\n' '$(PYTHON)' > '$(PYTHON_SHIM)/python3' && chmod 0755 '$(PYTHON_SHIM)/python3')
ifeq ($(findstring $(PYTHON_SHIM),$(PATH)),)
export PATH := $(PYTHON_SHIM):$(PATH)
endif
endif
# How far back check-loki-coverage looks, and the reason it is 24h rather than
# the 7d it started as (#335). The window is not a sensitivity dial in the
# direction it looks: BOTH sides of the check's comparison use it, so a host
# that goes quiet drops out of the denominator as well as the numerator and the
# check goes vacuous rather than wrong. What the window actually sets is
# DETECTION LAG. `reach` counts lines over the window, so a selector that went
# blind an hour ago still shows the host as reached until the last pre-breakage
# line falls out of range — a 7d window hides a new gap for up to a week.
#
# 24h is as short as the estate allows. Measured 2026-09-06, lines per host:
#
#   window   Saruman   morpheus   oracle   prometheus
#   1h            15      5,048      204          326
#   24h          186     89,340    5,379       45,998
#
# Saruman is the constraint at ~15 lines an hour and is comfortably present at
# 24h. The cost of the shorter window, stated rather than glossed: a host whose
# *subject* lines are rare — Saruman produced 2 SSH accepts in seven days — has
# none inside 24h, so a real gap there is reported as a latent WARN instead of a
# live FAIL. Verified both ways against the pre-#261 rules, where 24h still
# exits 1 on morpheus.
#
# Overridable per run (WINDOW=7d make check-loki-coverage) and inherited by
# homelab-loki-coverage.service, which sets no window of its own — one number,
# in one place.
WINDOW      ?= 24h
# The firewall, for the jobs that read it directly rather than over SNMP.
# `smart-state` reads its NVMe SMART log over SSH, because morpheus is FreeBSD
# with no node_exporter and no textfile directory of its own. Same names and
# same defaults as scripts/backup-firewall.sh, which has reached it this way
# since #92 — one pair of variables, overridable in /etc/default/homelab-timers.
FW_HOST     ?= 10.0.99.1
FW_USER     ?= root
STACK_DIR   := stacks/$(STACK)
COMPOSE     := docker compose -f $(STACK_DIR)/compose.yaml
SECRETS     := secrets/$(STACK).sops.yaml

.PHONY: help
help: ## Show this help
	@printf '\033[1mHomeLab\033[0m — make <target> [STACK=observability]\n\n'
	@grep -hE '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) \
		| awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[36m%-16s\033[0m %s\n", $$1, $$2}'
	@printf '\n'

# ---------------------------------------------------------------------------
# Lifecycle
# ---------------------------------------------------------------------------

.PHONY: up
up: ## Render config and start the stack
	@# One deploy of a stack at a time, from this checkout. On 2026-10-07 a
	@# hand `make up` and a `make converge` eighteen seconds apart recreated the
	@# same containers at once and one failed halfway on a container-name
	@# conflict. The recipe below, render included, runs under the stack's
	@# deploy lock; a second `make up` waits up to HOMELAB_DEPLOY_LOCK_WAIT
	@# seconds (900), naming the holder, then exits 75. converge.sh holds the
	@# same lock and its own `make up` runs straight through it
	@# (scripts/deploy-lock.sh says how).
	@./scripts/deploy-lock.sh $(STACK) -- $(MAKE) --no-print-directory _up STACK=$(STACK)

# The deploy itself. Not for calling directly: `make up` is the same recipe
# under the deploy lock, and the lock is the point.
.PHONY: _up
_up: render
	@# Home Assistant's HTTP settings live in its volume's .storage/http, not in
	@# configuration.yaml, and a fresh volume without them trusts no proxy, so
	@# every request through Caddy gets 400. Seeded before any container starts;
	@# an existing store is only checked, never overwritten (seed-ha-http.sh).
	@if [ "$(STACK)" = sensitive ]; then ./scripts/seed-ha-http.sh; fi
	@# Actual has no password setting: a fresh server belongs to whoever reaches
	@# it first. Claimed from SOPS with no network before its first start; a
	@# claimed one is only checked (seed-actual-password.sh, ADR-0062).
	@if [ "$(STACK)" = sensitive ]; then ./scripts/seed-actual-password.sh; fi
	@# A stack whose Alloy mounts /rootfs reads the host's textfile directory,
	@# and nothing else creates it on a host that runs its own Alloy: alexander,
	@# odin and eden all came up without it (ensure-textfile-dir.sh).
	./scripts/ensure-textfile-dir.sh $(STACK)
	$(COMPOSE) up -d --remove-orphans
	@# `up -d` recreates a container only when its *service definition* changes,
	@# so a freshly rendered snmp.yaml or an edited prometheus.yaml is invisible
	@# to it and the container keeps serving what it parsed at startup. Without
	@# this line `make up` reports success over a stale config — see
	@# scripts/reload-config.sh for the incident that produced it.
	@# BEFORE the reload, because the reload is not what is broken. A single-file
	@# bind mount is pinned to the inode and git replaces it, so a config-only
	@# commit leaves the container on the pre-merge bytes and POST /-/reload
	@# returns 200 having faithfully re-read them. That is #355, found when #166
	@# deployed clean and its three targets never appeared. --fix recreates only
	@# the services that actually diverged, then asserts the recreate worked.
	$(PYTHON) scripts/check_mounted_config.py --fix $(STACK)
	./scripts/reload-config.sh $(STACK)
	@# The deploy asserts on its own result. Everything above reports success
	@# from having ISSUED the commands — compose accepted the file, the reloads
	@# returned 200 — and none of it asks whether what came up is actually
	@# serving. snmp-exporter sat permanently unhealthy on a healthcheck pointed
	@# at a 404 for as long as it did precisely because nothing ever asked
	@# (#205). This is the only place the question can be asked, since CI has no
	@# stack to ask it of.
	@#
	@# It waits, so a slow start is not a failure: the per-service deadline is
	@# derived from that service's own start_period, retries, interval and
	@# timeout, and is the point Docker itself would have given up.
	$(PYTHON) scripts/check_container_health.py $(STACK)
	@# And that Alertmanager can still read the URLs it notifies through. The
	@# container's view, not this host's: in #214 the files were present here and
	@# absent inside the container, and every receiver reads from the same
	@# directory, so all four went dark together for ten and a half hours. The
	@# alert that fired about it could not be delivered, because it was the
	@# delivery path that was broken — so this has to be a check rather than an
	@# alert.
	$(PYTHON) scripts/check_alert_channels.py --files --live $(STACK)
	@# Then remove the images this deploy superseded (#1027). Only now, after
	@# every check above passed: if the new images were unhealthy, make stopped
	@# there and the old ones are still on disk to go back to. Without this
	@# they waited for the weekly prune, and a Wazuh re-pin on a Tuesday left
	@# odin's root at 99% for a week, short of the room the next pull needs.
	@# Only this stack's repositories, never an image a container uses or any
	@# stack pins; the script says why each of those is kept.
	$(PYTHON) scripts/prune_superseded_images.py $(STACK)
	@# The port is read back out of the rendered .env rather than expanded here.
	@# GRAFANA_PORT lives in $(STACK_DIR)/.env, which docker compose reads and make
	@# does not, so a bare $${GRAFANA_PORT:-3000} in a recipe yields 3000 whatever
	@# the operator actually set — a wrong URL that looks right. This line was
	@# previously inside a single-quoted printf format, so it did not expand at
	@# all and printed the literal text `$${GRAFANA_PORT:-3000}`: the same bug,
	@# just honest about it. Passed as a %s argument, not interpolated into the
	@# format, so a stray % in the value cannot be read as a format spec.
	@# tail -1 because compose takes the last of duplicate keys.
	@#
	@# Only for a stack that has a Grafana. The sensitive tier has none, and its
	@# own Caddy serves names from step-ca rather than the lab CA, so the same
	@# line there pointed at a port nothing listens on and a CA that signs
	@# nothing it serves — on every deploy, and in converge's journal hourly
	@# on trinity once it applied (#533). Read off the compose file's services
	@# rather than a list of stack names, so a stack that gains or drops a
	@# Grafana is right without this recipe changing.
	@if grep -qE '^  grafana:' $(STACK_DIR)/compose.yaml; then \
		port="$$(grep -E '^GRAFANA_PORT=' $(STACK_DIR)/.env 2>/dev/null | tail -1 | cut -d= -f2- || true)"; \
		printf '\n\033[0;32mup\033[0m — Grafana: https://localhost:%s\n' "$${port:-3000}"; \
		printf '   (self-signed by the lab CA — trust certificates/ca.pem, see docs/runbooks/generate-certificates.md)\n'; \
	else \
		printf '\n\033[0;32mup\033[0m — %s\n' "$(STACK)"; \
	fi
	@# Last, so it is only reached when every check above passed: the revision
	@# this deploy applied. converge.sh compares HEAD with it, because HEAD alone
	@# says where the checkout is, not what is running — on 2026-10-01 a
	@# checkout moved by hand read as "converged" for hours over stale rules.
	@./scripts/record-applied.sh $(STACK)

.PHONY: down
down: ## Stop the stack (volumes are preserved)
	$(COMPOSE) down --remove-orphans

.PHONY: restart
restart: down up ## Restart the stack

.PHONY: converge
converge: ## Fetch main, verify it, fast-forward and deploy (ARGS=--dry-run)
	@# What the hourly timer runs, and what a human runs to deploy on purpose
	@# without waiting for it. It ends in `make up` rather than replacing it, so
	@# there is exactly one deployment path and both callers exercise it.
	@#
	@# It refuses to run anywhere but the stack's deployment checkout —
	@# /home/robo/code/Gerrrt/HomeLab for observability, ~/code/Gerrrt/HomeLab
	@# on trinity for STACK=sensitive (#533) — for the reason `make up` cares
	@# about and `make deploy-agent` does not: render writes into the .rendered/
	@# of the tree it is run from, and no container mounts a worktree's copy.
	@# A bare `make converge` on trinity names robo's checkout and is refused,
	@# rather than deploying the wrong stack. ARGS=--dry-run says what it would do.
	./scripts/converge.sh --stack $(STACK) $(ARGS)

.PHONY: pull
pull: ## Pull the pinned images
	$(COMPOSE) pull

.PHONY: ps
ps: ## Show container status
	$(COMPOSE) ps

.PHONY: logs
logs: ## Tail logs (SERVICE=grafana to narrow)
	$(COMPOSE) logs -f --tail=100 $(SERVICE)

.PHONY: reload
reload: ## Hot-reload Prometheus, Alertmanager and snmp-exporter (no restart)
	@# `make up` runs this too. It stays a separate target because a config-only
	@# change — an SNMP community rotation, an Alertmanager route edit — needs
	@# only `make render && make reload`, with no compose round trip.
	./scripts/reload-config.sh $(STACK)

.PHONY: nuke
nuke: ## Stop the stack AND delete its volumes (destroys all metrics and logs)
	@printf '\033[0;33mThis deletes every metric and log stored by the stack.\033[0m\n'
	@read -p "Type 'nuke' to continue: " c; [ "$$c" = "nuke" ] || exit 1
	$(COMPOSE) down --volumes --remove-orphans

# ---------------------------------------------------------------------------
# Secrets
# ---------------------------------------------------------------------------

.PHONY: secrets-init
secrets-init: ## Generate an age keypair and create the encrypted secrets file
	./scripts/bootstrap.sh $(STACK)

.PHONY: secrets-edit
secrets-edit: ## Edit the encrypted secrets in $$EDITOR
	@# Not a bare `sops $(SECRETS)`. sops decrypts to a temp file and opens
	@# $$EDITOR on it, and a vim or neovim with `undofile` set then writes that
	@# buffer — the decrypted secrets — into a permanent undodir. sops shreds
	@# its own temp file on exit; nothing shreds the undo file. Found in the
	@# wild on the monitoring host: three of them holding the live SNMP
	@# community strings for pfSense, the APC NMC and iLO, mode 664, on an
	@# unencrypted disk. scripts/secrets-edit.sh silences the editor first.
	./scripts/secrets-edit.sh $(STACK)

.PHONY: secrets-add-recipient
secrets-add-recipient: ## Add a second age recipient and re-key (PUBKEY=age1...)
	@# Under Secrets and not Maintenance because it is part of setting the
	@# secrets up, but it is the one target here that rewrites a committed file
	@# — .sops.yaml and secrets/$(STACK).sops.yaml both change and must be
	@# committed together. ADR-0024 says why more than one recipient exists.
	@#
	@# PUBKEY rather than ARGS, for the reason secrets-verify-backup takes KEY:
	@# exactly one argument, required, and an empty ARGS would reach the script
	@# as no argument at all and print usage, which reads like a broken target.
	@#
	@# The PUBLIC half only. The private half of the key being added must never
	@# be generated on, copied to, or pasted into this host — that is the whole
	@# property the second recipient exists to have.
	@[[ -n "$(PUBKEY)" ]] || { \
		printf '\033[0;31merror:\033[0m PUBKEY is required\n' >&2; \
		printf 'Generate the keypair where it will LIVE, then bring back its public half:\n' >&2; \
		printf '  make secrets-add-recipient PUBKEY=age1...\n' >&2; \
		printf 'See docs/runbooks/back-up-the-age-key.md\n' >&2; \
		exit 2; \
	}
	./scripts/add-recipient.sh "$(PUBKEY)" $(STACK)

.PHONY: secrets-remove-recipient
secrets-remove-recipient: ## Remove an age recipient and re-key (PUBKEY=age1...)
	@# The other half of secrets-add-recipient, and the same two committed files.
	@# Refuses to remove this host's own key, to leave fewer than two
	@# recipients (ADR-0024), or to leave only recipients never proved. Removal
	@# does not reach git history: a key that may have been SEEN also means
	@# rotating what it opens (back-up-the-age-key.md).
	@[[ -n "$(PUBKEY)" ]] || { \
		printf '\033[0;31merror:\033[0m PUBKEY is required\n' >&2; \
		printf '  make secrets-remove-recipient PUBKEY=age1...\n' >&2; \
		printf 'See docs/runbooks/back-up-the-age-key.md\n' >&2; \
		exit 2; \
	}
	./scripts/remove-recipient.sh "$(PUBKEY)" $(STACK)

.PHONY: secrets-show
secrets-show: ## Print the decrypted secrets to stdout (careful)
	sops --decrypt $(SECRETS)

.PHONY: render
render: ## Decrypt secrets and render runtime config
	./scripts/render-config.sh $(STACK)

# ---------------------------------------------------------------------------
# Validation
# ---------------------------------------------------------------------------

.PHONY: validate
validate: ## Run every check CI runs
	./scripts/validate.sh

.PHONY: lint
lint: ## Lint YAML, Markdown, shell, workflows and EditorConfig
	./scripts/lint.sh

.PHONY: check-docs
check-docs: ## Verify the documents agree with the configs
	$(PYTHON) scripts/check_docs.py

.PHONY: check-dashboards
check-dashboards: ## Validate dashboard JSON and datasource references
	$(PYTHON) scripts/check_dashboards.py

.PHONY: check-rules
check-rules: ## Validate and unit-test Prometheus rules and config
	promtool check config $(STACK_DIR)/prometheus/prometheus.yaml
	promtool check rules $(STACK_DIR)/prometheus/rules/*.rules.yaml
	promtool test rules $(STACK_DIR)/prometheus/tests/*.test.yaml
	$(PYTHON) scripts/check_rule_tests.py

.PHONY: check-compose-health
check-compose-health: ## Verify health deps are satisfiable, probe the images (needs docker)
	@# --probe unconditionally, like check-rules above needs promtool. It execs
	@# each healthcheck's binary inside that service's pinned image, which is
	@# the only way to know an image has not moved to a distroless base under a
	@# Dependabot bump (#79). Without a docker daemon this fails and says to
	@# drop the flag, rather than handing back a green run it did not earn.
	@# `make validate` is the graceful path — it skips the probe and says so.
	$(PYTHON) scripts/check_compose_health.py --probe

.PHONY: check-hardened-boot
check-hardened-boot: ## Boot Home Assistant under its hardening and throw it away (needs docker)
	@# The application, not just its healthcheck binary: --probe above never
	@# starts it. Pulls the pinned image (~1.5 GB) on a cold cache. CI runs the
	@# same script with a proof cache, so a Dependabot bump cannot merge without
	@# a hardened boot (#534).
	./scripts/check_hardened_boot.sh --stack sensitive --service home-assistant

.PHONY: check-container-health
check-container-health: ## Ask the RUNNING stack whether its healthchecks pass (deploy-time)
	@# The other half of check-compose-health above, and deliberately not part
	@# of `make validate`: that one asks whether a healthcheck CAN run, from the
	@# config and the image, and gates pull requests. This one asks whether the
	@# endpoint it probes actually answers, which only a running daemon knows.
	@# `make up` runs it; this target is for asking again later without a
	@# redeploy. Add --no-wait for a snapshot of a stack that is already up.
	$(PYTHON) scripts/check_container_health.py $(STACK)

.PHONY: check-loki-rules
check-loki-rules: ## Validate Loki (LogQL) rules and panel queries, and behaviour-test the rules
	./scripts/check_loki_rules.sh
	$(PYTHON) scripts/test_loki_rules.py

.PHONY: check-syslog-senders
check-syslog-senders: ## Send spoofed and real syslog through a scratch Alloy and Loki; only named senders are stored
	./scripts/check_syslog_senders.sh

.PHONY: check-socket-proxy
check-socket-proxy: ## Boot the pinned socket proxy on its config; Alloy's paths answer, archive/export/top are refused
	./scripts/check_socket_proxy.sh

.PHONY: check-gosu-skip
check-gosu-skip: ## Boot each image whose gosu the scan skips as 999:999; gosu must never run
	./scripts/check_gosu_skip.sh

.PHONY: patch-state
patch-state: ## Collect this host's package patch state into the textfile dir
	@# Needs no root: apt-check runs unprivileged, /var/run/reboot-required is a
	@# world-readable flag, and the textfile directory is owned by the user the
	@# timers run as. --print writes to stdout instead, for looking without
	@# touching the collector.
	./scripts/collect-patch-state.sh

.PHONY: install-agent-collectors
install-agent-collectors: ## Put the textfile collectors on an agent host (needs sudo THERE)
	@# AGENT=user@host, one or more. #152 covered the monitoring host and only it;
	@# this covers a host that runs Alloy but has no checkout of this repository,
	@# which on 2026-09-06 meant `oracle` had a kernel update unbooted for two days
	@# that nothing in the estate could see (#360) and no SMART at all (#351).
	@#
	@# It ships every collector in the script's COLLECTORS table and checks each
	@# host's requirements per collector, so a host without apt still gets SMART.
	@# ARGS='--only smart-state' narrows it.
	@#
	@# Deliberately NOT part of deploy-agent.sh, which goes out of its way to need
	@# no privilege on the target. This step does — /usr/local/bin, /etc/systemd
	@# and the root-owned textfile directory — so it is run by hand, once per host,
	@# and prompts for a sudo password there. AGENT='...' ARGS=--check verifies an
	@# existing install and changes nothing.
	@test -n "$(AGENT)" || { echo "set AGENT=user@host (e.g. AGENT=atropos@10.0.99.30)"; exit 1; }
	./scripts/install-agent-collectors.sh $(ARGS) $(AGENT)
.PHONY: smart-state
smart-state: ## Collect SMART health from THIS host's disks (needs root) and render the recorded baselines
	@# The local half, and the only half that needs root — smartctl issues ATA and
	@# NVMe pass-through ioctls. morpheus is `smart-state-remote`, a separate job
	@# running as robo, because the key that reaches the firewall is robo's and
	@# root has none. One job used to do both and failed twice trying to be both
	@# users at once (#351).
	@#
	@# Two reasons to skip, and they are different findings. smartctl absent is a
	@# provisioning gap; smartctl present but run by a non-root user is a human
	@# running a root job by hand. Neither may read as success.
	@#
	@# The baseline table renders first and unconditionally: it needs neither
	@# smartctl nor root, and SmartDriveBadSectors reads it for EVERY host from
	@# this one file, so a count recorded in git reaches the rule on the next
	@# daily run of this job (#572). scripts/render-smart-baselines.sh says why
	@# it is rendered here and not by the collector on each host.
	./scripts/render-smart-baselines.sh
	@if ! command -v smartctl >/dev/null 2>&1; then \
		printf '\033[0;33m  SKIP\033[0m local disks: smartctl is not installed — %s\n' 'sudo apt install smartmontools'; \
	elif [ "$$(id -u)" != 0 ]; then \
		printf '\033[0;33m  SKIP\033[0m local disks: smartctl needs root for raw device access, running as %s\n' "$$(id -un)"; \
		printf '       homelab-smart-state.service runs as root; by hand use %s\n' 'sudo ./scripts/collect-smart-state.sh'; \
	else \
		./scripts/collect-smart-state.sh; \
	fi

.PHONY: gateway-state
gateway-state: ## Collect the firewall's view of its uplinks, DDNS record and GUI certificate (#353, #604, #857)
	@# Two measurements per family: what pfSense reports, and whether traffic of
	@# that family actually leaves the building. They disagreed on 2026-09-07 —
	@# WAN_DHCP6 reported 100% loss while v6 reached the internet through it in
	@# 11ms, because dpinger was pointed at a link-local address that does not
	@# answer echo. One measurement alone cannot tell those apart.
	@#
	@# And whether the remote path's dynamic DNS name still resolves, at a
	@# public resolver, to the WAN address (#604). Compared on the firewall:
	@# the name and the address are both withheld, and only the verdict comes back.
	./scripts/collect-gateway-state.sh --ssh $(FW_USER)@$(FW_HOST) --host morpheus
	@#
	@# And when the GUI's certificate expires (#857, ADR-0084), read off the
	@# handshake on the firewall itself: the GUI is blocked from this host on
	@# purpose, and the ssh it already allows is enough. Rides on this target so
	@# it needs no new timer; a failure fails the job like any other.
	./scripts/collect-cert-expiry.sh --ssh $(FW_USER)@$(FW_HOST) --host morpheus \
		--probe pfsense-ui=morpheus=127.0.0.1:443

.PHONY: silence-state
silence-state: ## Collect Alertmanager's silences as metrics (#575)
	@# One series per active silence, with the owning issue parsed from its
	@# comment. alertmanager_silences is a count per state and cannot say which
	@# alert a silence covers, when it ends or who owns it — and a silence that
	@# lapses returns its alert to a phone with nothing to say why. Loopback
	@# only: Alertmanager binds to 127.0.0.1 (ADR-0012), so this runs here.
	$(PYTHON) scripts/collect_silences.py

.PHONY: prune-images
prune-images: ## Remove Docker images no container uses (weekly on the monitoring host)
	@# The unit says why: nothing else removes superseded digests, and they
	@# filled `/` once (#136's cutover found it, 2026-09-29). Only images no
	@# container references, running or stopped. The before and after are
	@# printed so the journal records what each run reclaimed.
	@docker system df --format '{{.Type}}: {{.Size}} ({{.Reclaimable}} reclaimable)' | grep '^Images'
	docker image prune -a -f
	@docker system df --format '{{.Type}}: {{.Size}} ({{.Reclaimable}} reclaimable)' | grep '^Images'

.PHONY: pkg-state
pkg-state: ## Collect package state from morpheus over SSH (FreeBSD, runs as robo)
	@# morpheus is the one host `patch-state` cannot cover: it is FreeBSD, with
	@# neither apt-check nor apt-get, and it is where docs/security.md's accepted
	@# "vulnerability in pfSense itself" makes the gap matter most (#378).
	@#
	@# One command answers both questions. pfSense ships the system as pkg
	@# meta-packages, so `pkg version -vRL=` reports how many packages are behind
	@# AND whether a system upgrade is among them — no prose parsing of
	@# `pfSense-upgrade -c`, whose output has no format contract.
	./scripts/collect-pkg-state.sh --ssh $(FW_USER)@$(FW_HOST) --host morpheus

.PHONY: recipient-state
recipient-state: ## Record which age recipients can open the secrets, and when each was proved (#400)
	@# Reads the recipient list out of the encrypted file's own metadata and
	@# writes one series per recipient into the textfile dir, carrying every
	@# existing proof timestamp forward and setting none. Needs no key: the
	@# `sops:` block is plaintext. This is what SecretsKeyBackupUnproven reads;
	@# without it a host that proved its key before ADR-0024 has no series and
	@# the ninety-day nag is silent. secrets-verify-backup is what sets a proof.
	./scripts/key-recipients.sh --record --stack $(STACK)

.PHONY: ca-key-state
ca-key-state: ## Record the estate CA key's fingerprint and when a copy of it was last proved (#496)
	@# The CA-key twin of recipient-state. Reads the PUBLIC certificate, hashes
	@# its public key, and writes one series for that fingerprint into the
	@# textfile dir, carrying an existing proof forward and setting none. Never
	@# opens ca-key.pem. This is what CaKeyBackupUnproven reads; without it a
	@# host that has never proved the key has no series and the nag is silent,
	@# and a re-minted CA would inherit the old key's proof. certs-verify-backup
	@# is what sets a proof.
	./scripts/ca-key-state.sh --record

.PHONY: smart-state-remote
smart-state-remote: ## Collect SMART health from morpheus over SSH (runs as robo)
	@# morpheus is FreeBSD with no node_exporter and no textfile directory, but
	@# pfSense already ships smartctl and this host already has a key that reaches
	@# it — the same one backup-firewall.sh uses. So it is read over SSH and
	@# written into THIS host's textfile directory under host="morpheus"; those
	@# series carry instance="prometheus" as a result, which the collector says.
	@#
	@# Deliberately NOT root: this half needs a credential, not a privilege, and
	@# the credential belongs to robo.
	./scripts/collect-smart-state.sh --ssh $(FW_USER)@$(FW_HOST) --host morpheus --device nvme:/dev/nvme0

.PHONY: check-mounted-config
check-mounted-config: ## Verify each container runs the config the repo has (deploy-time)
	@# Read-only without --fix, so it is safe to ask at any time. `make up` runs
	@# it with --fix. Compares BYTES rather than inodes: an inode check would
	@# report a file rewritten with identical content as stale, which git and
	@# render-config.sh both do routinely.
	$(PYTHON) scripts/check_mounted_config.py $(STACK)

.PHONY: check-alert-channels
check-alert-channels: ## Verify Alertmanager can read every receiver URL (deploy-time)
	@# The static half runs in `make validate` and in CI, where it needs neither
	@# a secret nor a host. This adds --files and --live, which need both: the
	@# rendered files on this host, and what the running container can actually
	@# open. #214 is the difference between those two.
	$(PYTHON) scripts/check_alert_channels.py --files --live $(STACK)

.PHONY: check-loki-coverage
check-loki-coverage: ## Ask the LIVE Loki whether any rule is blind to a host (deploy-time)
	@# Not in `make validate`, and not in CI, for the reason check_loki_rules.sh
	@# gives in its own header: that script boots the pinned image against a
	@# throwaway config with no data and asks whether the rules PARSE. This asks
	@# whether they can SEE, which only a Loki holding real logs can answer, and
	@# CI has no log store at all (#327). WINDOW=24h to narrow it.
	$(PYTHON) scripts/check_loki_coverage.py $(STACK) --window $(WINDOW)

.PHONY: check-firewall
check-firewall: ## Diff docs/firewall-claims.yaml against the LIVE pfSense ruleset (deploy-time)
	@# Not in `make validate` and not in CI, and the reason is the same one that
	@# keeps backups/ gitignored: the ruleset carries rule bodies and the WAN
	@# address, docs/security.md says neither is published, and putting it in the
	@# repository so CI could read it would trade a documentation defect for a
	@# blueprint of the network (ADR-0026). So this asks the firewall directly,
	@# over SSH, and `make check-docs` covers the half that is pure text.
	@# Prose about pfctl was wrong three times on 2026-09-06 alone (#363).
	$(PYTHON) scripts/check_firewall_claims.py

.PHONY: check-ruleset
check-ruleset: ## Diff .github/rulesets/main.json against the LIVE ruleset on main
	@# The gate converge.sh relies on lives in a GitHub settings page (#833).
	@# Read anonymously; also run weekly by .github/workflows/digests.yml.
	./scripts/check-ruleset.sh

.PHONY: apply-ruleset
apply-ruleset: ## Make the ruleset on main match .github/rulesets/main.json (needs gh as admin)
	./scripts/check-ruleset.sh --apply

.PHONY: check-dashboard-roundtrip
check-dashboard-roundtrip: ## Boot the pinned Grafana and verify the dashboards round-trip
	@# Under Validation and not Maintenance, unlike `dashboards-export` below,
	@# because it needs neither a secret nor the live stack: it boots a
	@# throwaway Grafana from the pinned image, provisions the committed JSON
	@# into it and reads it back. Same shape and the same reasoning as
	@# `check-loki-rules` — the only thing that genuinely understands the format
	@# is the thing that will serve it.
	@#
	@# It also asserts that Grafana still ACCEPTS a save to a provisioned
	@# dashboard. That is not incidental: while allowUiUpdates is false the
	@# export is a silent no-op, so this is the check standing between #100 and
	@# a command that succeeds without doing anything.
	./scripts/check_dashboard_roundtrip.sh

.PHONY: check-image-pins
check-image-pins: ## Verify every docker image comes from compose.yaml
	$(PYTHON) scripts/check_image_pins.py

.PHONY: check-timers
check-timers: ## Verify the schedule and its staleness thresholds agree
	@# Under Validation rather than Maintenance, unlike `install-timers` below:
	@# --check reads files, shells out to `systemd-analyze calendar`, and touches
	@# neither the host nor a secret. It is what stops the cadence in a .timer
	@# and the threshold in scripts/install-timers.sh drifting apart — the same
	@# two-copies-of-one-fact problem the amtool route assertions exist for (#68).
	./scripts/install-timers.sh --check

.PHONY: pin-digests
pin-digests: ## Re-resolve image digests in every stack's compose.yaml (--write applies)
	@# Every stack, not just the estate's. pin-digests.sh takes one compose file
	@# and rewrites it in place, which is the right shape for the work it does —
	@# so the loop lives here rather than inside it, driven by the same
	@# scripts/stacks.sh that validate.sh and ci.yml read. Left single-stack,
	@# `stacks/lab`'s digests would be re-resolved by nothing and verified by
	@# nothing, which is the #263 defect in the one place it costs a supply-chain
	@# guarantee rather than a test.
	@paths="$$(./scripts/stacks.sh --paths)"; for sd in $$paths; do \
		printf '\033[0;34m--\033[0m %s\n' "$$sd"; \
		COMPOSE_FILE="$$sd/compose.yaml" ./scripts/pin-digests.sh --write; \
	done

.PHONY: check-digests
check-digests: ## Verify pinned digests still match the registry
	@# Every stack is checked before this fails. Under .SHELLFLAGS' -e the loop
	@# used to stop at the first stack that drifted, so one stale pin hid every
	@# stack after it in stacks.sh order. This is also what digests.yml runs
	@# weekly, so the CI check and the local one cannot cover different stacks.
	@paths="$$(./scripts/stacks.sh --paths)"; failed=""; for sd in $$paths; do \
		printf '\033[0;34m--\033[0m %s\n' "$$sd"; \
		COMPOSE_FILE="$$sd/compose.yaml" ./scripts/pin-digests.sh || failed="$$failed $${sd#stacks/}"; \
	done; \
	if [[ -n "$$failed" ]]; then \
		printf '\n\033[0;31mdigest drift or resolve failure in:\033[0m%s\n' "$$failed"; exit 1; \
	fi

.PHONY: scan-images
scan-images: ## Scan every pinned image for fixable HIGH/CRITICAL CVEs (reports in .scan/)
	@# What .github/workflows/cve-scan.yml runs weekly, minus the issues: the
	@# summary goes to the terminal. `cve_report.py --dir .scan --sync-issues
	@# --dry-run` shows what the weekly run would open and close.
	@# The summary prints even when a scan failed, as the workflow's does: it is
	@# what says which image failed. The scan's failure is still the exit status.
	@rc=0; ./scripts/scan-images.sh --out .scan $(ARGS) || rc=$$?; \
	if [[ -f .scan/images.tsv ]]; then $(PYTHON) scripts/cve_report.py --dir .scan; fi; \
	exit $$rc

.PHONY: scan
scan: ## Scan the working tree and history for secrets
	gitleaks detect --no-banner --redact -c .gitleaks.toml
	gitleaks detect --no-banner --redact -c .gitleaks.toml --log-opts="--all"

# ---------------------------------------------------------------------------
# Maintenance
# ---------------------------------------------------------------------------

.PHONY: snmp-mibs
snmp-mibs: ## Download the vendor MIBs snmp-generate needs (gitignored)
	./scripts/snmp-mibs.sh $(ARGS)

.PHONY: snmp-generate
snmp-generate: ## Regenerate snmp.yaml from generator.yaml (needs make snmp-mibs)
	STACK=$(STACK) ./scripts/snmp-generate.sh

.PHONY: check-versions
check-versions: ## Check the documented OS versions against what the hosts report
	@# Under Maintenance rather than Validation, for the same reason snmp-verify
	@# is: it reads the running system. Everything under Validation is offline and
	@# safe for CI, and this one needs Prometheus — so `make validate` must not
	@# grow it, or CI starts failing on a host it cannot reach.
	@#
	@# This is the half check_docs.py structurally cannot do. That script asserts
	@# hardware.md and network.md agree with each other, which passed for a whole
	@# pfSense release while both said FreeBSD 15 and the box ran 16 (#292).
	./scripts/check_versions.py $(ARGS)

.PHONY: snmp-verify
snmp-verify: ## Check each SNMP device answers to its community (ARGS=--old)
	@# Deliberately under Maintenance, not Validation: everything under
	@# Validation is offline and safe for CI, whereas this needs the age key and
	@# sends packets to production devices. Keeping it here stops anyone folding
	@# it into `make validate`.
	./scripts/snmp-verify.sh $(ARGS)

.PHONY: qga-resync
qga-resync: ## Resync a guest agent that stopped answering, on Saruman as root (VMID="140 190")
	@# Maintenance: it talks to QEMU's sockets on the hypervisor, so it is run
	@# there by hand when GuestAgentSilent fires or `qm agent <id> ping` says the
	@# agent is not running. It resets qemu-ga's parser and proves the channel
	@# with a ping; it restarts nothing. The header says why that is enough.
	@test -n "$(VMID)" || { echo 'usage: make qga-resync VMID="<vmid> ..."' >&2; exit 2; }
	./scripts/qga-resync.py $(VMID)

.PHONY: snmp-walk
snmp-walk: ## Walk one OID subtree on one SNMP device, exporter-shaped (ARGS="--device neo <oid>")
	@# Maintenance for the same reason as snmp-verify: needs the age key and
	@# sends packets to a production device. It exists so a new column is read
	@# off the switch before it goes into generator.yaml — see the mokerlink
	@# module's comment for why that order matters on this hardware.
	./scripts/snmp-walk.sh $(ARGS)

.PHONY: secrets-verify-backup
secrets-verify-backup: ## Check a backup age key decrypts the secrets (KEY=/path/to/keys.txt)
	@# Under Maintenance rather than Validation for the same reason as
	@# snmp-verify: everything under Validation is offline and safe for CI,
	@# and this needs a private key on disk. It must never end up inside
	@# `make validate`, where it would either always skip or ask CI for a key.
	@#
	@# KEY rather than ARGS because there is exactly one argument and it is
	@# required — an empty ARGS would reach the script as no argument at all
	@# and print usage, which reads like the target is broken.
	@#
	@# Guarded here rather than left to the script so that a bare
	@# `make secrets-verify-backup` does not reach run-scheduled.sh and get
	@# recorded as a FAILED verification. A forgotten argument is a typo, not
	@# evidence about the key backup, and it must not fire an alert.
	@[[ -n "$(KEY)" ]] || { \
		printf '\033[0;31merror:\033[0m KEY is required\n' >&2; \
		printf 'Mount the offline copy, then:  make secrets-verify-backup KEY=/path/to/keys.txt\n' >&2; \
		printf 'See docs/runbooks/back-up-the-age-key.md\n' >&2; \
		exit 2; \
	}
	@# Wrapped, even though a human runs it, and that is the whole point of #77.
	@# This is the one job that cannot be put on a timer — verify-key-backup.sh
	@# refuses the live key by device:inode precisely so that what gets tested is
	@# a copy on removable media, and no timer can mount that. So the schedule is
	@# enforced from the other end: a successful run records its timestamp, and
	@# SecretsKeyBackupUnproven fires when that proof passes ninety days old.
	@# Nagging is not as good as running it, but it beats remembering.
	./scripts/run-scheduled.sh --job verify-key-backup --lock keys \
		-- ./scripts/verify-key-backup.sh "$(KEY)" $(STACK)

.PHONY: certs-verify-backup
certs-verify-backup: ## Check an offline copy of the CA key is this CA's key (KEY=/path/to/ca-key.pem)
	@# Maintenance, not Validation, for the reason secrets-verify-backup gives:
	@# it needs a private key on a mounted medium and must never reach CI.
	@# KEY rather than ARGS, and guarded here, for the same reasons too — a bare
	@# `make certs-verify-backup` is a typo and must not reach run-scheduled.sh
	@# to be recorded as a failed verification.
	@[[ -n "$(KEY)" ]] || { \
		printf '\033[0;31merror:\033[0m KEY is required\n' >&2; \
		printf 'Mount the offline copy, then:  make certs-verify-backup KEY=/path/to/ca-key.pem\n' >&2; \
		printf 'See docs/runbooks/back-up-the-ca-key.md\n' >&2; \
		exit 2; \
	}
	@# Wrapped so the run is recorded, though a human runs it: verify-ca-key-backup.sh
	@# refuses the live key by device:inode so that a copy is what gets tested,
	@# and no timer can mount one. The deadline is enforced from the other end —
	@# CaKeyBackupUnproven fires when the proof passes ninety days old.
	./scripts/run-scheduled.sh --job verify-ca-key-backup --lock keys \
		-- ./scripts/verify-ca-key-backup.sh "$(KEY)"

.PHONY: certs
certs: ## Create the internal CA / issue a leaf (ARGS="--host x.matrix.elysium --ip 10.0.0.1")
	@# certificates/ is gitignored, so a clean clone has neither the CA nor the
	@# leaf and this is a required deployment step, not a someday one: Grafana
	@# serves https from the leaf and Prometheus verifies it with the CA. It read
	@# as optional for as long as this comment claimed nothing terminated TLS,
	@# which is how #69 happened — `make up` bind-mounted the absent files and
	@# Docker silently created directories in their place. render-config.sh now
	@# refuses to render until they exist.
	./scripts/gen-certs.sh $(ARGS)

.PHONY: tier-ca
tier-ca: ## The sensitive tier's own CA — mint it here, install it on trinity (ARGS="--mint" | "--install FILE" | "--list")
	@# Not `certs`: that is the estate's CA and this is the tier's, and they are
	@# deliberately two — the estate's root carries pathlen:0, so nothing beneath
	@# it may be a CA (ADR-0037). The step binary runs from the pinned image, the
	@# way promtool and caddy do; nothing is installed on the host.
	./scripts/tier-ca.sh $(ARGS)

.PHONY: gen-secret
gen-secret: ## Generate a random secret (ARGS=--snmp for one per SNMP device)
	./scripts/gen-secret.sh $(ARGS)

.PHONY: hash-password
hash-password: ## Bcrypt a password for AdGuard's admin or ntfy's users (prompted, never an argument)
	@# AdGuard Home keeps its admin password as a bcrypt hash, and the hash is
	@# what secrets/sensitive.sops.yaml holds as ADGUARD_ADMIN_PASSWORD_HASH —
	@# so the plaintext is typed once, here, and lives in the password manager.
	@# ntfy's two NTFY_*_PASSWORD_HASH keys are made the same way: ntfy accepts
	@# this cost-14 hash as it is (checked against the pinned image, #136).
	@# `caddy hash-password` prompts without echo on a terminal and reads one
	@# line from stdin without one, so `-t` is passed only when there is a tty
	@# to pass; either way the password is never an argument, never in shell
	@# history and never in `ps`. Caddy's image rather than AdGuard's because
	@# AdGuard's ships no hashing tool and Caddy's is pinned in the same stack.
	@# The output starts `$$2a$$` — paste it whole into `make secrets-edit
	@# STACK=sensitive`; render-config.sh escapes it for compose's .env.
	@tty=""; [ -t 0 ] && tty="-t"; \
	img="$$(COMPOSE_FILE=stacks/sensitive/compose.yaml ./scripts/image-for.sh caddy)"; \
	docker run --rm -i $$tty "$$img" caddy hash-password

.PHONY: screenshots
screenshots: ## Render the dashboards to docs/images/ (stack must be up)
	@# Under Maintenance, not Validation, for the same reason as snmp-verify and
	@# secrets-verify-backup: it needs the decrypted Grafana password and a
	@# running stack, so it must never be reachable from `make validate`, where
	@# it would either always skip or ask CI for a secret.
	@#
	@# It starts the `capture` profile's renderer, shoots six PNGs and stops it
	@# again. Review every image before committing — docs/images/README.md says
	@# what to look for.
	./scripts/capture-screenshots.sh $(STACK)

.PHONY: dashboards-export
dashboards-export: ## Pull the dashboards out of the running Grafana into git (ARGS=--check)
	@# Under Maintenance, not Validation, for the same reason as `screenshots`:
	@# it needs the decrypted Grafana password and a running stack, so it must
	@# never be reachable from `make validate`, where it would either always skip
	@# or ask CI for a secret. `make check-dashboard-roundtrip` is the half that
	@# belongs there.
	@#
	@# The loop this replaces was Dashboard settings → JSON Model, select all,
	@# copy, paste over the file — manual, and therefore skipped under pressure
	@# (#100). It is now: edit in the UI, run this, read `git diff`.
	@#
	@# ARGS=--check writes nothing and exits non-zero when the running Grafana
	@# holds an edit that git does not. That is the half that makes
	@# allowUiUpdates: true safe rather than merely convenient, and it is what
	@# the `dashboards-drift` timer runs — see systemd/homelab-dashboards-drift.
	./scripts/export-dashboards.sh $(ARGS) $(STACK)

.PHONY: backup-firewall
backup-firewall: ## Pull morpheus's pfSense config, encrypt it to ./backups/, copy it to oracle
	@# The single largest unmitigated failure in the estate is morpheus dying
	@# with no config export. Output is gitignored and never committed — see
	@# the header of scripts/backup-firewall.sh for why.
	@#
	@# The copy to oracle is part of this target, not a second one: a run whose
	@# copy fails exits non-zero even though the local file was written, so the
	@# nightly timer's metric says "stopped leaving this host" rather than
	@# "fine" (#92). ARGS=--local-only skips it, for a bench.
	@#
	@# Retention is part of the same run: FW_KEEP exports survive on each side
	@# and the rest are removed, so a nightly job cannot grow without bound.
	@# ARGS=--list is its dry run and ARGS=--prune applies it on its own. Note
	@# FW_KEEP and not KEEP — every unit shares /etc/default/homelab-timers and
	@# backup-volumes.sh already owns KEEP there.
	./scripts/backup-firewall.sh $(ARGS)

.PHONY: backup
backup: ## Quiesce the stack, archive its volumes to ./backups/, verify, copy to oracle
	@# Thin on purpose. This target used to BE the implementation, and every
	@# defect in #64 followed from that: one fixed output filename that tar
	@# truncated at open, so the only way to lose a backup was to take one; a
	@# hardcoded volume list that had silently skipped alloy-data since Alloy
	@# was added; an unpinned `alpine`; a hot copy of an open TSDB; and no
	@# verification beyond tar's exit status.
	@#
	@# The volume list and the services to stop are now derived from
	@# compose.yaml, so a sixth volume cannot be forgotten. STACK goes in the
	@# environment rather than positionally: the script's arguments are flags.
	@#
	@# The copy to oracle is part of this target, not a second one, for the
	@# reasons backup-firewall gives above: a run whose copy fails exits
	@# non-zero after the stack is back up, so the weekly timer's metric says
	@# "stopped leaving this host" rather than "fine" (#535). ARGS=--local-only
	@# skips it; ARGS=--copy-only copies every set oracle lacks without
	@# stopping the stack, which is how the existing sets were seeded. KEEP
	@# bounds both sides; ARGS=--list shows both.
	STACK=$(STACK) ./scripts/backup-volumes.sh $(ARGS)

.PHONY: backup-nas
backup-nas: ## Pull the media tier's state off smaug from its newest ZFS snapshot, encrypt and verify
	@# The one backup that leaves this host to FETCH rather than to deliver.
	@# smaug cannot run backup-volumes.sh — no age, no checkout, no key, and
	@# ADR-0016 forbids it initiating anything upward — so this host reads a
	@# snapshot of erebor/apps over the 99 → 40:22 pass and encrypts what
	@# arrives here (ADR-0045) — Jellyfin's, Audiobookshelf's and Navidrome's
	@# state, one archive each, from one snapshot (ADR-0050). Nothing on smaug is stopped:
	@# the snapshot is the quiesce. NAS_KEEP and not KEEP, for the reason backup-firewall gives
	@# for FW_KEEP. Sets land in backups/nas/, apart from the volume sets, and
	@# `make verify-backups` reads both. The copy to oracle is a step of this
	@# target too, by the helpers `backup` gained under #535 — same far-side
	@# rules, its own directory there — so ARGS=--local-only and
	@# ARGS=--copy-only mean here what they mean above.
	./scripts/backup-nas.sh $(ARGS)

.PHONY: backup-wiki
backup-wiki: ## Pull a pg_dump of the wiki's database off oracle, encrypt and verify
	@# The wiki's Postgres, which nothing backed up before #251. Pulled for
	@# backup-nas's reason: oracle holds no age identity (ADR-0015), so this
	@# host asks it for a dump over ssh and encrypts what arrives. A dump and
	@# not a stopped volume, the first in the repository (ADR-0065): nothing on
	@# oracle stops, and a set restores into a newer Postgres. Sets land in
	@# backups/wiki/ and stay here — the set leaving oracle IS the off-host
	@# copy. WIKI_KEEP and not KEEP, for the reason backup-firewall gives for
	@# FW_KEEP. ARGS=--prove restores the newest set into a scratch Postgres.
	./scripts/backup-wiki.sh $(ARGS)

.PHONY: backup-library
backup-library: ## On trinity: archive Immich's library off its USB disk, encrypt, verify, copy to oracle
	@# The photographs, which no volume set can hold: they are a bind mount
	@# on trinity's USB disk, and `make backup` archives named volumes. Nothing
	@# is stopped — originals are written once, and Immich's own 02:00 dump
	@# rides in the same archive — so this runs nightly beside the volume
	@# sets rather than inside them. LIB_KEEP and not KEEP, for the reason
	@# backup-firewall gives for FW_KEEP; two, because every set is the whole
	@# library and oracle's root volume is the limit. ARGS=--prove hashes
	@# every original in the newest set against immich-db's checksums.
	@# Off-host, NOT off-estate: ADR-0064 is the interim, #455 the answer.
	./scripts/backup-library.sh $(ARGS)

.PHONY: verify-backups
verify-backups: ## Re-verify every retained set of every kind: volume, NAS and wiki sets; on trinity (STACK=sensitive) volume and library sets
	@# What homelab-verify-backups.timer runs nightly. Two directories, one
	@# job: backups/volumes/ is verified against the stack's derived volume
	@# list and backups/nas/ each set against its own MANIFEST, and a media set in the volume
	@# directory would fail the first — which is why they are apart, and why
	@# one target walks both rather than a second unit doing the second half.
	@# Both halves run even when the first fails, so one morning's journal
	@# says which sets are bad rather than stopping at the first directory.
	@# backups/wiki/ (#251) is walked once it exists: before the first
	@# backup-wiki run there is nothing to verify, and that run not happening
	@# is ScheduledJobNeverRan's to report, not this job's.
	@#
	@# STACK=sensitive is trinity's half, and what
	@# homelab-verify-backups-sensitive.timer runs (#856): the tier's volume
	@# sets and Immich's library sets (ADR-0064), and no NAS or wiki set,
	@# which trinity never holds. Not backups/paperless-documents/ either:
	@# household-copy re-makes that set on every visit and nothing else reads
	@# it (#942, carry-the-household-copy.md §4). Both scripts' --verify-only also has oracle
	@# hash every archive it holds against the MANIFEST here, so oracle's
	@# copies are re-verified by the same run, and nothing is decrypted there.
	@rc=0; \
	STACK=$(STACK) ./scripts/backup-volumes.sh --verify-only --all || rc=1; \
	if [ "$(STACK)" = sensitive ]; then \
		./scripts/backup-library.sh --verify-only --all || rc=1; \
	else \
		./scripts/backup-nas.sh --verify-only --all || rc=1; \
		if [ -d backups/wiki ]; then ./scripts/backup-wiki.sh --verify-only --all || rc=1; fi; \
	fi; \
	exit $$rc

.PHONY: backup-offsite
backup-offsite: ## Copy the newest set of each kind to the offline medium and prove it (DEST=/path/to/the/medium)
	@# The third human job with a deadline and no timer, after
	@# secrets-verify-backup and certs-verify-backup, and for the same reason:
	@# the destination is the medium that holds the second age recipient
	@# (ADR-0048), which is off-estate by construction and visits this host
	@# every ninety days to prove that key. No timer can mount it. What a
	@# timer CAN do is notice that nobody has: a successful run records
	@# offsite-copy, and OffsiteCopyStale fires when that passes ninety days.
	@#
	@# DEST rather than ARGS for the reason secrets-verify-backup takes KEY:
	@# exactly one required argument, guarded here so that a bare
	@# `make backup-offsite` never reaches run-scheduled.sh and gets recorded
	@# as a FAILED copy. A forgotten argument is a typo, not evidence.
	@[[ -n "$(DEST)" ]] || { \
		printf '\033[0;31merror:\033[0m DEST is required\n' >&2; \
		printf 'Mount the medium, then:  make backup-offsite DEST=/path/to/the/medium\n' >&2; \
		printf 'See docs/runbooks/copy-the-backups-offsite.md\n' >&2; \
		exit 2; \
	}
	@# Wrapped only for the copy of record. ARGS=--list, --verify-only and
	@# --prune inspect or tidy the medium and run unwrapped on purpose: looking
	@# at a ninety-day-old copy is not the same as refreshing it, and must not
	@# reset the deadline. The `backups` lock is shared with the backup jobs so
	@# a set is never copied while its own retention is removing it.
	@if [[ -n "$(ARGS)" ]]; then \
		STACK=$(STACK) ./scripts/backup-offsite.sh "$(DEST)" $(ARGS); \
	else \
		STACK=$(STACK) ./scripts/run-scheduled.sh --job offsite-copy --lock backups \
			-- ./scripts/backup-offsite.sh "$(DEST)"; \
	fi

.PHONY: household-copy
household-copy: ## On trinity: carry the household's photographs and documents to the holder's drive (DEST=/path/to/the/drive)
	@# ADR-0073: the copy ADR-0023 requires, on the drive the household holder
	@# keeps at their own address. backup-offsite's shape for the same reason:
	@# a deadline and no timer, recorded as household-copy only for a copy of
	@# record, with HouseholdCopyStale at ninety days. Two guards before the
	@# wrapper, so that neither a forgotten DEST nor a recipients file with no
	@# household key is recorded as a FAILED copy: the first is a typo and the
	@# second is a decision nobody has taken yet. ARGS=--rehearse writes the
	@# same copy without a holder and records nothing; --list, --verify-only and
	@# --prune look at or tidy the drive without resetting the deadline.
	@[[ -n "$(DEST)" ]] || { \
		printf '\033[0;31merror:\033[0m DEST is required\n' >&2; \
		printf 'Mount the drive, then:  make household-copy DEST=/path/to/the/drive\n' >&2; \
		printf 'See docs/runbooks/carry-the-household-copy.md\n' >&2; \
		exit 2; \
	}
	@if [[ -n "$(ARGS)" ]]; then \
		./scripts/carry-household-copy.sh "$(DEST)" $(ARGS); \
	else \
		./scripts/household-recipients.sh --has-holder || { \
			printf '\033[0;31merror:\033[0m stacks/sensitive/household.recipients has no household key yet\n' >&2; \
			printf 'The copy of record needs the holder (ADR-0073). Rehearse instead:  make household-copy DEST=$(DEST) ARGS=--rehearse\n' >&2; \
			exit 2; \
		}; \
		./scripts/run-scheduled.sh --job household-copy --lock backups \
			-- ./scripts/carry-household-copy.sh "$(DEST)"; \
	fi

.PHONY: household-proof
household-proof: ## On trinity: record the holder's own proof, from the code they read out (CODE=1234-5678-9012)
	@# ADR-0023's second condition, made checkable: the holder opened
	@# PROOF/proof.txt.age on their own device, with their own key and without
	@# the operator, and read the code back. Checked unwrapped first, so a
	@# misheard digit is a retry and not a FAILED household-proof; recorded
	@# only on a match. Due yearly, beside ADR-0011's drill.
	@[[ -n "$(CODE)" ]] || { \
		printf '\033[0;31merror:\033[0m CODE is required: the twelve digits the holder read from PROOF/proof.txt.age\n' >&2; \
		printf 'See docs/runbooks/open-the-household-copy.md\n' >&2; \
		exit 2; \
	}
	@./scripts/carry-household-copy.sh --check-code "$(CODE)"
	@./scripts/run-scheduled.sh --job household-proof -- ./scripts/carry-household-copy.sh --prove-code "$(CODE)"

.PHONY: restore
restore: ## Restore the stack's volumes from a backup set (ARGS="--from <stamp>")
	@# Deliberately a separate script from `backup`. One script that both writes
	@# archives and overwrites live volumes is one mistyped flag from an outage,
	@# and scripts/backup-firewall.sh — the model for both — never writes to the
	@# thing it backs up and never deletes outside its own retention window
	@# (#92). The volume inventory is not duplicated: restore-volumes.sh
	@# reads `backup-volumes.sh --inventory`, the way every SNMP tool reads
	@# scripts/snmp-targets.sh.
	STACK=$(STACK) ./scripts/restore-volumes.sh $(ARGS)

.PHONY: install-timers
install-timers: ## Install and enable the systemd timers on this host (needs sudo)
	@# Under Maintenance for the same reason as snmp-verify and
	@# secrets-verify-backup: it changes the host. It must never be reachable
	@# from `make validate` — `make check-timers` is the half that is.
	@#
	@# The units carry absolute paths, so the script refuses to install from
	@# anywhere but the deployment checkout. Installing from a worktree would
	@# point every timer at a directory that gets deleted, and the symptom would
	@# be jobs that silently never run — the exact condition this exists to make
	@# visible.
	@#
	@# PROFILE=sensitive installs trinity's schedule instead (#404 step 9). sudo
	@# keeps SUDO_USER, which is who those units are rendered to run as.
	sudo ./scripts/install-timers.sh --install $(if $(PROFILE),--profile $(PROFILE))

.PHONY: deploy-agent
deploy-agent: ## Deploy or redeploy the Alloy agent on a host (ARGS="[--runtime native] user@host")
	@# Under Maintenance because it changes a host — a remote one — and must
	@# never be reachable from `make validate`. The image and the config come
	@# from this checkout, so run it from the checkout whose state you want the
	@# host to have; the script ships files over ssh and needs no render, so a
	@# worktree is fine here, unlike `make up`.
	./scripts/deploy-agent.sh $(ARGS)

.PHONY: purge-history-dry-run
purge-history-dry-run: ## Preview the git-history secret purge (safe)
	./scripts/purge-history.sh --dry-run