From f51e6094d79f7ee23f6227dbee290f6df48447ce Mon Sep 17 00:00:00 2001 From: lowie27 Date: Mon, 24 Aug 2026 12:50:35 +0200 Subject: [PATCH] feat(secrets): add a reusable workflow for manual secret writes --- .../infra/secrets/preflight/action.yml | 261 +++++++++++ .../actions/infra/secrets/write/action.yml | 113 +++++ .../secrets/write/scripts/manual_secrets.sh | 421 ++++++++++++++++++ .github/workflows/infra-secret-write.yml | 404 +++++++++++++++++ Docs/infra-secret-write.md | 182 ++++++++ README.md | 1 + 6 files changed, 1382 insertions(+) create mode 100644 .github/actions/infra/secrets/preflight/action.yml create mode 100644 .github/actions/infra/secrets/write/action.yml create mode 100755 .github/actions/infra/secrets/write/scripts/manual_secrets.sh create mode 100644 .github/workflows/infra-secret-write.yml create mode 100644 Docs/infra-secret-write.md diff --git a/.github/actions/infra/secrets/preflight/action.yml b/.github/actions/infra/secrets/preflight/action.yml new file mode 100644 index 0000000..ed8200f --- /dev/null +++ b/.github/actions/infra/secrets/preflight/action.yml @@ -0,0 +1,261 @@ +--- +name: Secret Preflight +description: Prove that a manual secret write can succeed, before anyone is asked to approve it +inputs: + vendor: + description: The vendor to communicate with (scaleway) + required: true + project: + description: Application name, the prefix of the vendor project + required: true + environment: + description: Environment name, the suffix of the vendor project + required: true + key: + description: Variable name to add or overwrite + required: true + value: + description: Secret value, checked for length and for collisions with strings in the log + required: true + # Vendor: Scaleway + scaleway-organization-id: + description: Scaleway organization ID + required: false + scaleway-project-id: + description: Scaleway project ID + required: false + scaleway-region: + description: Scaleway deployment region (such as nl-ams) + required: false + scaleway-access-key: + description: Scaleway API key ID + required: false + scaleway-secret-key: + description: Scaleway API key secret + required: false + +outputs: + project-name: + description: Resolved vendor project name + value: ${{ steps.target.outputs.project-name }} + secret-name: + description: Resolved secret name + value: ${{ steps.target.outputs.secret-name }} + versions: + description: Number of enabled versions the secret has now + value: ${{ steps.secret.outputs.versions }} + warning: + description: Warning to surface to the approver, empty when there is nothing to warn about + value: ${{ steps.secret.outputs.warning }} + available: + description: Environments that do exist, for the error when the target does not + value: ${{ steps.projects.outputs.available }} + +runs: + using: composite + steps: + - name: Validate inputs + shell: bash + env: + VENDOR: ${{ inputs.vendor }} + run: | + echo "Validating inputs for vendor: $VENDOR" + case "$VENDOR" in + scaleway) + ;; + digitalocean) + echo "DigitalOcean has no secret manager, so there is nothing to write to" + exit 1 + ;; + azure) + echo "Azure Key Vault is not implemented, see Docs/infra-secret-write.md" + exit 1 + ;; + *) + echo "Unsupported vendor '$VENDOR'" + exit 1 + ;; + esac + + # Vendor-neutral: the key becomes an env var name whatever stores it, and the masking rules are + # GitHub's. Before the CLI setup deliberately, so a bad input fails without any network work. + - name: Validate the key and the value + shell: bash + env: + KEY: ${{ inputs.key }} + SECRET_VALUE: ${{ inputs.value }} + run: | + set -euo pipefail + + # [[ =~ ]] rather than grep: grep's ^ and $ are line anchors, so a key containing a + # newline would pass as long as one of its lines matched. The key becomes an env var name + # in the pod through envFrom, and envFrom silently skips one that is not a valid name. + # Not echoed back, because a key holding a newline would break the annotation. + # A vendor whose own naming is stricter adds that check in its own block below. + if [[ ! "$KEY" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then + echo "::error::invalid key: letters, digits and underscores only, not starting with a" \ + "digit. It becomes an environment variable name, so no hyphens" + exit 1 + fi + + # Masking is literal replacement, so a value occurring inside a string the reader knows is + # recoverable by subtraction: value "test" logged new-env-*** against new-env-test. + if [ "${#SECRET_VALUE}" -lt 8 ]; then + echo "::error::value is ${#SECRET_VALUE} characters, minimum 8," \ + "shorter values are recoverable from their own redaction" + exit 1 + fi + + # Quoted inside the pattern, so this is a substring test and not a glob. + case "$KEY" in + *"$SECRET_VALUE"*) + echo "::error::value occurs inside the key, so its redaction would reveal it." \ + "Pick a value that does not collide" + exit 1 + ;; + esac + + # Scaleway names every project - and every manual secret -secrets-manual, + # from sysops-tf-modules/monorepo/scw/environment. The collision check needs that name, so it + # cannot run above. + - name: Resolve the Scaleway target + id: target + if: ${{ inputs.vendor == 'scaleway' }} + shell: bash + env: + PROJECT: ${{ inputs.project }} + ENVIRONMENT: ${{ inputs.environment }} + SECRET_VALUE: ${{ inputs.value }} + run: | + set -euo pipefail + + project_name="${PROJECT}-${ENVIRONMENT}" + secret_name="${project_name}-secrets-manual" + + case "$secret_name" in + *"$SECRET_VALUE"*) + echo "::error::value occurs inside the environment or the secret name, so its" \ + "redaction would reveal it. Pick a value that does not collide" + exit 1 + ;; + esac + + echo "project-name=$project_name" >> "$GITHUB_OUTPUT" + echo "secret-name=$secret_name" >> "$GITHUB_OUTPUT" + + - uses: wisemen-digital/devops-github-actions/.github/actions/infra/common/setup@v1 + if: ${{ inputs.vendor == 'scaleway' }} + with: + vendor: ${{ inputs.vendor }} + scaleway-access-key: ${{ inputs.scaleway-access-key }} + scaleway-secret-key: ${{ inputs.scaleway-secret-key }} + scaleway-organization-id: ${{ inputs.scaleway-organization-id }} + scaleway-project-id: ${{ inputs.scaleway-project-id }} + + # Without this a stale dropdown option surfaces further down as "secret not found", which + # reads like a permissions or region problem. Says what does exist, so a wrong guess gets one. + - name: Verify the target project exists + id: projects + if: ${{ inputs.vendor == 'scaleway' }} + shell: bash + env: + PROJECT_NAME: ${{ steps.target.outputs.project-name }} + PROJECT: ${{ inputs.project }} + run: | + set -euo pipefail + + projects="$(scw account project list -o json)" + + # -infra-shared shares the prefix but holds the cluster and the registry, never a + # manual secret, so it is not an environment and is not offered as one. + available="$(printf '%s' "$projects" \ + | jq -r --arg p "$PROJECT-" '.[].name + | select(startswith($p)) | select(endswith("-infra-shared") | not)' \ + | sort | paste -sd, - | sed 's/,/, /g')" + + # A wrong app matches no prefix, and "none" would be the whole error. Widen to every + # project, in full, so the reader still gets something to compare against. "default" is + # dropped only here, since no app name matches it above. + if [ -z "$available" ]; then + available="$(printf '%s' "$projects" \ + | jq -r '.[].name | select(endswith("-infra-shared") | not) | select(. != "default")' \ + | sort | paste -sd, - | sed 's/,/, /g')" + fi + + # Exported before the failure below, so the summary can show it either way. + echo "available=${available:-none}" >> "$GITHUB_OUTPUT" + + project_id="$(printf '%s' "$projects" \ + | jq -r --arg n "$PROJECT_NAME" '.[] | select(.name == $n) | .id')" + + if [ -n "$project_id" ]; then + echo "Target project $PROJECT_NAME exists ($project_id)." + echo "project-id=$project_id" >> "$GITHUB_OUTPUT" + exit 0 + fi + + echo "::error::No Scaleway project named $PROJECT_NAME." \ + "Projects that do exist: ${available:-none}" + exit 1 + + # Fails rather than warns: write has `needs: preflight`, so nobody is asked to approve a write + # that could not have succeeded. + - name: Verify the target secret exists + id: secret + if: ${{ inputs.vendor == 'scaleway' }} + shell: bash + env: + SECRET_NAME: ${{ steps.target.outputs.secret-name }} + PROJECT_ID: ${{ steps.projects.outputs.project-id }} + REGION: ${{ inputs.scaleway-region }} + run: | + set -euo pipefail + + # The API's name filter is not guaranteed to be exact, so filter again here and assert + # exactly one match, the same way manual_secrets.sh does. + secrets="$(scw secret secret list name="$SECRET_NAME" project-id="$PROJECT_ID" \ + region="$REGION" -o json)" + match="$(printf '%s' "$secrets" | jq --arg n "$SECRET_NAME" '[.[] | select(.name == $n)]')" + count="$(printf '%s' "$match" | jq 'length')" + + if [ "$count" -eq 0 ]; then + # Terraform creates this secret with prevent_destroy, so its absence means the + # environment was never applied rather than that someone deleted it. + echo "::error::No secret named $SECRET_NAME in project $PROJECT_ID, region $REGION." \ + "Terraform creates it, so the environment has probably never been applied" + exit 1 + fi + if [ "$count" -ne 1 ]; then + echo "::error::$count secrets named $SECRET_NAME; refusing to guess" + exit 1 + fi + + secret_id="$(printf '%s' "$match" | jq -r '.[0].id')" + secret_type="$(printf '%s' "$match" | jq -r '.[0].type')" + + if [ "$secret_type" != "key_value" ]; then + echo "::error::secret $SECRET_NAME is type $secret_type, not key_value" + exit 1 + fi + + # Enabled versions only, not version_count, which also counts versions scheduled for + # deletion. Same reasoning as resolve_secret in manual_secrets.sh. + versions="$(scw secret version list "$secret_id" region="$REGION" -o json)" + enabled="$(printf '%s' "$versions" | jq '[.[] | select(.status == "enabled")] | length')" + total="$(printf '%s' "$versions" | jq 'length')" + + # Nothing readable, so the write starts from {} and the keys already stored stop being + # synced. Warned rather than refused: the old versions survive and can be re-enabled, and + # refusing would need the Scaleway access this workflow exists to avoid needing. + warning="" + if [ "$enabled" -eq 0 ] && [ "$total" -gt 0 ]; then + warning="**Warning:** all $total version(s) of this secret are disabled, so the write" + warning="$warning starts from empty and every other key in it stops being synced." + fi + + { + echo "secret-id=$secret_id" + echo "versions=$enabled" + echo "warning=$warning" + } >> "$GITHUB_OUTPUT" + echo "Secret $SECRET_NAME exists ($secret_id), $enabled of $total version(s) enabled." diff --git a/.github/actions/infra/secrets/write/action.yml b/.github/actions/infra/secrets/write/action.yml new file mode 100644 index 0000000..3372358 --- /dev/null +++ b/.github/actions/infra/secrets/write/action.yml @@ -0,0 +1,113 @@ +--- +name: Secret Write +description: Add or overwrite one key in a manual secret, keeping every other key it holds +inputs: + vendor: + description: The vendor to communicate with (scaleway) + required: true + secret-name: + description: Name of the secret to write, as resolved by the preflight action + required: true + project-name: + description: Name of the vendor project holding the secret + required: true + key: + description: Variable name to add or overwrite + required: true + value: + description: Secret value + required: true + description: + description: Description recorded on the new version + required: false + default: '' + # Vendor: Scaleway + scaleway-organization-id: + description: Scaleway organization ID + required: false + scaleway-project-id: + description: Scaleway project ID + required: false + scaleway-region: + description: Scaleway deployment region (such as nl-ams) + required: false + scaleway-access-key: + description: Scaleway API key ID + required: false + scaleway-secret-key: + description: Scaleway API key secret + required: false + +outputs: + log: + description: Path to the run log, for a caller that wants to summarise it + value: ${{ steps.write.outputs.log }} + +runs: + using: composite + steps: + - name: Validate inputs + shell: bash + env: + VENDOR: ${{ inputs.vendor }} + run: | + echo "Validating inputs for vendor: $VENDOR" + case "$VENDOR" in + scaleway) + ;; + digitalocean) + echo "DigitalOcean has no secret manager, so there is nothing to write to" + exit 1 + ;; + azure) + echo "Azure Key Vault is not implemented, see Docs/infra-secret-write.md" + exit 1 + ;; + *) + echo "Unsupported vendor '$VENDOR'" + exit 1 + ;; + esac + + - uses: wisemen-digital/devops-github-actions/.github/actions/infra/common/setup@v1 + if: ${{ inputs.vendor == 'scaleway' }} + with: + vendor: ${{ inputs.vendor }} + scaleway-access-key: ${{ inputs.scaleway-access-key }} + scaleway-secret-key: ${{ inputs.scaleway-secret-key }} + scaleway-organization-id: ${{ inputs.scaleway-organization-id }} + scaleway-project-id: ${{ inputs.scaleway-project-id }} + + # Project by name: the name is derivable from -, the id is not. Region, project and + # organization are all explicit, because a wrong one returns an empty list rather than an error. + - name: Write the new secret version + id: write + if: ${{ inputs.vendor == 'scaleway' }} + shell: bash + env: + SECRET_NAME: ${{ inputs.secret-name }} + SCW_SECRETS_PROJECT_NAME: ${{ inputs.project-name }} + SCW_SECRETS_ORGANIZATION_ID: ${{ inputs.scaleway-organization-id }} + SCW_SECRETS_REGION: ${{ inputs.scaleway-region }} + KEY: ${{ inputs.key }} + DESCRIPTION: ${{ inputs.description }} + SECRET_VALUE: ${{ inputs.value }} + run: | + set -euo pipefail + umask 077 + export TMPDIR="$RUNNER_TEMP" + + args=(set "$SECRET_NAME" - --yes) + if [ -n "$DESCRIPTION" ]; then + args+=(--description "$DESCRIPTION") + fi + + echo "log=$RUNNER_TEMP/manual-secret.log" >> "$GITHUB_OUTPUT" + + # $GITHUB_ACTION_PATH rather than github.action_path, because of a bug in GA + # Refs: https://github.com/actions/runner/issues/2185 + # + # jq reads $ENV and the patch goes over stdin: no shell string, no argv, no file in this step. + jq -n '{($ENV.KEY): $ENV.SECRET_VALUE}' \ + | "$GITHUB_ACTION_PATH/scripts/manual_secrets.sh" "${args[@]}" 2>&1 \ + | tee "$RUNNER_TEMP/manual-secret.log" diff --git a/.github/actions/infra/secrets/write/scripts/manual_secrets.sh b/.github/actions/infra/secrets/write/scripts/manual_secrets.sh new file mode 100755 index 0000000..12c0c02 --- /dev/null +++ b/.github/actions/infra/secrets/write/scripts/manual_secrets.sh @@ -0,0 +1,421 @@ +#!/usr/bin/env bash +# +# Read and update a Scaleway Secret Manager key_value secret, without losing keys. +# +# Scaleway secret versions are immutable, so "add one key" means: read the latest version, layer +# the change on top, write a whole new version. Done by hand in the console, that is exactly where +# an existing key gets dropped. This does it mechanically and checks nothing was lost. +# +# Built for the *-secrets-manual secrets that sysops-tf-modules/monorepo/scw/environment creates +# with prevent_destroy and deliberately never writes a scaleway_secret_version for. +# +# Nothing sensitive reaches argv: the write uses scw's data=@file form, and the token comes from +# the scw config or the environment. Values are never printed, only key names. The get outfile does +# contain values, that being the point of it. + +set -euo pipefail +# Both the get outfile and the temp files hold plaintext secrets, and the caller's umask would +# usually leave them world-readable. +umask 077 + +readonly DEFAULT_REGION="nl-ams" + +REGION="${SCW_SECRETS_REGION:-$DEFAULT_REGION}" +PROJECT_ID="${SCW_SECRETS_PROJECT_ID:-}" +PROJECT_NAME="${SCW_SECRETS_PROJECT_NAME:-}" +ORGANIZATION_ID="${SCW_SECRETS_ORGANIZATION_ID:-}" +ASSUME_YES=0 +DRY_RUN=0 +DESCRIPTION="" + +SECRET_ID="" # set by resolve_secret +SECRET_VERSION_COUNT=0 # enabled versions, set by resolve_secret +SECRET_TOTAL_VERSION_COUNT=0 # versions in any state, set by resolve_secret +CREATED_REVISION="" # set by create_version + +TMPDIR_RUN="" +cleanup() { + if [[ -n "$TMPDIR_RUN" ]]; then + rm -rf -- "$TMPDIR_RUN" + fi +} +# Not EXIT alone: bash does not run an EXIT trap on SIGTERM, and these files hold plaintext. +trap cleanup EXIT INT TERM HUP + +die() { printf 'error: %s\n' "$*" >&2; exit 1; } +warn() { printf 'warning: %s\n' "$*" >&2; } +info() { printf '%s\n' "$*" >&2; } + +usage() { + cat >&2 <<-'EOF' + Usage: + manual_secrets.sh get [outfile] + manual_secrets.sh set [--dry-run] [--yes] + + get Fetch the latest version and write it as pretty JSON. Read-only. + outfile defaults to ./_.json + + set Merge a JSON object of key/value pairs over the latest version and write a new + version. An existing key is overwritten, a new key is appended, nothing else is + touched. Pass - to read the patch from stdin. + + Options: + --region Scaleway region (default nl-ams, or $SCW_SECRETS_REGION). + Note: your scw profile may default to fr-par, where these secrets + do not exist, and a wrong region looks like "secret not found". + --project-id Scaleway project (default: the scw profile's default-project-id, + or $SCW_SECRETS_PROJECT_ID) + --project-name Scaleway project by name, looked up at run time (or + $SCW_SECRETS_PROJECT_NAME). Terraform names every project + -, so a caller that knows both needs no id. + Mutually exclusive with --project-id. + --organization-id Organization to look --project-name up in (or + $SCW_SECRETS_ORGANIZATION_ID). Defaults to the scw profile's. + --description Description recorded on the new version + --dry-run Show the summary, write nothing + --yes Do not prompt for confirmation + + Requires: scw (authenticated, see scw init), jq, base64. + EOF + exit "${1:-2}" +} + +require_tools() { + local t + for t in scw jq base64; do + command -v "$t" >/dev/null 2>&1 || die "$t is not installed" + done +} + +# Exists so a caller never has to hold an id: Terraform names every project - +# (sysops-tf-modules monorepo/wrappers/env), so app plus environment is enough to find it. +resolve_project() { + local name="$1" json count + local -a args=(account project list name="$name" -o json) + if [[ -n "$ORGANIZATION_ID" ]]; then + args+=(organization-id="$ORGANIZATION_ID") + fi + + json=$(scw "${args[@]}" 2>&1) || die "scw account project list failed: $json" + jq -e 'type == "array"' >/dev/null 2>&1 <<<"$json" \ + || die "unexpected response from scw account project list: $json" + + json=$(jq --arg n "$name" '[.[] | select(.name == $n)]' <<<"$json") + count=$(jq 'length' <<<"$json") + + case "$count" in + 0) die "no project named '$name'${ORGANIZATION_ID:+ in organization $ORGANIZATION_ID}. \ +The filter returns an empty list rather than an error, so a project that does not exist and the \ +wrong organization look identical here." ;; + 1) ;; + *) die "$count projects named '$name'; refusing to guess" ;; + esac + + PROJECT_ID=$(jq -r '.[0].id' <<<"$json") + [[ -n "$PROJECT_ID" && "$PROJECT_ID" != "null" ]] || die "could not read the id of project '$name'" +} + +# Both commands scope their secret lookup to a project, so this runs before resolve_secret in each. +ensure_project() { + if [[ -z "$PROJECT_ID" && -n "$PROJECT_NAME" ]]; then + resolve_project "$PROJECT_NAME" + info "project $PROJECT_NAME -> $PROJECT_ID" + fi +} + +# The API's name filter is not guaranteed to be exact, so check the name ourselves and assert +# exactly one match. +# +# Sets globals rather than echoing: called as $(resolve_secret ...) it would run in a subshell, +# where die() exits only that subshell and the caller carries on with an empty id. +resolve_secret() { + local name="$1" json count + local -a args=(secret secret list name="$name" region="$REGION" -o json) + if [[ -n "$PROJECT_ID" ]]; then + args+=(project-id="$PROJECT_ID") + fi + + json=$(scw "${args[@]}" 2>&1) || die "scw secret list failed: $json" + jq -e 'type == "array"' >/dev/null 2>&1 <<<"$json" \ + || die "unexpected response from scw secret list: $json" + + json=$(jq --arg n "$name" '[.[] | select(.name == $n)]' <<<"$json") + count=$(jq 'length' <<<"$json") + + case "$count" in + 0) die "no secret named '$name' in region $REGION${PROJECT_ID:+ project $PROJECT_ID}. \ +Wrong region is the usual cause: the filter returns an empty list rather than an error." ;; + 1) ;; + *) die "$count secrets named '$name'; refusing to guess" ;; + esac + + jq -e '.[0].type == "key_value"' >/dev/null <<<"$json" \ + || die "secret '$name' is type $(jq -r '.[0].type' <<<"$json"), not key_value" + + SECRET_ID=$(jq -r '.[0].id' <<<"$json") + [[ -n "$SECRET_ID" && "$SECRET_ID" != "null" ]] || die "could not read the id of '$name'" + + # Enabled versions rather than the secret's version_count, which also counts versions only + # scheduled for deletion, so the read would 404 on a count that looks non-zero. Filtered here + # rather than with status.0=enabled, which the CLI accepts and ignores. + local versions + versions=$(scw secret version list "$SECRET_ID" region="$REGION" -o json 2>&1) \ + || die "scw secret version list failed: $versions" + SECRET_VERSION_COUNT=$(jq '[.[] | select(.status == "enabled")] | length' <<<"$versions") + SECRET_TOTAL_VERSION_COUNT=$(jq 'length' <<<"$versions") +} + +# Writes to a file rather than stdout so it is never called from a pipeline, where it would run in +# a subshell and die() could not stop the caller. Same reason as resolve_secret. +fetch_latest() { + local dest="$1" json data + + if [[ "$SECRET_VERSION_COUNT" -eq 0 ]]; then + # No versions at all is how Terraform leaves *-secrets-manual. Versions that exist but are + # all disabled is a different state: nothing is readable, so the new version holds only the + # patch. Warned rather than refused, because the disabled versions survive and can be + # re-enabled, and refusing would leave a caller with no Scaleway access unable to proceed. + if [[ "$SECRET_TOTAL_VERSION_COUNT" -gt 0 ]]; then + warn "all $SECRET_TOTAL_VERSION_COUNT version(s) are disabled, starting from empty; \ +every other key in this secret stops being synced" + else + info "secret has no versions at all, starting from an empty object" + fi + printf '{}' > "$dest" + return + fi + + # latest_enabled matches the enabled count above. A rollback disables the newest version, which + # latest would still point at, and the API refuses to read it. + json=$(scw secret version access "$SECRET_ID" revision=latest_enabled region="$REGION" -o json 2>&1) \ + || die "scw secret version access failed: $(head -c 400 <<<"$json")" + + # scw -o json leaves .data base64-encoded; it only decodes for human output. + data=$(jq -r '.data // empty' <<<"$json") + [[ -n "$data" ]] || die "latest version of $SECRET_ID has no data" + + base64 -d <<<"$data" > "$dest" || die "latest version of $SECRET_ID is not valid base64" + # -s, not a bare -e: a stream of objects passes `type == "object"` value by value, and the + # slurped merge in cmd_set would then read the second object as the patch. + jq -se 'length == 1 and (.[0] | type == "object")' >/dev/null 2>&1 < "$dest" \ + || die "latest version of $SECRET_ID is not a single JSON object" +} + +# A key_value secret holds flat string values. Asserting that keeps the merge unambiguous and +# catches a malformed payload before it reaches a running pod through External Secrets Operator. +# A non-empty $3 downgrades an invalid key name to a warning, for a payload this run did not write: +# envFrom skips such a key and the pod runs on without it, so refusing would let one key added in the +# console block every later write by everyone. +assert_flat_string_object() { + local file="$1" what="$2" lenient="${3:-}" bad + jq -e 'type == "object"' >/dev/null < "$file" \ + || die "$what is not a JSON object" + bad=$(jq -r '[to_entries[] | select(.value | type != "string") | .key] | join(", ")' "$file") + [[ -z "$bad" ]] || die "$what has non-string values for: $bad (key_value secrets hold strings)" + + # Fatal either way: a key holding a newline desyncs the four-line summary read in cmd_set, which + # would then print a confident, wrong summary. + bad=$(jq -r '[keys[] | select(test("[\\n\\r]")) | @json] | join(", ")' "$file") + [[ -z "$bad" ]] || die "$what has key(s) containing a newline: $bad" + + # Keys become env var names in the pod through envFrom. + bad=$(jq -r '[keys[] | select(test("^[A-Za-z_][A-Za-z0-9_]*$") | not) | @json] | join(", ")' "$file") + if [[ -n "$bad" ]] && [[ -n "$lenient" ]]; then + warn "$what has key(s) that are not valid env var names: $bad. envFrom skips them, so the \ +pod never sees them. Not written by this run; fix them where they were added." + elif [[ -n "$bad" ]]; then + die "$what has key(s) that are not valid env var names: $bad" + fi +} + +cmd_get() { + local name="${1:-}" outfile="${2:-}" + [[ -n "$name" ]] || usage + + ensure_project + resolve_secret "$name" + info "secret $name -> $SECRET_ID ($SECRET_VERSION_COUNT version(s), region $REGION)" + + [[ -n "$outfile" ]] || outfile="./${name}_$(date -u +%Y%m%dT%H%M%SZ).json" + + # Global, not local, so the EXIT trap can remove it whichever way we leave. + TMPDIR_RUN=$(mktemp -d) + fetch_latest "$TMPDIR_RUN/latest.json" + # -S sorts the keys, so two dumps of the same secret diff cleanly. + jq -S . < "$TMPDIR_RUN/latest.json" > "$outfile" + + info "" + info "wrote $(jq 'length' "$outfile") key(s) to $outfile:" + jq -r 'keys[] | " " + .' "$outfile" >&2 +} + +cmd_set() { + local name="${1:-}" patch_arg="${2:-}" + [[ -n "$name" && -n "$patch_arg" ]] || usage + + # Three files: what is stored now, what the caller wants changed, and the result. Keeping them + # separate is what makes the summary and the safety check possible. + TMPDIR_RUN=$(mktemp -d) + local old="$TMPDIR_RUN/old.json" + local patch="$TMPDIR_RUN/patch.json" + local merged="$TMPDIR_RUN/merged.json" + + if [[ "$patch_arg" == "-" ]]; then + cat > "$patch" + else + [[ -f "$patch_arg" ]] || die "patch file not found: $patch_arg" + cat -- "$patch_arg" > "$patch" + fi + # -s rejects a stream of values here, where the message is accurate. Plain `jq -e .` accepts + # `{"a":"1"}{"b":"2"}` and the failure surfaces later with the wrong diagnosis. + jq -se 'length == 1' >/dev/null 2>&1 < "$patch" || die "patch must be a single JSON object" + assert_flat_string_object "$patch" "patch" + [[ "$(jq 'length' "$patch")" -gt 0 ]] || die "patch is empty, nothing to do" + + ensure_project + resolve_secret "$name" + info "secret $name -> $SECRET_ID ($SECRET_VERSION_COUNT version(s), region $REGION)" + + fetch_latest "$old" + assert_flat_string_object "$old" "current version of $name" lenient + + # `+` is a shallow, right-wins merge: existing key overwritten, new key appended, every other + # key carried over. Deliberately not `*`, which merges recursively. + jq -sS '.[0] + .[1]' "$old" "$patch" > "$merged" + + # Nothing already stored may be lost. Checking only for missing keys can never fail, since `+` + # cannot drop one, so check the values of the keys this call does not touch: that is what breaks + # if the merge operator above is ever changed. + local harmed + harmed=$(jq -rn --slurpfile o "$old" --slurpfile p "$patch" --slurpfile m "$merged" ' + $o[0] as $old | $p[0] as $patch | $m[0] as $merged + | [ $old | keys[] | . as $k + | select(($patch | has($k)) | not) + | select((($merged | has($k)) | not) or ($merged[$k] != $old[$k])) ] + | join(", ")') + [[ -z "$harmed" ]] || die "merge would drop or alter untouched key(s): $harmed. Refusing to write." + + # Computed from old versus patch, not from merged, so the summary shows the intent of this + # call: what is new, what changes, what was sent but changes nothing, and how much this call + # does not touch. Names only, never values. + local added changed identical untouched summary + summary=$(jq -rn --slurpfile o "$old" --slurpfile p "$patch" ' + $o[0] as $old | $p[0] as $new + | ($new | keys) as $new_keys + | [ ($new_keys - ($old | keys) | join(", ")), + ([$new_keys[] | . as $k | select(($old | has($k)) and $old[$k] != $new[$k])] | join(", ")), + ([$new_keys[] | . as $k | select(($old | has($k)) and $old[$k] == $new[$k])] | length), + (($old | keys) - $new_keys | length) ] + | .[]') || die "could not compute the change summary" + # Four lines, in order. Fewer means the jq above changed shape, and carrying on would print a + # confident summary built from empty variables. + { + read -r added + read -r changed + read -r identical + read -r untouched + } <<<"$summary" || die "change summary was incomplete, refusing to write" + + info "" + info "added: ${added:-(none)}" + info "overwritten: ${changed:-(none)}" + info "already identical: $identical" + info "carried over: $untouched" + info "result: $(jq 'length' "$merged") key(s), was $(jq 'length' "$old")" + info "" + + # Versions are immutable and accumulate, and a run of identical revisions makes the console + # history useless for working out when a value actually changed. + if [[ -z "$added" && -z "$changed" ]]; then + info "nothing would change, not writing a new version" + return 0 + fi + + if [[ "$DRY_RUN" -eq 1 ]]; then + info "dry run, not writing" + return 0 + fi + + if [[ "$ASSUME_YES" -eq 0 ]]; then + local reply + # A runner has no controlling terminal, and the bare redirect below fails with "No such + # device or address", which does not say what to do about it. + ( exec < /dev/tty ) 2>/dev/null \ + || die "no terminal for the confirmation prompt; pass --yes" + # From the terminal, not stdin: with `set -` the patch came from stdin, which is now + # at EOF, and the prompt would answer itself with an empty line. + read -r -p "Write a new version of $name? [y/N] " reply < /dev/tty + [[ "$reply" == "y" || "$reply" == "Y" ]] || die "aborted" + fi + + create_version "$merged" "$name" + + if [[ -n "$CREATED_REVISION" ]]; then + info "wrote revision $CREATED_REVISION of $name" + else + info "wrote a new version of $name" + fi + info "the previous revision stays enabled, so rollback in the console remains possible" +} + +# data=@file is scw's file-loading form: the CLI reads the file itself, so the payload never reaches +# argv, and it base64-encodes it for the API on its own. +# +# Sets CREATED_REVISION rather than echoing it, for the same subshell reason as resolve_secret. +create_version() { + local merged="$1" name="$2" json + + # disable-previous omitted deliberately, so the previous version stays enabled. + json=$(scw secret version create "$SECRET_ID" \ + "data=@$merged" \ + "description=${DESCRIPTION:-updated by manual_secrets.sh}" \ + region="$REGION" -o json 2>&1) \ + || die "creating a version of $name failed: $(head -c 400 <<<"$json")" + + # Left empty rather than fatal if it cannot be read: the version has been written by this + # point, and failing here would report a successful write as an error. + CREATED_REVISION=$(jq -r '.revision // empty' <<<"$json" 2>/dev/null) || CREATED_REVISION="" +} + +main() { + local -a positional=() + while [[ $# -gt 0 ]]; do + case "$1" in + --region) REGION="${2:?--region needs a value}"; shift 2 ;; + --project-id) PROJECT_ID="${2:?--project-id needs a value}"; shift 2 ;; + --project-name) PROJECT_NAME="${2:?--project-name needs a value}"; shift 2 ;; + --organization-id) + ORGANIZATION_ID="${2:?--organization-id needs a value}"; shift 2 ;; + --description) DESCRIPTION="${2:?--description needs a value}"; shift 2 ;; + --dry-run) DRY_RUN=1; shift ;; + --yes|-y) ASSUME_YES=1; shift ;; + -h|--help) usage 0 ;; + --) shift; positional+=("$@"); break ;; + -) positional+=("$1"); shift ;; # the stdin marker, not an option + -*) die "unknown option: $1" ;; + *) positional+=("$1"); shift ;; + esac + done + + # Either can also arrive through the environment, so name both sources: a precedence rule here + # would resolve to one project while the caller believes it asked for the other. + if [[ -n "$PROJECT_ID" && -n "$PROJECT_NAME" ]]; then + die "--project-id and --project-name are mutually exclusive (also settable as \ +SCW_SECRETS_PROJECT_ID and SCW_SECRETS_PROJECT_NAME)" + fi + + # After parsing, so --help works on a machine without the tools installed. + require_tools + + [[ ${#positional[@]} -ge 1 ]] || usage + # Both subcommands take at most two operands. Silently ignoring a third hides a typo. + [[ ${#positional[@]} -le 3 ]] || die "too many arguments: ${positional[*]:3}" + + case "${positional[0]}" in + get) cmd_get "${positional[@]:1}" ;; + set) cmd_set "${positional[@]:1}" ;; + *) usage ;; + esac +} + +main "$@" diff --git a/.github/workflows/infra-secret-write.yml b/.github/workflows/infra-secret-write.yml new file mode 100644 index 0000000..51db17c --- /dev/null +++ b/.github/workflows/infra-secret-write.yml @@ -0,0 +1,404 @@ +--- +name: Add a manual secret (vendor agnostic) + +# +# Adds or overwrites one key in --secrets-manual without losing the keys already +# there. preflight validates and proves the write can succeed, approval waits for a listed +# approver, write does the read-merge-write. +# +# The caller's workflow_dispatch input holding the value must be named `value`: the mask step reads +# it out of the event payload, because putting it in a step's env: would print it before the mask +# exists. +# + +on: + workflow_call: + # Depends on the following vars + # - RUNNER_INFRA + # For Scaleway vendor: + # - SCALEWAY_ORGANIZATION_ID + # - SCALEWAY_PROJECT_ID + # - SCALEWAY_REGION + inputs: + vendor: + description: The vendor to communicate with (scaleway) + type: string + required: true + project: + description: Application name, the prefix of the vendor project + type: string + required: true + environment: + description: Environment name, the suffix of the vendor project + type: string + required: true + key: + description: Variable name to add or overwrite + type: string + required: true + approvers: + description: Comma-separated GitHub logins allowed to approve. Usernames, never a team slug + type: string + required: true + write-environment: + description: Environment the write job runs in, for its branch policy + type: string + required: false + default: secret-write-approval + secrets: + SECRET_VALUE: + description: The secret value to store + required: true + # Vendor: Scaleway + SCALEWAY_ACCESS_KEY: + description: Scaleway API key ID + required: false + SCALEWAY_SECRET_KEY: + description: Scaleway API key secret + required: false + +jobs: + preflight: + name: Check ${{ inputs.key }} against ${{ inputs.project }}-${{ inputs.environment }} + runs-on: ${{ vars.RUNNER_INFRA || 'ubuntu-latest' }} + timeout-minutes: 5 + permissions: {} + outputs: + project-name: ${{ steps.preflight.outputs.project-name }} + secret-name: ${{ steps.preflight.outputs.secret-name }} + versions: ${{ steps.preflight.outputs.versions }} + warning: ${{ steps.preflight.outputs.warning }} + steps: + # Must run first: a mask applies to the rest of the job, not retroactively. Reads the event + # payload rather than env: or ${{ }}, because the runner prints both a step's environment and + # its script body before running it. Per line, because ::add-mask:: consumes to end of line. + - name: Mask the value + run: | + set -euo pipefail + + # Without this, a caller naming the input differently masks nothing and still succeeds. + if ! jq -e 'has("inputs") and (.inputs | has("value"))' "$GITHUB_EVENT_PATH" >/dev/null; then + echo "::error::the dispatch input holding the value must be named 'value'," \ + "see Docs/infra-secret-write.md" + exit 1 + fi + + jq -r '.inputs.value // empty' "$GITHUB_EVENT_PATH" \ + | while IFS= read -r line; do + if [ -n "$line" ]; then + echo "::add-mask::$line" + fi + done + + - name: Preflight + id: preflight + uses: wisemen-digital/devops-github-actions/.github/actions/infra/secrets/preflight@v1 + with: + vendor: ${{ inputs.vendor }} + project: ${{ inputs.project }} + environment: ${{ inputs.environment }} + key: ${{ inputs.key }} + value: ${{ secrets.SECRET_VALUE }} + scaleway-access-key: ${{ secrets.SCALEWAY_ACCESS_KEY }} + scaleway-secret-key: ${{ secrets.SCALEWAY_SECRET_KEY }} + scaleway-organization-id: ${{ vars.SCALEWAY_ORGANIZATION_ID }} + scaleway-project-id: ${{ vars.SCALEWAY_PROJECT_ID }} + scaleway-region: ${{ vars.SCALEWAY_REGION || 'nl-ams' }} + + - name: Preflight summary + if: always() + env: + PROJECT_NAME: ${{ steps.preflight.outputs.project-name }} + SECRET_NAME: ${{ steps.preflight.outputs.secret-name }} + KEY: ${{ inputs.key }} + ACTOR: ${{ github.actor }} + VERSIONS: ${{ steps.preflight.outputs.versions }} + REGION: ${{ vars.SCALEWAY_REGION || 'nl-ams' }} + STATUS: ${{ job.status }} + AVAILABLE: ${{ steps.preflight.outputs.available }} + WARNING: ${{ steps.preflight.outputs.warning }} + run: | + set -euo pipefail + + { + echo "## Preflight" + echo "" + echo "| | |" + echo "| --- | --- |" + echo "| Requested by | \`$ACTOR\` |" + echo "| Key | \`$KEY\` |" + echo "| Project | \`${PROJECT_NAME:-unresolved}\` |" + echo "| Secret | \`${SECRET_NAME:-unresolved}\` |" + echo "| Region | $REGION |" + echo "| Enabled versions now | ${VERSIONS:-unknown} |" + echo "| Preflight | $STATUS |" + echo "" + if [ -n "$WARNING" ]; then + echo "$WARNING" + echo "" + fi + if [ "$STATUS" = "success" ]; then + echo "Nothing has been written. The next job opens an approval request and waits" + echo "for a code owner, and it links the issue from its own summary." + elif [ "$STATUS" = "cancelled" ]; then + echo "The run was cancelled before preflight finished. Nothing has been written and" + echo "no approval was requested." + else + echo "Preflight failed, so the write job will not run and no approval is needed." + echo "" + echo "Projects that exist: ${AVAILABLE:-unknown, the project list was never read}" + fi + } >> "$GITHUB_STEP_SUMMARY" + + # Stands in for GitHub's required reviewers, which need Enterprise on a private repo. Holds no + # vendor credential and no environment, so the only thing it can reach is its own issue. + approval: + name: Await approval to write ${{ inputs.key }} + needs: preflight + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + issues: write + env: + APPROVERS: ${{ inputs.approvers }} + steps: + - name: Wait for a code owner + id: approval + uses: trstringer/manual-approval@v1 + with: + secret: ${{ github.token }} + approvers: ${{ env.APPROVERS }} + minimum-approvals: 1 + exclude-workflow-initiator-as-approver: false + fail-on-denial: true + close-issue-means-denial: true + # Not the default 10, which exceeds the rate limit: trstringer/manual-approval#235 + polling-interval-seconds: 90 + issue-labels: secret-approval + issue-title: >- + Approve secret write: ${{ inputs.key }} to + ${{ inputs.project }}-${{ inputs.environment }} (run ${{ github.run_id }}) + issue-body: |- + **@${{ github.actor }}** wants to write `${{ inputs.key }}` + to `${{ needs.preflight.outputs.secret-name }}`. + + | | | + | --- | --- | + | Key | `${{ inputs.key }}` | + | Environment | ${{ inputs.environment }} | + | Vendor project | `${{ needs.preflight.outputs.project-name }}` | + | Enabled versions now | ${{ needs.preflight.outputs.versions }} | + | Run | ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | + + ${{ needs.preflight.outputs.warning }} + + **APPROVE**: comment `approve`, `approved`, `lgtm` or `yes` + **DENY**: comment `deny`, `denied` or `no`, or close this issue + + Comment the word on its own. `approve this` and `+1` do nothing. + + # The audit record. always() because a denial fails the action, and a rejected request is + # precisely the outcome worth recording. + - name: Record the outcome + if: always() + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + STATUS: ${{ steps.approval.outputs.approval-status }} + OUTCOME: ${{ steps.approval.outcome }} + ISSUE_URL: ${{ steps.approval.outputs.issue-url }} + ISSUE_NUMBER: ${{ steps.approval.outputs.issue-number }} + JOB_STATUS: ${{ job.status }} + KEY: ${{ inputs.key }} + ACTOR: ${{ github.actor }} + # Never read: without a reference here, Set up job prints the caller's input in the clear. + # On the step, not the job, so the approval container never gets it. + SECRET_VALUE: ${{ secrets.SECRET_VALUE }} + run: | + set -euo pipefail + + # The action exposes no decider, so it is recovered from the comments. Word list from + # its constants.go. + decider="" + if [ -n "${ISSUE_NUMBER:-}" ]; then + decider="$(gh issue view "$ISSUE_NUMBER" --repo "$REPO" --json comments \ + | jq -r --arg approvers "$APPROVERS" ' + ($approvers | ascii_downcase | split(",") + | map(gsub("^\\s+|\\s+$"; ""))) as $allowed + | [ .comments[] + | select(.author.login | ascii_downcase | IN($allowed[])) + | select(.body | test("^(approve|approved|lgtm|yes|deny|denied|no)[.!]*\\s*$"; "i")) + ] + | first | .author.login // empty' 2>/dev/null || true)" + fi + + # The table is the audit record, so it must not state a denial the comments cannot + # evidence. The action reports denied on an API error as well as on a real one. + decision="${STATUS:-none recorded}" + if [ "${STATUS:-}" = "denied" ] && [ -z "$decider" ]; then + decision="denied, unconfirmed (no decision comment found)" + fi + + { + echo "## Approval outcome" + echo "" + echo "| | |" + echo "| --- | --- |" + echo "| Key | \`$KEY\` |" + echo "| Requested by | @$ACTOR |" + echo "| Decision | $decision |" + echo "| Decided by | ${decider:+@}${decider:-unknown} |" + if [ -n "${ISSUE_URL:-}" ]; then + echo "| Request | [#${ISSUE_NUMBER}]($ISSUE_URL) |" + fi + echo "| Job | $JOB_STATUS |" + echo "" + if [ "${STATUS:-}" = "approved" ]; then + echo "The write job runs next." + elif [ "${STATUS:-}" = "denied" ] && [ -n "$decider" ]; then + echo "Denied by @$decider, so the write job will not run. Nothing has been written." + elif [ "${STATUS:-}" = "denied" ]; then + echo "Reported as denied, but no decision comment was found. That is either a closed" + echo "issue or an approval-action failure, not necessarily a rejection. Nothing has" + echo "been written; check the job log before dispatching again." + elif [ "${OUTCOME:-}" = "failure" ]; then + echo "The approval step failed before recording a decision. Nothing has been written." + echo "This is not a rejection; check the job log." + else + echo "No decision was recorded, so the request timed out or the run was cancelled." + echo "Nothing has been written. Dispatch again if the write is still wanted." + fi + } >> "$GITHUB_STEP_SUMMARY" + + # The action closes its own issue on a decision, but not when its polling loop errors or the + # process is killed. Not gated on approval-status, which reads denied on an API error too. + - name: Close the issue if it is still open + if: always() + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ github.run_id }} + REPO: ${{ github.repository }} + ISSUE_NUMBER: ${{ steps.approval.outputs.issue-number }} + run: | + set -euo pipefail + + # The output first: the search index lags a new issue, so an action erroring right after + # opening one would leave it orphaned. Search is the fallback when no output was set. + number="${ISSUE_NUMBER:-}" + if [ -z "$number" ]; then + number="$(gh issue list --repo "$REPO" --label secret-approval --state open \ + --search "run $RUN_ID in:title" --json number --jq '.[0].number // empty')" + fi + + if [ -z "$number" ]; then + echo "no open approval issue left for run $RUN_ID, nothing to clean up" + exit 0 + fi + + # The search index lags, so confirm against the issue itself before writing to it. + state="$(gh issue view "$number" --repo "$REPO" --json state --jq '.state')" + if [ "$state" != "OPEN" ]; then + echo "issue #$number is already $state, nothing to clean up" + exit 0 + fi + + # Built in a variable rather than inline, so the YAML indentation of a wrapped string + # does not end up inside the comment. + msg="Closed automatically: the job ended with no decision recorded on this issue." + msg="$msg **Nothing was written.** Dispatch the workflow again if the write is still" + msg="$msg wanted." + + gh issue comment "$number" --repo "$REPO" --body "$msg" + gh issue close "$number" --repo "$REPO" + echo "closed approval issue #$number, which no decision had been recorded on" + + # The only job that writes, and the only one holding a credential that can. + write: + name: Add ${{ inputs.key }} to ${{ inputs.project }}-${{ inputs.environment }} + needs: [preflight, approval] + environment: ${{ inputs.write-environment }} + # Read-merge-write is not atomic and there is no compare-and-swap, so overlapping runs would + # both read the same version. Not at workflow level, where the lock would span the approval wait. + concurrency: + group: add-manual-secret-${{ inputs.project }}-${{ inputs.environment }} + cancel-in-progress: false + runs-on: ${{ vars.RUNNER_INFRA || 'ubuntu-latest' }} + permissions: {} + # A hung run holds the concurrency lock and blocks every run behind it. + timeout-minutes: 5 + steps: + # Repeated from preflight, not redundant: a mask is registered per job, and this is a + # different job on a different runner. Must be first here too. + - name: Mask the value + run: | + set -euo pipefail + + # Without this, a caller naming the input differently masks nothing and still succeeds. + if ! jq -e 'has("inputs") and (.inputs | has("value"))' "$GITHUB_EVENT_PATH" >/dev/null; then + echo "::error::the dispatch input holding the value must be named 'value'," \ + "see Docs/infra-secret-write.md" + exit 1 + fi + + jq -r '.inputs.value // empty' "$GITHUB_EVENT_PATH" \ + | while IFS= read -r line; do + if [ -n "$line" ]; then + echo "::add-mask::$line" + fi + done + + - name: Write + id: write + uses: wisemen-digital/devops-github-actions/.github/actions/infra/secrets/write@v1 + with: + vendor: ${{ inputs.vendor }} + secret-name: ${{ needs.preflight.outputs.secret-name }} + project-name: ${{ needs.preflight.outputs.project-name }} + key: ${{ inputs.key }} + value: ${{ secrets.SECRET_VALUE }} + description: >- + added ${{ inputs.key }} via GitHub run ${{ github.run_id }} by ${{ github.actor }} + scaleway-access-key: ${{ secrets.SCALEWAY_ACCESS_KEY }} + scaleway-secret-key: ${{ secrets.SCALEWAY_SECRET_KEY }} + scaleway-organization-id: ${{ vars.SCALEWAY_ORGANIZATION_ID }} + scaleway-project-id: ${{ vars.SCALEWAY_PROJECT_ID }} + scaleway-region: ${{ vars.SCALEWAY_REGION || 'nl-ams' }} + + - name: Summary + if: always() + env: + SECRET_NAME: ${{ needs.preflight.outputs.secret-name }} + ENVIRONMENT: ${{ inputs.environment }} + KEY: ${{ inputs.key }} + STATUS: ${{ job.status }} + LOG: ${{ steps.write.outputs.log }} + run: | + set -euo pipefail + + { + echo "## Manual secret update" + echo "" + echo "| | |" + echo "| --- | --- |" + echo "| Secret | \`${SECRET_NAME:-unresolved}\` |" + echo "| Environment | $ENVIRONMENT |" + echo "| Key | \`$KEY\` |" + echo "| Status | $STATUS |" + echo "" + } >> "$GITHUB_STEP_SUMMARY" + + # Only the script's own summary lines, never the whole log: a failure can carry API + # response text, and this page is more widely read than the log. + keep='^(added|overwritten|already identical|carried over|result)' + keep="$keep"'|^(wrote revision|wrote a new version|nothing would change)' + keep="$keep"'|^secret has no versions|^warning: all [0-9]+ version' + keep="$keep"'|^warning: current version of .* not valid env var names' + + if [ -n "${LOG:-}" ] && [ -f "$LOG" ]; then + { + echo '```' + grep -E "$keep" "$LOG" \ + || echo "(no summary produced, see the job log)" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + fi diff --git a/Docs/infra-secret-write.md b/Docs/infra-secret-write.md new file mode 100644 index 0000000..45c647c --- /dev/null +++ b/Docs/infra-secret-write.md @@ -0,0 +1,182 @@ +# Workflow: infra secret write + +## Description + +Adds or overwrites **one key** in an application's manual secret, without losing the other keys it +already holds, behind a human approval gate. + +The secret is `--secrets-manual`, created empty by Terraform with +`prevent_destroy` and managed by hand from then on. External Secrets Operator rebuilds the +Kubernetes secret from its sources on every sync, so a write that dropped a key would take that +variable out of the running pods. The workflow therefore reads the current version, merges the one +new key into it, and writes the result as a new version. + +Three jobs: + +1. **preflight** validates the key and the value, resolves the project and the secret, and proves + the write can succeed. Nothing is written and nobody is asked to approve a request that cannot + work. +2. **approval** opens an issue and waits for one of the listed approvers to comment. This stands in + for GitHub's required reviewers, which need Enterprise on a private repository. The job holds no + vendor credential and declares no environment, so the only thing it can reach is its own issue. +3. **write** does the read-merge-write. It is the only job holding a credential that can write. + +The gate stops mistakes, not people with push access: anyone who can dispatch the workflow can also +change its definition. Point `write-environment` at an environment with a protected-branch policy, +which refuses the write job unless the ref is a protected branch. What is left depends on that +branch's own protection rules. + +## Caller requirements + +Two things are not optional. + +**The dispatch input holding the value must be named `value`.** The mask step reads it out of the +event payload, because putting it in a step's `env:` would print it to the log before the mask +exists. + +**Secrets must be mapped explicitly, not inherited.** `SECRET_VALUE` is not a repository secret, so +`secrets: inherit` cannot supply it. + +```yaml +name: Add manual secret + +on: + workflow_dispatch: + inputs: + project: + description: Application + type: choice + required: true + options: + - my-app + environment: + description: Environment + type: choice + required: true + default: development + options: + - development + - staging + - production + key: + description: Variable name, e.g. STRIPE_SECRET_KEY + type: string + required: true + value: + description: Secret value + type: string + required: true + +permissions: {} + +jobs: + add-manual-secret: + uses: wisemen-digital/devops-github-actions/.github/workflows/infra-secret-write.yml@v1 + with: + vendor: scaleway + project: ${{ inputs.project }} + environment: ${{ inputs.environment }} + key: ${{ inputs.key }} + approvers: alice,bob + secrets: + SECRET_VALUE: ${{ inputs.value }} + SCALEWAY_ACCESS_KEY: ${{ secrets.SCALEWAY_ACCESS_KEY }} + SCALEWAY_SECRET_KEY: ${{ secrets.SCALEWAY_SECRET_KEY }} +``` + +## Inputs + +### Common Inputs + +| Input | Description | Required | +| ----- | ----------- | -------- | +| `vendor` | The vendor to communicate with. Only `scaleway` is supported, see Vendor support below | Yes | +| `project` | Application name, the prefix of the vendor project | Yes | +| `environment` | Environment name, the suffix of the vendor project | Yes | +| `key` | Variable name to add or overwrite. Letters, digits and underscores, not starting with a digit | Yes | +| `approvers` | Comma-separated GitHub logins allowed to approve. Usernames, never a team slug | Yes | +| `write-environment` | Environment the write job runs in, for its branch policy. Defaults to `secret-write-approval` | No | + +`approvers` must be usernames. The approval action resolves each entry as a team slug before falling +back to a username, so a team works for approving but the audit record cannot name who decided, and +reporting that correctly would need an org-read token this workflow deliberately does not hold. + +### Variables & Secrets + +| Name | Description | Type | Required | +| ---- | ----------- | ---- | -------- | +| `SECRET_VALUE` | The value to store. Minimum 8 characters | Secret | Yes | +| `RUNNER_INFRA` | The CI runner for infra actions. Defaults to `ubuntu-latest` | Variable | No | + +### Vendor-Specific Inputs + +Only provide the following for your chosen vendor. + +#### Scaleway + +
+Click to expand Scaleway-specific inputs + +| Name | Description | Type | Required | +| ---- | ----------- | ---- | -------- | +| `SCALEWAY_ACCESS_KEY` | Scaleway API key ID | Secret | Yes | +| `SCALEWAY_SECRET_KEY` | Scaleway API key secret | Secret | Yes | +| `SCALEWAY_ORGANIZATION_ID` | Scaleway organization ID | Variable | Yes | +| `SCALEWAY_PROJECT_ID` | Scaleway project ID for the CLI profile. The organization's `default` project, not the app's | Variable | Yes | +| `SCALEWAY_REGION` | Scaleway region. Defaults to `nl-ams` | Variable | No | + +
+ +## Vendor support + +Only `scaleway` works. The other two are blocked by the vendors, not by this code: + +- **DigitalOcean** has no managed secret manager, and External Secrets Operator has no DigitalOcean + provider, so there is nothing to write to. +- **Azure Key Vault** holds one value per secret, so there is nothing to merge. Supporting it means a + different algorithm, not a different backend. + +Both are rejected explicitly rather than falling through to `Unsupported vendor`, so the message +says why. + +### Adding a vendor + +Both actions follow `infra/common/setup`: a `case` listing the implemented vendors, then that +vendor's steps behind `if: ${{ inputs.vendor == '' }}`. Add the arm and the steps in the same +change, or the vendor skips every step and reports success having written nothing. + +Already neutral: the key must be a valid environment variable name, the minimum value length, and the +value-inside-key check. The workflow itself is neutral throughout. + +The vendor's: resolving the target name, the value-inside-target-name check, the CLI setup, and +proving the target exists. A vendor with stricter naming adds that check too, Key Vault allowing no +underscores where this key rule requires them. + +Every output of `secrets/preflight` reads a Scaleway step id, so a second vendor's outputs come back +empty until each one falls back across both: `${{ steps.a.outputs.x || steps.b.outputs.x }}`. + +## Validation + +The write is refused before anything happens if: + +- the key is not a valid environment variable name. It reaches the pod through `envFrom`, which + silently skips an invalid name +- the value is shorter than 8 characters. Masking is literal replacement, so a short value is + recoverable from its own redaction +- the value occurs inside the key or the secret name, for the same reason +- the vendor project does not exist. The error lists the environments that do +- the secret does not exist, is not `key_value`, or more than one secret shares its name + +If the secret exists but every version of it is disabled, the write proceeds from an empty object +and the approver is warned in the issue and in both summaries, because the disabled versions survive +and can be re-enabled. + +## Concurrency + +The write job takes a lock per project and environment, with `cancel-in-progress: false`. +Read-merge-write is not atomic and there is no compare-and-swap, so two overlapping runs would both +read the same version and the second would discard the first key. + +The lock is on the job rather than the workflow, so it is held for the seconds the write takes and +not for the hour the approval may wait. Note that GitHub queues only one pending run per group, so a +third concurrent request cancels the second while it is pending. diff --git a/README.md b/README.md index 01376ba..49cd6f5 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ Infra deployments: | -------- | ----------- | | [infra-k8s-rollout.yml](Docs/infra-k8s-rollout.md) | Apply k8s configuration for the given deployments | | [infra-s3-rollout.yml](Docs/infra-s3-rollout.md) | Apply s3 configuration to the given buckets | +| [infra-secret-write.yml](Docs/infra-secret-write.md) | Add or overwrite one key in an application's manual secret | Verification (PR check):