From a79d84ec08c50bbb8a0fe5e6e9db3ad438ad7ad9 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 13 Oct 2026 07:22:00 +0200 Subject: [PATCH 1/3] deploy: run ftwcloudd at cloud.ftw.energy on one t4g.small in Stockholm bootstrap.sh brings up an Amazon Linux 2023 host through SSM: swap, core dumps and sshd off, Docker as on the relay host, and ftw-cloud-deploy, which builds the checked-out commit and starts a new container. The service key is made on the host and copied to tmpfs before Docker starts. deploy/README.md records the running host: ids, how to update, check and tear it down, and the cost. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 +- README.md | 4 +- deploy/README.md | 159 +++++++++++++++++++++++++++++++++++++------- deploy/bootstrap.sh | 134 +++++++++++++++++++++++++++++++++++++ 4 files changed, 274 insertions(+), 25 deletions(-) create mode 100644 deploy/bootstrap.sh diff --git a/CLAUDE.md b/CLAUDE.md index 5f88b21..9f0971f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -35,7 +35,7 @@ maintains it. - Keep the service small. The budget is about 3,000 lines of Go; tiles go first if it is overrun. - Nothing is deployed from here without the owner. `deploy/` describes the - intended host; it does not run itself. + running host; it does not run itself. - Keep task breakdowns and agent notes out of the repository. ## Check diff --git a/README.md b/README.md index cbd5fca..44d20dc 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,9 @@ This is `ftwcloudd`, the service. The decision is [ADR 0008](https://github.com/srcfl/ftw/blob/master/docs/adr/0008-ftw-cloud.md) and the wire format is the normative [FTW Cloud protocol](https://github.com/srcfl/ftw/blob/master/docs/protocol-cloud.md), -both in `srcfl/ftw`. Both are proposed; nothing here is deployed. +both in `srcfl/ftw`. Both are proposed. The service runs at +`cloud.ftw.energy` ([deploy/README.md](deploy/README.md)), but no box sends +to it yet. ## What works today (C2) diff --git a/deploy/README.md b/deploy/README.md index 97fe1ba..0788f0f 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1,39 +1,152 @@ # Running FTW Cloud -Proposed, not deployed. ADR 0008 in `srcfl/ftw` chose one `t4g.small` in AWS -eu-north-1 (Stockholm), in an AWS account of its own, apart from the relay and -escrow. The account is not decided yet; nothing here creates anything. +FTW Cloud runs on one `t4g.small` in AWS eu-north-1 (Stockholm) and answers +at `https://cloud.ftw.energy`. No box sends to it yet: release builds of Core +name no service, and a home can turn FTW Cloud on only from a Core built with +the uplink (srcfl/ftw#1595), after its owner decides to. + +## The host + +| What | Value | +|---|---| +| Account | `976075258475`, shared with other Sourceful work. FTW Cloud moves to an account of its own when it has real users (ADR 0008). | +| Region | eu-north-1, zone eu-north-1a, default VPC | +| Instance | `i-0ecac4cbc6b5f8adc`, `t4g.small`, Amazon Linux 2023 arm64, IMDSv2 required, no key pair | +| Disk | `vol-0760cad0e8551e5e4`, 20 GB gp3, encrypted with `aws/ebs` | +| Elastic IP | `13.50.84.149` (`eipalloc-0085f3e68b6f36da4`) | +| Security group | `sg-098992ad382ecf94a` `ftw-cloud`: 443/tcp from anywhere, nothing else in | +| Instance role | `ftw-cloud-host`, with `AmazonSSMManagedInstanceCore` and nothing else | +| DNS | `cloud.ftw.energy` A `13.50.84.149` in the Cloudflare zone `ftw.energy`, grey cloud (record `22d28943eea9f6b8584214e1a8256ef1`) | +| Branch | `main`, checked out in `/opt/ftw-cloud` | +| Service public key | `BG6EvBxFzSHPVsQOES6PYaKFo5vHQmte9lJ6D5OBIT9JgcBrJAOxAucdqKXxS76pToQ7orI1Ivsno0OtxbMuliQ` | + +Every resource carries the tag `Project=ftw-cloud`. ## Shape - One instance, one container (`compose.yml`), no standby. A rebuilt host starts empty and boxes fill it again when they reconnect. -- `ftwcloudd` ends TLS 1.3 itself and gets its certificate by ACME - (TLS-ALPN-01 on 443). **No TLS-terminating proxy in front**: box and service - bind the handshake to the TLS session, so Caddy, an ALB or an orange - Cloudflare cloud would break every box's connection, on purpose. An NLB in - TCP mode is fine. DNS for `cloud.ftw.energy` is a grey-cloud A record. -- The service key comes from Secrets Manager into `/run/ftw-cloud/service.key` - (tmpfs) at boot. Its public half is compiled into Core - (`cloudlink.ServiceKeys`); Core carries up to two, so it can rotate across a - release. -- The volume holds the SQLite registry (home ids, box keys, signed rosters, - tombstones) and the ACME cache. No readings. Use an encrypted EBS volume. +- `ftwcloudd` ends TLS 1.3 itself and gets its certificate from Let's Encrypt + by TLS-ALPN-01 on 443, so port 80 stays shut. **No TLS-terminating proxy in + front**: box and service bind the handshake to the TLS session, so Caddy, + an ALB or an orange Cloudflare cloud would break every box's connection, on + purpose. An NLB in TCP mode is fine. That is also why the DNS record is + grey. +- The host builds the image from its checkout and tags it with the commit + (`ftw-cloud:<12 hex>`); `/etc/ftw-cloud/compose.env` names it for + `compose.yml`. Images built in CI and deployed by digest come later. +- The Docker volume `ftw-cloud_ftw-cloud-state` holds the SQLite registry + (home ids, box keys, signed rosters, tombstones) and the ACME cache, on the + encrypted root disk. No readings. No backup. + +## The service key + +`ftw-cloud-deploy` makes the key on the host the first time it runs +(`ftwcloudd keygen`) and keeps it at `/var/lib/ftw-cloud/service.key`, root +only, on the encrypted disk. At boot, `ftw-cloud-key.service` copies it to +`/run/ftw-cloud/service.key` (tmpfs) before Docker starts, and `compose.yml` +mounts that copy. + +The key never leaves the host, and nothing backs it up. Its public half, +above, goes into Core's `cloudlink.ServiceKeys`. A new host makes a new key, +and Core then needs a release that carries it; Core carries up to two keys so +it can rotate across a release. + +ADR 0008 keeps the key in Secrets Manager. That needs a role that may read a +secret, and this host's role may only talk to SSM. Move the key to Secrets +Manager before a Core release carries its public half. ## Host rules -- Swap off (`swapoff -a`, no swap in fstab) and core dumps off (the compose - file sets `ulimits: core: 0`; `ftwcloudd` also marks itself not dumpable). +- Swap off (`swapoff -a`, no swap in fstab) and core dumps off: the kernel + pipes cores to `/bin/false`, systemd-coredump stores nothing, `compose.yml` + sets `ulimits: core: 0`, and `ftwcloudd` marks itself not dumpable. Household readings exist only in this process's memory. -- Security group: 443 from anywhere. Nothing else; administration is through - SSM, with every session logged to a separate account. -- Outbound only to AWS endpoints, Let's Encrypt and, from C4, APNs. -- Deploy by image digest, built in CI from a reviewed commit. The running - build's source is at `https://cloud.ftw.energy/source`. +- Nothing comes in but 443. There is no SSH and no key pair; administration + is through SSM. Session logs to a separate account are not set up yet. +- Outbound is open, because the host builds its own image (Docker Hub, + gcr.io, the Go module proxy, GitHub). Narrow it to AWS endpoints, Let's + Encrypt and, from C4, APNs once images come from CI. +- The running build's source is at `https://cloud.ftw.energy/source`. + +## Bringing it up + +`bootstrap.sh` is the whole host. It is idempotent: it turns off swap and +core dumps, installs Docker, Compose and buildx as the relay host does, +clones this repository to `/opt/ftw-cloud`, installs the two systemd units and +`/usr/local/sbin/ftw-cloud-deploy`, and runs that. + +```bash +aws ssm send-command --region eu-north-1 --instance-ids i-0ecac4cbc6b5f8adc \ + --document-name AWS-RunShellScript \ + --parameters "$(jq -n --rawfile s deploy/bootstrap.sh '{commands: [$s]}')" +``` + +The DNS record must exist before the first TLS connection, since that is when +`ftwcloudd` asks Let's Encrypt for a certificate. + +## Updating + +```bash +aws ssm send-command --region eu-north-1 --instance-ids i-0ecac4cbc6b5f8adc \ + --document-name AWS-RunShellScript \ + --parameters 'commands=["/usr/local/sbin/ftw-cloud-deploy"]' +``` + +or, in a session (`aws ssm start-session --region eu-north-1 --target +i-0ecac4cbc6b5f8adc`), `sudo ftw-cloud-deploy`. It pulls the checked-out branch, +builds, and starts a new container. Every box reconnects within seconds. + +It always makes a new container, even when only a mounted file changed. The +relay ran into this: a file mounted on its own, like the service key, keeps +the copy the container started with after the file changes on the host. + +## Checking it + +```bash +curl -s https://cloud.ftw.energy/healthz +``` + +`ok` means DNS, the certificate and the service all work. +`https://cloud.ftw.energy/source` returns the source archive. +`/v1/app` and `/v1/now` answer 501 until C3 and C5. + +## Cost + +eu-north-1 on-demand prices, October 2026: + +| Item | Price | Per month | +|---|---|---| +| `t4g.small` | $0.0172 an hour | $12.56 | +| 20 GB gp3 | $0.0836 a GB-month | $1.67 | +| Public IPv4 (the Elastic IP) | $0.005 an hour | $3.65 | +| **Total** | | **about $18** | + +Traffic from boxes comes in, which AWS does not charge for. Traffic out to +phones (C3) is free for the first 100 GB a month across the account. SSM costs +nothing. CPU credits run unlimited, which costs extra only if the CPU averages +more than 20% over a day. + +## Tearing it down + +```bash +cf dns records delete 22d28943eea9f6b8584214e1a8256ef1 -z ftw.energy +aws ec2 terminate-instances --region eu-north-1 --instance-ids i-0ecac4cbc6b5f8adc +aws ec2 wait instance-terminated --region eu-north-1 --instance-ids i-0ecac4cbc6b5f8adc +aws ec2 release-address --region eu-north-1 --allocation-id eipalloc-0085f3e68b6f36da4 +aws ec2 delete-security-group --region eu-north-1 --group-id sg-098992ad382ecf94a +aws iam remove-role-from-instance-profile --instance-profile-name ftw-cloud-host --role-name ftw-cloud-host +aws iam delete-instance-profile --instance-profile-name ftw-cloud-host +aws iam detach-role-policy --role-name ftw-cloud-host \ + --policy-arn arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore +aws iam delete-role --role-name ftw-cloud-host +``` + +The disk goes with the instance, and with it the registry and the service +key. Nothing else holds FTW Cloud data. ## Not here yet APNs (C4), the app session (C3), widget tokens (C5) and statistics (C7–C8) add secrets, egress and storage when they land. The erase test, canary homes -and the external heartbeat that pages Sourceful belong with the first -deployment. +and the external heartbeat that pages Sourceful belong with the first homes. diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh new file mode 100644 index 0000000..764517c --- /dev/null +++ b/deploy/bootstrap.sh @@ -0,0 +1,134 @@ +#!/bin/bash +# Bring up an FTW Cloud host. Amazon Linux 2023, arm64, run as root through +# SSM. Safe to re-run: everything below is idempotent, and a re-run is also +# how the host's own tooling gets updated. +set -euxo pipefail + +export HOME=${HOME:-/root} + +# Readings live only in ftwcloudd's memory. No swap may page them to disk and +# no core file may carry them out. +swapoff -a +sed -i '/\sswap\s/d' /etc/fstab +cat > /etc/sysctl.d/99-ftw-cloud.conf <<'SYSCTL' +kernel.core_pattern = |/bin/false +fs.suid_dumpable = 0 +SYSCTL +sysctl --system >/dev/null +install -d /etc/systemd/coredump.conf.d +printf '[Coredump]\nStorage=none\nProcessSizeMax=0\n' > /etc/systemd/coredump.conf.d/ftw-cloud.conf +printf '* hard core 0\n' > /etc/security/limits.d/ftw-cloud.conf + +# No SSH, not even behind the security group. Administration is through SSM. +systemctl disable --now sshd.service + +# Docker, Compose and buildx, as the relay host does it. +dnf install -y docker git +systemctl enable --now docker + +install -d /usr/local/lib/docker/cli-plugins +case "$(uname -m)" in + aarch64) PLUGIN_ARCH=arm64 ; COMPOSE_ARCH=aarch64 ;; + *) PLUGIN_ARCH=amd64 ; COMPOSE_ARCH=x86_64 ;; +esac +curl -fsSL "https://github.com/docker/compose/releases/latest/download/docker-compose-linux-${COMPOSE_ARCH}" \ + -o /usr/local/lib/docker/cli-plugins/docker-compose +chmod +x /usr/local/lib/docker/cli-plugins/docker-compose +# sed reads to the end. head or grep -m1 closes the pipe early, and under +# pipefail that fails the script. +BUILDX_URL=$(curl -fsS https://api.github.com/repos/docker/buildx/releases/latest \ + | grep -o "https://[^\"]*linux-${PLUGIN_ARCH}" | sed -n 1p) +curl -fsSL "$BUILDX_URL" -o /usr/local/lib/docker/cli-plugins/docker-buildx +chmod +x /usr/local/lib/docker/cli-plugins/docker-buildx +docker compose version +docker buildx version + +if [ ! -d /opt/ftw-cloud ]; then + git clone https://github.com/srcfl/ftw-cloud /opt/ftw-cloud +fi + +install -d -m 0700 /var/lib/ftw-cloud /etc/ftw-cloud + +# The service key rests on the encrypted root volume, readable by root only. +# At boot, before Docker starts, a copy goes to tmpfs, where compose.yml +# mounts it into the container. +cat > /etc/systemd/system/ftw-cloud-key.service <<'UNIT' +[Unit] +Description=FTW Cloud service key into tmpfs +Before=docker.service +ConditionPathExists=/var/lib/ftw-cloud/service.key + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/bin/install -d -m 0700 /run/ftw-cloud +ExecStart=/usr/bin/install -m 0400 -o 65532 -g 65532 /var/lib/ftw-cloud/service.key /run/ftw-cloud/service.key + +[Install] +WantedBy=docker.service +UNIT + +# Come back after a reboot without anyone logging in. +cat > /etc/systemd/system/ftw-cloud.service <<'UNIT' +[Unit] +Description=FTW Cloud +Requires=docker.service ftw-cloud-key.service +After=docker.service ftw-cloud-key.service network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes +WorkingDirectory=/opt/ftw-cloud/deploy +ExecStart=/usr/bin/docker compose --env-file /etc/ftw-cloud/compose.env up -d +ExecStop=/usr/bin/docker compose --env-file /etc/ftw-cloud/compose.env down + +[Install] +WantedBy=multi-user.target +UNIT + +# One command updates the host: pull, build, run. The checkout's branch +# decides what runs. +cat > /usr/local/sbin/ftw-cloud-deploy <<'DEPLOY' +#!/bin/bash +# Build the commit checked out in /opt/ftw-cloud and run it. Every box +# reconnects within seconds; there is nothing to drain. +set -euo pipefail +export HOME=${HOME:-/root} +cd /opt/ftw-cloud +git pull --ff-only +rev=$(git rev-parse --short=12 HEAD) +docker build --pull -t "ftw-cloud:$rev" . + +# The key is made once, on this host, and never leaves it. Its public half +# is what Core carries in cloudlink.ServiceKeys. +if [ ! -s /var/lib/ftw-cloud/service.key ]; then + docker run --rm --user 0:0 --network none -v /var/lib/ftw-cloud:/out \ + "ftw-cloud:$rev" keygen -out /out/service.key +fi +systemctl enable ftw-cloud-key.service ftw-cloud.service +systemctl restart ftw-cloud-key.service + +printf 'FTW_CLOUD_IMAGE=ftw-cloud:%s\n' "$rev" > /etc/ftw-cloud/compose.env +cd deploy +# Always a new container. A file mounted on its own, such as the key, keeps +# the copy the container started with even after the file on the host +# changes; only a new container reads it again. +docker compose --env-file /etc/ftw-cloud/compose.env up -d --force-recreate + +for img in $(docker image ls ftw-cloud --format '{{.Repository}}:{{.Tag}}'); do + [ "$img" = "ftw-cloud:$rev" ] || docker image rm "$img" +done +docker image prune -f +docker builder prune -f --filter until=168h +echo "ftw-cloud:$rev running" +DEPLOY +chmod 0755 /usr/local/sbin/ftw-cloud-deploy + +systemctl daemon-reload +/usr/local/sbin/ftw-cloud-deploy + +# Show that the host rules hold. +swapon --show +sysctl kernel.core_pattern +docker compose --env-file /etc/ftw-cloud/compose.env -f /opt/ftw-cloud/deploy/compose.yml ps From 0d4e5da2345a30f0553e5bf3bf3ac42a44350f4e Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 13 Oct 2026 07:50:53 +0200 Subject: [PATCH 2/3] deploy: mask sshd, since cloud-init starts it at every boot A reboot test showed sshd listening on 22 again: cloud-init.service wants it, so disabling the unit is not enough. Masking it keeps it off across reboots. The security group already dropped 22. Co-Authored-By: Claude Opus 5.5 --- deploy/bootstrap.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index 764517c..117314b 100644 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -20,7 +20,8 @@ printf '[Coredump]\nStorage=none\nProcessSizeMax=0\n' > /etc/systemd/coredump.co printf '* hard core 0\n' > /etc/security/limits.d/ftw-cloud.conf # No SSH, not even behind the security group. Administration is through SSM. -systemctl disable --now sshd.service +# cloud-init wants sshd at every boot, so disabling it is not enough. +systemctl mask --now sshd.service # Docker, Compose and buildx, as the relay host does it. dnf install -y docker git From 541bc772e28525e22e941b31b339781a253aec01 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 13 Oct 2026 07:52:56 +0200 Subject: [PATCH 3/3] deploy: run images CI built, by digest, with the service key in Secrets Manager ADR 0008 asks for images built in CI from a reviewed commit, deployed by digest, and for the service key in Secrets Manager. Until now the host built its own image and kept the key on its disk. - The Image workflow builds linux/arm64 from every commit on main, pushes it to the public ghcr.io/srcfl/ftw-cloud and attests its provenance. A pull request builds and pushes nothing. - The Dockerfile cross-compiles, so CI needs no arm64 emulation, and passes the commit in with -ldflags, since .git stays out of the context. - /source names the commit (X-Source-Revision) and the running image digest (X-Source-Image), as ADR 0008 asks. - ftw-cloud-deploy takes a digest, refuses an image whose commit is not on main, and checks out that commit for compose.yml. It builds nothing and makes no key. - ftw-cloud-key fetches the key from Secrets Manager into tmpfs at boot. The role gets one inline policy: GetSecretValue on that one secret. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/image.yml | 70 ++++++++++++++++++ Dockerfile | 12 +++- README.md | 3 +- deploy/README.md | 118 +++++++++++++++++++++--------- deploy/bootstrap.sh | 126 +++++++++++++++++++++------------ deploy/compose.yml | 7 +- deploy/host-policy.json | 11 +++ internal/server/server.go | 12 ++++ internal/server/server_test.go | 20 ++++++ 9 files changed, 296 insertions(+), 83 deletions(-) create mode 100644 .github/workflows/image.yml create mode 100644 deploy/host-policy.json diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml new file mode 100644 index 0000000..3cd5ade --- /dev/null +++ b/.github/workflows/image.yml @@ -0,0 +1,70 @@ +name: Image + +# Builds the host's image (linux/arm64) from every commit on main and pushes +# it to ghcr.io/srcfl/ftw-cloud, a public package the host pulls without a +# credential. A pull request builds the image and pushes nothing. Nothing +# here deploys: the owner picks a digest from a run on main and runs +# ftw-cloud-deploy with it (deploy/README.md). + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + attestations: write + env: + PUSH: ${{ github.event_name == 'push' }} + steps: + - uses: actions/checkout@v5 + - uses: docker/setup-buildx-action@v4 + - if: env.PUSH == 'true' + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - id: build + uses: docker/build-push-action@v7 + with: + context: . + platforms: linux/arm64 + push: ${{ env.PUSH == 'true' }} + tags: ghcr.io/srcfl/ftw-cloud:${{ github.sha }} + build-args: REVISION=${{ github.sha }} + # ftw-cloud-deploy reads the revision label to check out the + # matching compose.yml and to refuse an image built off main. + labels: | + org.opencontainers.image.source=https://github.com/srcfl/ftw-cloud + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.licenses=AGPL-3.0-only + # One plain manifest, so the digest names exactly one image. The + # provenance goes to GitHub's attestation store below. + provenance: false + sbom: false + # Anyone can check that a digest came from this workflow on a commit + # in this repository: gh attestation verify oci:// -R srcfl/ftw-cloud + - if: env.PUSH == 'true' + uses: actions/attest-build-provenance@v4 + with: + subject-name: ghcr.io/srcfl/ftw-cloud + subject-digest: ${{ steps.build.outputs.digest }} + - if: env.PUSH == 'true' + name: The digest to deploy + env: + DIGEST: ${{ steps.build.outputs.digest }} + run: | + { + echo "Built \`${GITHUB_SHA}\` as \`ghcr.io/srcfl/ftw-cloud@${DIGEST}\`." + echo + echo "Deploy it with \`ftw-cloud-deploy ${DIGEST}\` (deploy/README.md)." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/Dockerfile b/Dockerfile index 144f538..7d05a26 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,13 +4,21 @@ # AGPL-3.0 asks that everyone who uses the service over a network can get the # code running it. -FROM golang:1.26-alpine AS build +# Go cross-compiles, so CI builds the arm64 image on any runner without +# emulating arm64. +FROM --platform=$BUILDPLATFORM golang:1.26-alpine AS build +ARG TARGETOS TARGETARCH +# The commit, for X-Source-Revision. CI passes it; .git stays out of the +# build context, so Go cannot read it itself. +ARG REVISION= WORKDIR /src COPY go.mod go.sum ./ RUN go mod download COPY . . # No cgo: the SQLite driver is pure Go, so the binary runs on a static base. -RUN CGO_ENABLED=0 go build -trimpath -ldflags "-s -w" -o /out/ftwcloudd ./cmd/ftwcloudd +RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -trimpath \ + -ldflags "-s -w -X github.com/srcfl/ftw-cloud/internal/server.Revision=${REVISION}" \ + -o /out/ftwcloudd ./cmd/ftwcloudd RUN mkdir -p /out/share && tar -czf /out/share/source.tar.gz \ --exclude=./.git --exclude=./state . \ && mkdir -p /out/state diff --git a/README.md b/README.md index 44d20dc..e00d613 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,8 @@ to it yet. never written to disk. - The registry (SQLite) holds the home id, the box key and the signed roster. A box that turns FTW Cloud off gets its home deleted and a signed receipt. -- `/source` serves the source of the running build, as AGPL-3.0 asks. +- `/source` serves the source of the running build, as AGPL-3.0 asks, and + names its commit and image digest. Not built yet, with a marked place in the code: diff --git a/deploy/README.md b/deploy/README.md index 0788f0f..c678e7d 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -15,12 +15,13 @@ the uplink (srcfl/ftw#1595), after its owner decides to. | Disk | `vol-0760cad0e8551e5e4`, 20 GB gp3, encrypted with `aws/ebs` | | Elastic IP | `13.50.84.149` (`eipalloc-0085f3e68b6f36da4`) | | Security group | `sg-098992ad382ecf94a` `ftw-cloud`: 443/tcp from anywhere, nothing else in | -| Instance role | `ftw-cloud-host`, with `AmazonSSMManagedInstanceCore` and nothing else | +| Instance role | `ftw-cloud-host`, with `AmazonSSMManagedInstanceCore` and the inline policy `ftw-cloud-service-key` ([`host-policy.json`](host-policy.json)): read one secret, nothing else | +| Secret | `ftw-cloud/service-key` in Secrets Manager, eu-north-1, encrypted with `aws/secretsmanager` | | DNS | `cloud.ftw.energy` A `13.50.84.149` in the Cloudflare zone `ftw.energy`, grey cloud (record `22d28943eea9f6b8584214e1a8256ef1`) | -| Branch | `main`, checked out in `/opt/ftw-cloud` | +| Image | `ghcr.io/srcfl/ftw-cloud`, public, run by digest; `/etc/ftw-cloud/compose.env` names the running one | | Service public key | `BG6EvBxFzSHPVsQOES6PYaKFo5vHQmte9lJ6D5OBIT9JgcBrJAOxAucdqKXxS76pToQ7orI1Ivsno0OtxbMuliQ` | -Every resource carries the tag `Project=ftw-cloud`. +Every AWS resource carries the tag `Project=ftw-cloud`. ## Shape @@ -32,29 +33,37 @@ Every resource carries the tag `Project=ftw-cloud`. an ALB or an orange Cloudflare cloud would break every box's connection, on purpose. An NLB in TCP mode is fine. That is also why the DNS record is grey. -- The host builds the image from its checkout and tags it with the commit - (`ftw-cloud:<12 hex>`); `/etc/ftw-cloud/compose.env` names it for - `compose.yml`. Images built in CI and deployed by digest come later. +- CI builds the image. The `Image` workflow + ([`image.yml`](../.github/workflows/image.yml)) builds `linux/arm64` from + every commit on `main`, pushes it to `ghcr.io/srcfl/ftw-cloud` tagged with + the commit, and records its provenance in GitHub's attestation store. The + host builds nothing and needs no credential to pull. +- The host runs one image, named by its digest, never by a tag. + `/opt/ftw-cloud` is checked out at the commit that image was built from and + supplies `compose.yml`. - The Docker volume `ftw-cloud_ftw-cloud-state` holds the SQLite registry (home ids, box keys, signed rosters, tombstones) and the ACME cache, on the encrypted root disk. No readings. No backup. ## The service key -`ftw-cloud-deploy` makes the key on the host the first time it runs -(`ftwcloudd keygen`) and keeps it at `/var/lib/ftw-cloud/service.key`, root -only, on the encrypted disk. At boot, `ftw-cloud-key.service` copies it to -`/run/ftw-cloud/service.key` (tmpfs) before Docker starts, and `compose.yml` -mounts that copy. +The key lives in Secrets Manager as `ftw-cloud/service-key`, in PEM. At boot, +before Docker starts, `ftw-cloud-key.service` runs +`/usr/local/sbin/ftw-cloud-key`, which fetches it into +`/run/ftw-cloud/service.key` (tmpfs, owned by the container's user, mode +`0400`), and `compose.yml` mounts that file. The key is never on disk and +never in the image. `ftw-cloud-deploy` fetches it again, so a new version +takes effect on the next deploy or reboot. -The key never leaves the host, and nothing backs it up. Its public half, -above, goes into Core's `cloudlink.ServiceKeys`. A new host makes a new key, -and Core then needs a release that carries it; Core carries up to two keys so -it can rotate across a release. +The role may read this one secret and write nothing. A rebuilt host reads the +same key, so Core needs no new release for it. The public half, above, goes +into Core's `cloudlink.ServiceKeys`. -ADR 0008 keeps the key in Secrets Manager. That needs a role that may read a -secret, and this host's role may only talk to SSM. Move the key to Secrets -Manager before a Core release carries its public half. +To rotate, ship a Core release that carries both public keys first (Core +carries up to two), then put the new key into the secret. Writing the secret +needs `secretsmanager:PutSecretValue`, which the role lacks: grant it for the +change and take it away after. The key moved from the host's disk to the +secret the same way. ## Host rules @@ -64,17 +73,32 @@ Manager before a Core release carries its public half. Household readings exist only in this process's memory. - Nothing comes in but 443. There is no SSH and no key pair; administration is through SSM. Session logs to a separate account are not set up yet. -- Outbound is open, because the host builds its own image (Docker Hub, - gcr.io, the Go module proxy, GitHub). Narrow it to AWS endpoints, Let's - Encrypt and, from C4, APNs once images come from CI. -- The running build's source is at `https://cloud.ftw.energy/source`. +- Outbound is open. The host now needs only GitHub and `ghcr.io` (the image + and `compose.yml`), AWS endpoints (SSM, Secrets Manager), Let's Encrypt + and, from C4, APNs. Narrowing it is still to do. +- The running build's source is at `https://cloud.ftw.energy/source`, with + its commit in `X-Source-Revision` and the image digest in `X-Source-Image`. ## Bringing it up -`bootstrap.sh` is the whole host. It is idempotent: it turns off swap and -core dumps, installs Docker, Compose and buildx as the relay host does, -clones this repository to `/opt/ftw-cloud`, installs the two systemd units and -`/usr/local/sbin/ftw-cloud-deploy`, and runs that. +The secret and the role's policy come first; they outlive the host: + +```bash +aws secretsmanager create-secret --region eu-north-1 --name ftw-cloud/service-key \ + --description 'FTW Cloud service key, PEM' --tags Key=Project,Value=ftw-cloud +aws iam put-role-policy --role-name ftw-cloud-host --policy-name ftw-cloud-service-key \ + --policy-document file://deploy/host-policy.json +``` + +A secret without a value needs a key in it before the host can start (see +the service key, above). + +`bootstrap.sh` is the rest of the host. It is idempotent: it turns off swap +and core dumps, installs Docker and Compose, clones this repository to +`/opt/ftw-cloud`, and installs `ftw-cloud-key`, the two systemd units and +`/usr/local/sbin/ftw-cloud-deploy`. On a host that already runs an image from +CI, it runs that image again with the new tooling. A new host waits for +`ftw-cloud-deploy` with a digest. ```bash aws ssm send-command --region eu-north-1 --instance-ids i-0ecac4cbc6b5f8adc \ @@ -87,15 +111,33 @@ The DNS record must exist before the first TLS connection, since that is when ## Updating +Every commit on `main` gets an image. Take its digest from the summary of the +`Image` workflow's run, or from the `Digest:` line of: + +```bash +docker buildx imagetools inspect ghcr.io/srcfl/ftw-cloud: +``` + +To check that the digest came from this repository's CI: + +```bash +gh attestation verify oci://ghcr.io/srcfl/ftw-cloud@sha256: -R srcfl/ftw-cloud +``` + +Then run it: + ```bash aws ssm send-command --region eu-north-1 --instance-ids i-0ecac4cbc6b5f8adc \ --document-name AWS-RunShellScript \ - --parameters 'commands=["/usr/local/sbin/ftw-cloud-deploy"]' + --parameters 'commands=["/usr/local/sbin/ftw-cloud-deploy sha256:"]' ``` or, in a session (`aws ssm start-session --region eu-north-1 --target -i-0ecac4cbc6b5f8adc`), `sudo ftw-cloud-deploy`. It pulls the checked-out branch, -builds, and starts a new container. Every box reconnects within seconds. +i-0ecac4cbc6b5f8adc`), `sudo ftw-cloud-deploy sha256:`. It pulls the +image, refuses one whose commit is not on `main`, checks out that commit in +`/opt/ftw-cloud`, fetches the key, starts a new container and deletes the old +image. Every box reconnects within seconds. To go back, run it with the +earlier digest. It always makes a new container, even when only a mounted file changed. The relay ran into this: a file mounted on its own, like the service key, keeps @@ -105,10 +147,12 @@ the copy the container started with after the file changes on the host. ```bash curl -s https://cloud.ftw.energy/healthz +curl -sI https://cloud.ftw.energy/source ``` -`ok` means DNS, the certificate and the service all work. -`https://cloud.ftw.energy/source` returns the source archive. +`ok` means DNS, the certificate and the service all work. `/source` returns +the source archive, and its headers name the commit (`X-Source-Revision`) and +the image digest (`X-Source-Image`), which should match what CI built. `/v1/app` and `/v1/now` answer 501 until C3 and C5. ## Cost @@ -120,11 +164,13 @@ eu-north-1 on-demand prices, October 2026: | `t4g.small` | $0.0172 an hour | $12.56 | | 20 GB gp3 | $0.0836 a GB-month | $1.67 | | Public IPv4 (the Elastic IP) | $0.005 an hour | $3.65 | +| Secrets Manager, one secret | $0.40 a month | $0.40 | | **Total** | | **about $18** | Traffic from boxes comes in, which AWS does not charge for. Traffic out to phones (C3) is free for the first 100 GB a month across the account. SSM costs -nothing. CPU credits run unlimited, which costs extra only if the CPU averages +nothing, and reading the secret at each boot and deploy costs next to +nothing. The public image on `ghcr.io` is free. CPU credits run unlimited, which costs extra only if the CPU averages more than 20% over a day. ## Tearing it down @@ -137,13 +183,17 @@ aws ec2 release-address --region eu-north-1 --allocation-id eipalloc-0085f3e68b6 aws ec2 delete-security-group --region eu-north-1 --group-id sg-098992ad382ecf94a aws iam remove-role-from-instance-profile --instance-profile-name ftw-cloud-host --role-name ftw-cloud-host aws iam delete-instance-profile --instance-profile-name ftw-cloud-host +aws iam delete-role-policy --role-name ftw-cloud-host --policy-name ftw-cloud-service-key aws iam detach-role-policy --role-name ftw-cloud-host \ --policy-arn arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore aws iam delete-role --role-name ftw-cloud-host +aws secretsmanager delete-secret --region eu-north-1 --secret-id ftw-cloud/service-key ``` -The disk goes with the instance, and with it the registry and the service -key. Nothing else holds FTW Cloud data. +The disk goes with the instance, and with it the registry. Secrets Manager +keeps the service key for 30 days after `delete-secret`, so a mistaken +teardown can still be undone, and then deletes it. Nothing else holds FTW +Cloud data; the images on `ghcr.io` hold only code. ## Not here yet diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index 764517c..2ae79c3 100644 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -22,47 +22,67 @@ printf '* hard core 0\n' > /etc/security/limits.d/ftw-cloud.conf # No SSH, not even behind the security group. Administration is through SSM. systemctl disable --now sshd.service -# Docker, Compose and buildx, as the relay host does it. -dnf install -y docker git +# Docker and Compose. The host builds nothing: CI builds the image. +dnf install -y docker git awscli-2 systemctl enable --now docker install -d /usr/local/lib/docker/cli-plugins case "$(uname -m)" in - aarch64) PLUGIN_ARCH=arm64 ; COMPOSE_ARCH=aarch64 ;; - *) PLUGIN_ARCH=amd64 ; COMPOSE_ARCH=x86_64 ;; + aarch64) COMPOSE_ARCH=aarch64 ;; + *) COMPOSE_ARCH=x86_64 ;; esac curl -fsSL "https://github.com/docker/compose/releases/latest/download/docker-compose-linux-${COMPOSE_ARCH}" \ -o /usr/local/lib/docker/cli-plugins/docker-compose chmod +x /usr/local/lib/docker/cli-plugins/docker-compose -# sed reads to the end. head or grep -m1 closes the pipe early, and under -# pipefail that fails the script. -BUILDX_URL=$(curl -fsS https://api.github.com/repos/docker/buildx/releases/latest \ - | grep -o "https://[^\"]*linux-${PLUGIN_ARCH}" | sed -n 1p) -curl -fsSL "$BUILDX_URL" -o /usr/local/lib/docker/cli-plugins/docker-buildx -chmod +x /usr/local/lib/docker/cli-plugins/docker-buildx docker compose version -docker buildx version +# The checkout supplies compose.yml. ftw-cloud-deploy moves it to the commit +# the running image was built from. if [ ! -d /opt/ftw-cloud ]; then git clone https://github.com/srcfl/ftw-cloud /opt/ftw-cloud fi -install -d -m 0700 /var/lib/ftw-cloud /etc/ftw-cloud +install -d -m 0700 /etc/ftw-cloud + +# The service key lives in Secrets Manager. At boot, before Docker starts, +# it comes into tmpfs, where compose.yml mounts it into the container. It is +# never on disk. +cat > /usr/local/sbin/ftw-cloud-key <<'KEY' +#!/bin/bash +# Fetch the service key from Secrets Manager into /run/ftw-cloud (tmpfs). +set -euo pipefail +export AWS_PAGER="" +umask 077 +install -d -m 0700 /run/ftw-cloud +tmp=$(mktemp /run/ftw-cloud/service.key.XXXXXX) +trap 'rm -f "$tmp"' EXIT +# At boot the network and the instance's credentials may lag a little. +for try in $(seq 30); do + if aws secretsmanager get-secret-value --region eu-north-1 \ + --secret-id ftw-cloud/service-key --query SecretString --output text >"$tmp"; then + break + fi + [ "$try" -lt 30 ] || exit 1 + sleep 2 +done +grep -q '^-----BEGIN EC PRIVATE KEY-----$' "$tmp" +chown 65532:65532 "$tmp" +chmod 0400 "$tmp" +mv -f "$tmp" /run/ftw-cloud/service.key +KEY +chmod 0755 /usr/local/sbin/ftw-cloud-key -# The service key rests on the encrypted root volume, readable by root only. -# At boot, before Docker starts, a copy goes to tmpfs, where compose.yml -# mounts it into the container. cat > /etc/systemd/system/ftw-cloud-key.service <<'UNIT' [Unit] -Description=FTW Cloud service key into tmpfs +Description=FTW Cloud service key from Secrets Manager into tmpfs +Wants=network-online.target +After=network-online.target Before=docker.service -ConditionPathExists=/var/lib/ftw-cloud/service.key [Service] Type=oneshot RemainAfterExit=yes -ExecStart=/usr/bin/install -d -m 0700 /run/ftw-cloud -ExecStart=/usr/bin/install -m 0400 -o 65532 -g 65532 /var/lib/ftw-cloud/service.key /run/ftw-cloud/service.key +ExecStart=/usr/local/sbin/ftw-cloud-key [Install] WantedBy=docker.service @@ -87,48 +107,66 @@ ExecStop=/usr/bin/docker compose --env-file /etc/ftw-cloud/compose.env down WantedBy=multi-user.target UNIT -# One command updates the host: pull, build, run. The checkout's branch -# decides what runs. +# One command updates the host to an image CI built, named by its digest. cat > /usr/local/sbin/ftw-cloud-deploy <<'DEPLOY' #!/bin/bash -# Build the commit checked out in /opt/ftw-cloud and run it. Every box -# reconnects within seconds; there is nothing to drain. +# Run an image CI built, named by its digest: +# +# ftw-cloud-deploy sha256:<64 hex> +# +# The Image workflow's run on main shows the digest. With no argument, it +# runs the digest in /etc/ftw-cloud/compose.env again. Every box reconnects +# within seconds; there is nothing to drain. set -euo pipefail export HOME=${HOME:-/root} +repo=ghcr.io/srcfl/ftw-cloud +env=/etc/ftw-cloud/compose.env + +digest=${1:-$(sed -n "s|^FTW_CLOUD_IMAGE=$repo@||p" "$env" 2>/dev/null || true)} +if ! [[ $digest =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "usage: ftw-cloud-deploy sha256:<64 hex>" >&2 + exit 2 +fi +image=$repo@$digest +docker pull --quiet "$image" + +# Run only what CI built from main, and take compose.yml from that same +# commit, so the two cannot drift apart. +rev=$(docker image inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$image") cd /opt/ftw-cloud -git pull --ff-only -rev=$(git rev-parse --short=12 HEAD) -docker build --pull -t "ftw-cloud:$rev" . - -# The key is made once, on this host, and never leaves it. Its public half -# is what Core carries in cloudlink.ServiceKeys. -if [ ! -s /var/lib/ftw-cloud/service.key ]; then - docker run --rm --user 0:0 --network none -v /var/lib/ftw-cloud:/out \ - "ftw-cloud:$rev" keygen -out /out/service.key +git fetch --quiet origin main +if ! [[ $rev =~ ^[0-9a-f]{40}$ ]] || ! git merge-base --is-ancestor "$rev" origin/main; then + echo "$image was not built from a commit on main" >&2 + exit 1 fi -systemctl enable ftw-cloud-key.service ftw-cloud.service +git checkout --quiet --detach "$rev" + +# Fetch the key again, so a new version in Secrets Manager takes effect. systemctl restart ftw-cloud-key.service -printf 'FTW_CLOUD_IMAGE=ftw-cloud:%s\n' "$rev" > /etc/ftw-cloud/compose.env +printf 'FTW_CLOUD_IMAGE=%s\n' "$image" > "$env" cd deploy # Always a new container. A file mounted on its own, such as the key, keeps # the copy the container started with even after the file on the host # changes; only a new container reads it again. -docker compose --env-file /etc/ftw-cloud/compose.env up -d --force-recreate - -for img in $(docker image ls ftw-cloud --format '{{.Repository}}:{{.Tag}}'); do - [ "$img" = "ftw-cloud:$rev" ] || docker image rm "$img" -done -docker image prune -f -docker builder prune -f --filter until=168h -echo "ftw-cloud:$rev running" +docker compose --env-file "$env" up -d --force-recreate +docker image prune --all --force >/dev/null +echo "$image running, built from $rev" DEPLOY chmod 0755 /usr/local/sbin/ftw-cloud-deploy systemctl daemon-reload -/usr/local/sbin/ftw-cloud-deploy +systemctl enable ftw-cloud-key.service ftw-cloud.service # Show that the host rules hold. swapon --show sysctl kernel.core_pattern -docker compose --env-file /etc/ftw-cloud/compose.env -f /opt/ftw-cloud/deploy/compose.yml ps + +# A host that already runs an image from CI runs it again with the new +# tooling. A new host waits for the owner to name a digest. +if grep -q '^FTW_CLOUD_IMAGE=ghcr.io/srcfl/ftw-cloud@sha256:' /etc/ftw-cloud/compose.env 2>/dev/null; then + /usr/local/sbin/ftw-cloud-deploy + docker compose --env-file /etc/ftw-cloud/compose.env -f /opt/ftw-cloud/deploy/compose.yml ps +else + echo "Next: ftw-cloud-deploy sha256:" +fi diff --git a/deploy/compose.yml b/deploy/compose.yml index d632f35..9cc519f 100644 --- a/deploy/compose.yml +++ b/deploy/compose.yml @@ -20,6 +20,9 @@ services: - -state=/state - -service-key=/run/secrets/ftw-cloud-service.key - -acme-email=${FTW_CLOUD_ACME_EMAIL:-} + environment: + # /source publishes it, so anyone can compare it with what CI built. + FTW_CLOUD_IMAGE: ${FTW_CLOUD_IMAGE} volumes: # The registry and certificates. No readings are ever written here. - ftw-cloud-state:/state @@ -41,8 +44,8 @@ services: max-file: '3' secrets: - # Fetched from Secrets Manager at boot into a tmpfs path; never baked - # into the image or the volume. + # ftw-cloud-key.service fetches it from Secrets Manager into tmpfs at + # boot; it is never in the image, the volume or on disk. ftw-cloud-service.key: file: /run/ftw-cloud/service.key diff --git a/deploy/host-policy.json b/deploy/host-policy.json new file mode 100644 index 0000000..68f9158 --- /dev/null +++ b/deploy/host-policy.json @@ -0,0 +1,11 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "ReadTheServiceKey", + "Effect": "Allow", + "Action": "secretsmanager:GetSecretValue", + "Resource": "arn:aws:secretsmanager:eu-north-1:976075258475:secret:ftw-cloud/service-key-??????" + } + ] +} diff --git a/internal/server/server.go b/internal/server/server.go index eb4a0c0..3e4b009 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -78,6 +78,11 @@ func source(w http.ResponseWriter, r *http.Request) { if rev := revision(); rev != "" { w.Header().Set("X-Source-Revision", rev) } + // The image the host runs, by digest, as ftw-cloud-deploy named it. + // Anyone can compare it with the digest CI built from that revision. + if img := os.Getenv("FTW_CLOUD_IMAGE"); img != "" { + w.Header().Set("X-Source-Image", img) + } f, err := os.Open(SourceArchive) if err != nil { w.Header().Set("Content-Type", "text/plain; charset=utf-8") @@ -91,9 +96,16 @@ func source(w http.ResponseWriter, r *http.Request) { http.ServeContent(w, r, "ftw-cloud-source.tar.gz", time.Time{}, f) } +// Revision is the commit the image was built from. The Dockerfile sets it +// with -ldflags -X, because .git stays out of the build context. +var Revision string + // revision is the commit this binary was built from, when the build recorded // one. func revision() string { + if Revision != "" { + return Revision + } info, ok := debug.ReadBuildInfo() if !ok { return "" diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 1322d9c..203c80a 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -73,6 +73,26 @@ func TestSourceSaysWhereTheSourceIs(t *testing.T) { } } +// /source names the commit and the image digest, so anyone can check the +// running service against what CI built. +func TestSourceNamesRevisionAndImage(t *testing.T) { + s, _ := newServer(t) + const rev = "0123456789abcdef0123456789abcdef01234567" + const img = "ghcr.io/srcfl/ftw-cloud@sha256:0000000000000000000000000000000000000000000000000000000000000000" + old := Revision + Revision = rev + t.Cleanup(func() { Revision = old }) + t.Setenv("FTW_CLOUD_IMAGE", img) + w := httptest.NewRecorder() + s.Handler().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/source", nil)) + if got := w.Header().Get("X-Source-Revision"); got != rev { + t.Fatalf("X-Source-Revision = %q", got) + } + if got := w.Header().Get("X-Source-Image"); got != img { + t.Fatalf("X-Source-Image = %q", got) + } +} + func TestTheUplinkRefusesPlainHTTP(t *testing.T) { s, _ := newServer(t) w := httptest.NewRecorder()