From c2e7c496a8619317630749a1a591856f27150ade Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Thu, 15 Oct 2026 19:01:49 +0200 Subject: [PATCH 1/5] feat(keys): ftwkeysd, the one part that can open an agent's session A second process on the same host, as another user, behind a unix socket whose peer uid it checks. It makes an assistant's X25519 agent key when an authorization starts and seals it at rest: one AES-256 data key per agent key from AWS KMS (a file key in development), with an encryption context that names the purpose and the key id. A key stays pending until the owner approves, and opens no session until then; an unapproved key goes after an hour. It unseals a key only to open one Noise IK session to one box, with the box session's prologue set by ftwkeysd itself, and wipes the secret once message 1 is written. It keeps the transport keys and seals and opens each frame for ftwcloudd, which carries them as tun. Forget deletes the key and ends its sessions. Errors on the socket are a few fixed words, and nothing logs a key. Deploy files, not run: a compose service under the `agents` profile on the host's network (the one container that reaches the instance role with an IMDS hop limit of 1), the KMS key policy (root may administer, only the host role may use, only with the purpose), the role's inline policy, and a host script for the socket directory. The guide's first invariant changes as decided on 15 October 2026. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 4 +- CLAUDE.md | 18 +- Dockerfile | 11 +- README.md | 10 + cmd/ftwkeysd/harden_linux.go | 16 ++ cmd/ftwkeysd/harden_other.go | 6 + cmd/ftwkeysd/main.go | 150 ++++++++++ deploy/compose.yml | 43 +++ deploy/keys-host-policy.json | 14 + deploy/keys-kms-policy.json | 39 +++ deploy/keys.md | 129 +++++++++ deploy/keys.sh | 27 ++ go.mod | 15 + go.sum | 30 ++ internal/keys/client.go | 174 ++++++++++++ internal/keys/keys_test.go | 511 ++++++++++++++++++++++++++++++++++ internal/keys/peer_darwin.go | 35 +++ internal/keys/peer_linux.go | 35 +++ internal/keys/peer_other.go | 13 + internal/keys/seal.go | 178 ++++++++++++ internal/keys/server.go | 513 +++++++++++++++++++++++++++++++++++ internal/keys/wire.go | 157 +++++++++++ scripts/check-imports.sh | 10 + 23 files changed, 2129 insertions(+), 9 deletions(-) create mode 100644 cmd/ftwkeysd/harden_linux.go create mode 100644 cmd/ftwkeysd/harden_other.go create mode 100644 cmd/ftwkeysd/main.go create mode 100644 deploy/keys-host-policy.json create mode 100644 deploy/keys-kms-policy.json create mode 100644 deploy/keys.md create mode 100755 deploy/keys.sh create mode 100644 internal/keys/client.go create mode 100644 internal/keys/keys_test.go create mode 100644 internal/keys/peer_darwin.go create mode 100644 internal/keys/peer_linux.go create mode 100644 internal/keys/peer_other.go create mode 100644 internal/keys/seal.go create mode 100644 internal/keys/server.go create mode 100644 internal/keys/wire.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c838f0..9f1e2f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,4 +28,6 @@ jobs: - name: What a watch can post never panics the service run: go test -run XXX -fuzz FuzzOnceMessage -fuzztime 30s ./internal/appsession - name: Builds for the host - run: GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build -o /dev/null ./cmd/ftwcloudd + run: | + GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build -o /dev/null ./cmd/ftwcloudd + GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build -o /dev/null ./cmd/ftwkeysd diff --git a/CLAUDE.md b/CLAUDE.md index 4ad9eeb..257d30f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,12 +11,18 @@ maintains it. ## Invariants -- **FTW Cloud forwards ciphertext it cannot read, and never chooses an - action.** It sends a box only `challenge`, `hello_ok`, `watch`, `resync`, - `purged` and `tun` (`internal/uplink/downlink.go`). Commands go from the app - to the box inside the phone's own Noise session, which rides here as `tun` - under the device id the service's own handshake proved, never one a phone - names. +- **FTW Cloud forwards ciphertext it cannot read. It acts on a home only as + an agent its owner approved, within limits the box enforces.** It sends a + box only `challenge`, `hello_ok`, `watch`, `resync`, `purged` and `tun` + (`internal/uplink/downlink.go`). Commands go from the app to the box inside + the phone's own Noise session, which rides here as `tun` under the device + id the service's own handshake proved, never one a phone names. An agent's + requests ride the agent's own Noise session the same way, and the box + admits it only while its signed roster lists the agent's key. +- **Agent keys live only in `ftwkeysd`** (`internal/keys`, `cmd/ftwkeysd`). It + answers on a local unix socket, seals keys at rest under KMS, unseals a key + only to open one session, and logs no key. `ftwcloudd` never holds an + agent's static key or session keys. Keep it small: nothing else goes in. - **One way to a home:** `store.For(principal)`. No listing, no second lookup, no principal for staff. A new route needs a row in `server.Routes` and must pass `TestNoRouteReadsAHome`. The one other way in is `store.Seed`, called diff --git a/Dockerfile b/Dockerfile index 7d05a26..88dafd6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,15 +18,22 @@ COPY . . # No cgo: the SQLite driver is pure Go, so the binary runs on a static base. RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -trimpath \ -ldflags "-s -w -X github.com/srcfl/ftw-cloud/internal/server.Revision=${REVISION}" \ - -o /out/ftwcloudd ./cmd/ftwcloudd + -o /out/ftwcloudd ./cmd/ftwcloudd \ + && CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -trimpath -ldflags "-s -w" \ + -o /out/ftwkeysd ./cmd/ftwkeysd RUN mkdir -p /out/share && tar -czf /out/share/source.tar.gz \ --exclude=./.git --exclude=./state . \ - && mkdir -p /out/state + && mkdir -p /out/state /out/keys FROM gcr.io/distroless/static-debian12:nonroot COPY --from=build /out/ftwcloudd /usr/local/bin/ftwcloudd +# The keys part, run as its own container and user (deploy/keys.md). +COPY --from=build /out/ftwkeysd /usr/local/bin/ftwkeysd COPY --from=build /out/share/source.tar.gz /usr/share/ftw-cloud/source.tar.gz COPY --from=build --chown=nonroot:nonroot /out/state /state +# Owned by ftwkeysd's user, so a new volume mounted here starts out its own +# and ftwcloudd cannot read it. +COPY --from=build --chown=65533:65532 /out/keys /keys USER nonroot EXPOSE 443 ENTRYPOINT ["/usr/local/bin/ftwcloudd"] diff --git a/README.md b/README.md index 6dc9eb4..f096d50 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,16 @@ to it yet. the roster drops the device, moves its epoch or runs out, an owner suspends the device, or the home is purged. +- **The keys part (`ftwkeysd`).** A second process on the same host, as + another user, answering only on a unix socket. It makes each assistant's + agent key (X25519) when an authorization starts, seals it at rest with a + data key from AWS KMS, and keeps it pending until the owner approves. It + unseals a key only to open that agent's Noise session to its box, keeps + the session's transport keys, and seals and opens each frame for + `ftwcloudd`, which carries them as `tun`. Forgetting a key ends its + sessions. [deploy/keys.md](deploy/keys.md) has the shape and how to turn + it on. + Not built yet: | Step | What | Where | diff --git a/cmd/ftwkeysd/harden_linux.go b/cmd/ftwkeysd/harden_linux.go new file mode 100644 index 0000000..ae5ce09 --- /dev/null +++ b/cmd/ftwkeysd/harden_linux.go @@ -0,0 +1,16 @@ +package main + +import ( + "log/slog" + + "golang.org/x/sys/unix" +) + +// harden makes the process not dumpable: no core file, and no ptrace attach +// from another process of the same user. Agent keys and session keys live in +// this process's memory while a session is open. +func harden() { + if err := unix.Prctl(unix.PR_SET_DUMPABLE, 0, 0, 0, 0); err != nil { + slog.Warn("could not turn off core dumps", "err", err) + } +} diff --git a/cmd/ftwkeysd/harden_other.go b/cmd/ftwkeysd/harden_other.go new file mode 100644 index 0000000..bb16421 --- /dev/null +++ b/cmd/ftwkeysd/harden_other.go @@ -0,0 +1,6 @@ +//go:build !linux + +package main + +// harden has nothing to set outside Linux, where the service runs. +func harden() {} diff --git a/cmd/ftwkeysd/main.go b/cmd/ftwkeysd/main.go new file mode 100644 index 0000000..a266083 --- /dev/null +++ b/cmd/ftwkeysd/main.go @@ -0,0 +1,150 @@ +// Command ftwkeysd is FTW Cloud's keys part: the one process that can open an +// agent's Noise session to a box. It runs beside ftwcloudd on the same host, +// as another user, and answers only on a local unix socket. See +// internal/keys and deploy/keys.md. +// +// ftwkeysd filekey -out dev.key make a development file key +// ftwkeysd [flags] serve +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "log/slog" + "net" + "os" + "os/signal" + "path/filepath" + "syscall" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/kms" + "github.com/aws/aws-sdk-go-v2/service/kms/types" + "github.com/srcfl/ftw-cloud/internal/keys" +) + +func main() { + if len(os.Args) > 1 && os.Args[1] == "filekey" { + fs := flag.NewFlagSet("filekey", flag.ExitOnError) + out := fs.String("out", "", "file to write the key to") + _ = fs.Parse(os.Args[2:]) + if *out == "" { + fmt.Fprintln(os.Stderr, "filekey: -out is required") + os.Exit(1) + } + if err := keys.MakeFileKey(*out); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + return + } + if err := serve(os.Args[1:]); err != nil { + slog.Error("ftwkeysd stopped", "err", err) + os.Exit(1) + } +} + +func serve(args []string) error { + fs := flag.NewFlagSet("ftwkeysd", flag.ExitOnError) + socket := fs.String("socket", "/run/ftw-cloud-keys/keys.sock", "unix socket to answer on") + state := fs.String("state", "/state/keys", "directory for the sealed keys") + kmsKey := fs.String("kms-key", "", "AWS KMS key id, ARN or alias that wraps the data keys") + region := fs.String("region", "eu-north-1", "AWS region of the KMS key") + fileKey := fs.String("file-key", "", "a local file key instead of KMS (development only)") + peerUID := fs.Int("peer-uid", -1, "the one user allowed to connect; -1 is this process's own") + socketMode := fs.Uint("socket-mode", 0o660, "permissions of the socket file") + _ = fs.Parse(args) + + harden() + log := slog.New(slog.NewTextHandler(os.Stderr, nil)) + var sealer keys.Sealer + switch { + case *kmsKey != "" && *fileKey != "": + return errors.New("choose -kms-key or -file-key, not both") + case *kmsKey != "": + cfg, err := config.LoadDefaultConfig(context.Background(), config.WithRegion(*region)) + if err != nil { + return fmt.Errorf("aws config: %w", err) + } + sealer = keys.KMSSealer{KMS: kmsClient{kms.NewFromConfig(cfg)}, KeyID: *kmsKey} + case *fileKey != "": + s, err := keys.NewFileSealer(*fileKey) + if err != nil { + return err + } + log.Warn("agent keys sealed with a local file key; for development only") + sealer = s + default: + return errors.New("no sealer: set -kms-key, or -file-key in development") + } + + srv, err := keys.NewServer(*state, sealer, log) + if err != nil { + return err + } + srv.PeerUID = *peerUID + + // A socket left by an earlier run is removed; nothing else may sit at + // the path. + if fi, err := os.Lstat(*socket); err == nil { + if fi.Mode()&os.ModeSocket == 0 { + return fmt.Errorf("%s exists and is not a socket", *socket) + } + _ = os.Remove(*socket) + } + if err := os.MkdirAll(filepath.Dir(*socket), 0o750); err != nil { + return err + } + ln, err := net.Listen("unix", *socket) + if err != nil { + return err + } + if err := os.Chmod(*socket, os.FileMode(*socketMode)); err != nil { + return err + } + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + go func() { + t := time.NewTicker(time.Minute) + defer t.Stop() + for { + select { + case <-ctx.Done(): + _ = ln.Close() + return + case <-t.C: + srv.Sweep() + } + } + }() + log.Info("ftwkeysd listening", "socket", *socket) + return srv.Serve(ln) +} + +// kmsClient adapts the AWS SDK to keys.KMS. +type kmsClient struct{ c *kms.Client } + +func (k kmsClient) GenerateDataKey(ctx context.Context, keyID string, ec map[string]string) ([]byte, []byte, error) { + out, err := k.c.GenerateDataKey(ctx, &kms.GenerateDataKeyInput{ + KeyId: aws.String(keyID), KeySpec: types.DataKeySpecAes256, EncryptionContext: ec, + }) + if err != nil { + return nil, nil, err + } + return out.Plaintext, out.CiphertextBlob, nil +} + +func (k kmsClient) Decrypt(ctx context.Context, keyID string, blob []byte, ec map[string]string) ([]byte, error) { + out, err := k.c.Decrypt(ctx, &kms.DecryptInput{ + KeyId: aws.String(keyID), CiphertextBlob: blob, EncryptionContext: ec, + }) + if err != nil { + return nil, err + } + return out.Plaintext, nil +} diff --git a/deploy/compose.yml b/deploy/compose.yml index 5504f2a..2d06f07 100644 --- a/deploy/compose.yml +++ b/deploy/compose.yml @@ -34,6 +34,9 @@ services: # boot by deploy/noise.sh. Empty means /v1/app is off, and the # service runs. - /run/ftw-cloud/noise:/run/secrets/noise:ro + # The keys part's socket (deploy/keys.md). Without ftwkeysd it holds + # nothing, and assistants cannot connect. + - /run/ftw-cloud-keys:/run/ftw-cloud-keys secrets: - ftw-cloud-service.key read_only: true @@ -51,6 +54,45 @@ services: max-size: '10m' max-file: '3' + # The keys part: the only process that can act on a home, as an agent its + # owner approved (deploy/keys.md). It starts with COMPOSE_PROFILES=agents + # in compose.env, once the KMS key exists. + ftwkeysd: + image: ${FTW_CLOUD_IMAGE:?set FTW_CLOUD_IMAGE to an image digest} + profiles: [agents] + restart: unless-stopped + entrypoint: ['/usr/local/bin/ftwkeysd'] + command: + - -socket=/run/ftw-cloud-keys/keys.sock + - -state=/keys + - -kms-key=${FTW_CLOUD_KMS_KEY:-} + - -region=eu-north-1 + # ftwcloudd's user, the only one allowed on the socket. + - -peer-uid=65532 + # Its own user, in ftwcloudd's group so that ftwcloudd may connect to + # the socket and nothing more. + user: '65533:65532' + # The host's network, because the instance's credentials answer one hop + # away and the bridged containers sit two hops off (deploy/keys.md): + # this is the one container that can reach KMS. It listens on nothing + # but the unix socket. + network_mode: host + volumes: + # Agent keys, sealed under KMS. ftwcloudd never mounts this. + - ftw-cloud-keys:/keys + - /run/ftw-cloud-keys:/run/ftw-cloud-keys + read_only: true + ulimits: + core: 0 + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + logging: + driver: json-file + options: + max-size: '10m' + max-file: '3' + secrets: # ftw-cloud-key.service fetches it from Secrets Manager into tmpfs at # boot; it is never in the image, the volume or on disk. @@ -59,3 +101,4 @@ secrets: volumes: ftw-cloud-state: + ftw-cloud-keys: diff --git a/deploy/keys-host-policy.json b/deploy/keys-host-policy.json new file mode 100644 index 0000000..cd7d468 --- /dev/null +++ b/deploy/keys-host-policy.json @@ -0,0 +1,14 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "SealAndOpenAgentKeys", + "Effect": "Allow", + "Action": ["kms:GenerateDataKey", "kms:Decrypt"], + "Resource": "arn:aws:kms:eu-north-1:976075258475:key/KEY-ID", + "Condition": { + "StringEquals": { "kms:EncryptionContext:purpose": "ftw-agent-key" } + } + } + ] +} diff --git a/deploy/keys-kms-policy.json b/deploy/keys-kms-policy.json new file mode 100644 index 0000000..bcc2acf --- /dev/null +++ b/deploy/keys-kms-policy.json @@ -0,0 +1,39 @@ +{ + "Version": "2012-10-17", + "Id": "ftw-cloud-agent-keys", + "Statement": [ + { + "Sid": "AdministerButNeverUse", + "Effect": "Allow", + "Principal": { "AWS": "arn:aws:iam::976075258475:root" }, + "Action": [ + "kms:Create*", + "kms:Describe*", + "kms:Enable*", + "kms:List*", + "kms:Put*", + "kms:Update*", + "kms:Revoke*", + "kms:Disable*", + "kms:Get*", + "kms:Delete*", + "kms:TagResource", + "kms:UntagResource", + "kms:ScheduleKeyDeletion", + "kms:CancelKeyDeletion" + ], + "Resource": "*" + }, + { + "Sid": "TheHostSealsAndOpensAgentKeys", + "Effect": "Allow", + "Principal": { "AWS": "arn:aws:iam::976075258475:role/ftw-cloud-host" }, + "Action": ["kms:GenerateDataKey", "kms:Decrypt"], + "Resource": "*", + "Condition": { + "StringEquals": { "kms:EncryptionContext:purpose": "ftw-agent-key" }, + "ForAllValues:StringEquals": { "kms:EncryptionContextKeys": ["purpose", "kid"] } + } + } + ] +} diff --git a/deploy/keys.md b/deploy/keys.md new file mode 100644 index 0000000..2e55618 --- /dev/null +++ b/deploy/keys.md @@ -0,0 +1,129 @@ +# The keys part + +`ftwkeysd` is the one process in FTW Cloud that can act on a home. It holds +each assistant's agent key, an X25519 Noise static key that a home's owner +approved, and it is the only process that can open that agent's Noise session +to the box. `ftwcloudd` carries the session's messages as `tun` through the +box's uplink, as it does a phone's, and cannot read or forge them. + +Nothing here has run on the host. The owner creates the AWS resources below +and turns `ftwkeysd` on. + +## Shape + +| | `ftwcloudd` | `ftwkeysd` | +|---|---|---| +| User | 65532 (`nonroot`) | 65533, group 65532 | +| Network | bridge, 443 published | the host's, listens on nothing | +| Reaches | boxes, phones, assistants | KMS, through the instance role | +| State | `ftw-cloud-state` (registry, OAuth token hashes) | `ftw-cloud-keys` (sealed agent keys) | +| Socket | connects to `/run/ftw-cloud-keys/keys.sock` | owns it, mode 0660, checks the peer's uid | + +- **One KMS key wraps the data keys.** Each agent key has its own AES-256 data + key from `kms:GenerateDataKey`, and the agent key is sealed with it by + AES-GCM. The encryption context names the purpose (`ftw-agent-key`) and the + key id, so a wrapped data key opens only for its own agent key. +- **A key is unsealed only to open a session.** `kms:Decrypt` runs once per + session. The secret is wiped after handshake message 1. The session's + transport keys stay in `ftwkeysd` until the session ends: `ftwcloudd` closes + a warm session after a short idle time, and `ftwkeysd` closes any session + idle for 10 minutes. +- **A key waits for approval.** Each authorize request makes a key under + `pending/`. It opens no session until the owner approves the agent and the + box's signed roster lists it; then it moves to `kept/`. An unapproved key is + deleted after an hour. +- **Revoking deletes the key.** When the box's roster drops an agent, or it + expires, `ftwcloudd` asks `ftwkeysd` to forget the key, which ends its + sessions too. +- **Nothing logs a key.** Errors on the socket are a few fixed words. +- **The volume is not backed up.** Losing it means every assistant must be + approved again; no home is exposed by it. + +### Only `ftwkeysd` reaches KMS + +The instance role can call KMS, so the separation rests on who can reach the +role's credentials. The instance metadata service answers IMDSv2 token +requests one network hop away when the hop limit is 1. A container on Docker's +bridge network is two hops away and gets no credentials; `ftwkeysd` uses the +host's network and gets them. `ftwcloudd` needs no AWS credentials: the host +fetches its secrets before Docker starts. + +The KMS key's policy lets only the host role use it, and only with the +`ftw-agent-key` purpose. The account root can administer the key but not use +it, so another role in the shared account cannot be granted decrypt by IAM +alone. + +If the cloud is fully broken into, the attacker can act as the existing +agents, within the limits their boxes enforce, until the owners revoke them +or they expire. The attacker cannot act on a home without an agent, and +cannot read the apps' sessions with their boxes. + +## Turning it on + +1. **The KMS key** (once, from a workstation with admin rights): + + ```bash + key=$(aws kms create-key --region eu-north-1 \ + --description 'FTW Cloud agent keys (ftwkeysd)' \ + --key-spec SYMMETRIC_DEFAULT --key-usage ENCRYPT_DECRYPT \ + --policy file://deploy/keys-kms-policy.json \ + --tags TagKey=Project,TagValue=ftw-cloud \ + --query KeyMetadata.Arn --output text) + aws kms create-alias --region eu-north-1 --alias-name alias/ftw-cloud-agent-keys \ + --target-key-id "$key" + aws kms enable-key-rotation --region eu-north-1 --key-id "$key" + echo "$key" + ``` + +2. **The role's policy.** Put the key's id into + [`keys-host-policy.json`](keys-host-policy.json) in place of `KEY-ID`, then: + + ```bash + aws iam put-role-policy --role-name ftw-cloud-host --policy-name ftw-cloud-agent-keys \ + --policy-document file://deploy/keys-host-policy.json + ``` + +3. **One hop to the metadata service**, so no bridged container gets the + role's credentials: + + ```bash + aws ec2 modify-instance-metadata-options --region eu-north-1 \ + --instance-id i-0ecac4cbc6b5f8adc --http-tokens required \ + --http-put-response-hop-limit 1 + ``` + + Check from the host that a bridged container is refused: + `docker run --rm curlimages/curl -s -m 3 -X PUT + http://169.254.169.254/latest/api/token -H 'X-aws-ec2-metadata-token-ttl-seconds: 60'` + must time out or fail. + +4. **The host.** Through SSM, with the key's ARN: + + ```bash + aws ssm send-command --region eu-north-1 --instance-ids i-0ecac4cbc6b5f8adc \ + --document-name AWS-RunShellScript \ + --parameters "$(jq -n --rawfile s deploy/keys.sh --arg k "$key" \ + '{commands: ["set -- \($k)", $s]}')" + ``` + + SSM runs the lines as one script, so `set --` hands the script its + argument. It makes `/run/ftw-cloud-keys` at every boot and sets + `FTW_CLOUD_KMS_KEY` and `COMPOSE_PROFILES=agents` in + `/etc/ftw-cloud/compose.env`. + +5. **Deploy** as usual (`ftw-cloud-deploy sha256:`). Compose starts + `ftwkeysd` beside `ftwcloudd`. Its log says `ftwkeysd listening`. + +To turn assistants off again, remove `COMPOSE_PROFILES=agents` and deploy. +Approved agents stay sealed in the volume and work again when it comes back, +unless their owners revoked them meanwhile. + +## Development + +```bash +go run ./cmd/ftwkeysd filekey -out dev-keys.key +go run ./cmd/ftwkeysd -socket /tmp/ftwkeys/keys.sock -state /tmp/ftwkeys/state -file-key dev-keys.key +``` + +A file key is for development only: whoever reads it and the state directory +has every agent key. diff --git a/deploy/keys.sh b/deploy/keys.sh new file mode 100755 index 0000000..ad8a8d1 --- /dev/null +++ b/deploy/keys.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Prepare the FTW Cloud host for ftwkeysd, the keys part. Run as root on the +# host through SSM, after the KMS key exists (deploy/keys.md). Safe to re-run. +# +# It makes the socket's directory at every boot, owned by ftwkeysd's user +# and readable by ftwcloudd's group, and turns ftwkeysd on in compose.env. +# It touches no AWS resource. +set -euo pipefail +key_arn="${1:?usage: keys.sh }" +case "$key_arn" in + arn:aws:kms:eu-north-1:*:key/*) ;; + *) echo "not a KMS key ARN in eu-north-1: $key_arn" >&2; exit 1 ;; +esac + +# ftwkeysd runs as 65533 in ftwcloudd's group, 65532. The directory is the +# only way to the socket: ftwcloudd may enter it and connect, and nothing +# else on the host but root may. +cat > /etc/tmpfiles.d/ftw-cloud-keys.conf <<'TMPFILES' +d /run/ftw-cloud-keys 0750 65533 65532 - +TMPFILES +systemd-tmpfiles --create /etc/tmpfiles.d/ftw-cloud-keys.conf + +env=/etc/ftw-cloud/compose.env +touch "$env" +sed -i '/^FTW_CLOUD_KMS_KEY=/d; /^COMPOSE_PROFILES=/d' "$env" +printf 'FTW_CLOUD_KMS_KEY=%s\nCOMPOSE_PROFILES=agents\n' "$key_arn" >> "$env" +echo "ftwkeysd is on; the next ftw-cloud-deploy (or a restart of ftw-cloud.service) starts it" diff --git a/go.mod b/go.mod index fbaf99f..60a74e4 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,9 @@ module github.com/srcfl/ftw-cloud go 1.26.0 require ( + github.com/aws/aws-sdk-go-v2 v1.47.3 + github.com/aws/aws-sdk-go-v2/config v1.33.9 + github.com/aws/aws-sdk-go-v2/service/kms v1.61.4 github.com/fxamacker/cbor/v2 v2.9.4 github.com/gorilla/websocket v1.5.3 github.com/srcfl/ftw/go/protocol v0.0.0-20261015164244-f5cdb92058c1 @@ -12,6 +15,18 @@ require ( ) require ( + github.com/aws/aws-sdk-go-v2/credentials v1.20.9 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 // indirect + github.com/aws/smithy-go v1.28.5 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/google/uuid v1.6.0 // indirect github.com/mattn/go-isatty v0.0.24 // indirect diff --git a/go.sum b/go.sum index 83b0615..68aa95a 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,33 @@ +github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk= +github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A= +github.com/aws/aws-sdk-go-v2/config v1.33.9 h1:Hv6bcQUJVnmuWaKjqD6Ot8sE75NrWsWV20oGT5aWle0= +github.com/aws/aws-sdk-go-v2/config v1.33.9/go.mod h1:ivowmrZNfBQLcg4QkaItgi5wae4Bzot9RzIdbY1hfTg= +github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0= +github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 h1:CqL8NN3p4I0JTwwV5MEoatIl4Y2amZ2X9dRD+0hbRL0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6/go.mod h1:exdJ32hMqXl0Wyo6NwqEC5yIhtua3+PDaKE8P29m4qE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 h1:FpBq24TG6WVSObTNhQOv7Q0088rounUUlwn3zlNItkM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6/go.mod h1:ONzdPVYC95oBOu0VWNEsKEntcoemP3xVKjT+qWml1s4= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3VvGMOdbGemtUhX4= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 h1:e6YW5uME30TkrElR+4/xcY7NB6CCj2dcv7vtmGXhfhQ= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21/go.mod h1:obpHAMP693+f3/A8PkLhIqfK1C+NWWn8zXzqD3OH520= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 h1:LHHdobjo5le1sjJdolitV7uVvYg3wkaeM1Ju2fDgeZQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6/go.mod h1:CBGB0evAiaMD3Lk4wPO7LQCcCZLuhzL5Qf/wfEUHQ5w= +github.com/aws/aws-sdk-go-v2/service/kms v1.61.4 h1:vW7497x7W6ucLJAqDt96jIdO4+IWLMB3p98UubwjdqM= +github.com/aws/aws-sdk-go-v2/service/kms v1.61.4/go.mod h1:Vz59N7x42VGLXZ5dGL9b10QhrU4GQv7MqnO7wc8bl9U= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.4/go.mod h1:+FEO9jrVMpc9oN3CdAxJbtlQEu3z8UddDPSedpeM3PY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 h1:j6c+HA4AoS0EBEjArPjavFEBrucJEryfrQU0TffmACk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4/go.mod h1:RhghaXUk9q66VtOQUX1FdPddQuHnqQZzbB0fEYXmPRA= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 h1:OKEfbf2tcfsF2pBk/S9QqNpo6alaAtiBs4hysCsBjPw= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.4/go.mod h1:mCd6qGo9b8hTtfjgsmNkFlzwjyce4owd7HDVFnCLXhM= +github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k= +github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fxamacker/cbor/v2 v2.9.4 h1:xwjVlxEMR3S605oUlgBjKLTTeGFciYPGYCtF/35LKGo= diff --git a/internal/keys/client.go b/internal/keys/client.go new file mode 100644 index 0000000..77fc564 --- /dev/null +++ b/internal/keys/client.go @@ -0,0 +1,174 @@ +package keys + +import ( + "context" + "errors" + "net" + "sync" + "time" +) + +// Client is ftwcloudd's side of the keys socket. One connection carries +// every request; it is dialled on first use and again after it breaks. Safe +// for concurrent use. Requests on one session must not overlap: the caller +// keeps each session's frames in order. +type Client struct { + dial func() (net.Conn, error) + + mu sync.Mutex + conn net.Conn + next uint64 + waiting map[uint64]chan response + wmu sync.Mutex +} + +// Dial is a client for the socket at path. Nothing connects until the first +// request. +func Dial(path string) *Client { + return newClient(func() (net.Conn, error) { + var d net.Dialer + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + return d.DialContext(ctx, "unix", path) + }) +} + +func newClient(dial func() (net.Conn, error)) *Client { + return &Client{dial: dial, waiting: map[uint64]chan response{}} +} + +// ErrUnavailable is a keys part that cannot be reached. +var ErrUnavailable = errors.New("keys: not reachable") + +func (c *Client) call(ctx context.Context, req request) (response, error) { + c.mu.Lock() + if c.conn == nil { + conn, err := c.dial() + if err != nil { + c.mu.Unlock() + return response{}, ErrUnavailable + } + c.conn = conn + go c.read(conn) + } + conn := c.conn + c.next++ + req.ID = c.next + ch := make(chan response, 1) + c.waiting[req.ID] = ch + c.mu.Unlock() + + c.wmu.Lock() + err := writeFrame(conn, req) + c.wmu.Unlock() + if err != nil { + c.drop(conn) + return response{}, ErrUnavailable + } + select { + case resp, ok := <-ch: + if !ok { + return response{}, ErrUnavailable + } + return resp, errorFrom(resp.Err) + case <-ctx.Done(): + c.mu.Lock() + delete(c.waiting, req.ID) + c.mu.Unlock() + return response{}, ctx.Err() + } +} + +func (c *Client) read(conn net.Conn) { + for { + var resp response + if err := readFrame(conn, &resp); err != nil { + c.drop(conn) + return + } + c.mu.Lock() + ch := c.waiting[resp.ID] + delete(c.waiting, resp.ID) + c.mu.Unlock() + if ch != nil { + ch <- resp + } + } +} + +// drop forgets a broken connection and fails everything waiting on it. +func (c *Client) drop(conn net.Conn) { + _ = conn.Close() + c.mu.Lock() + defer c.mu.Unlock() + if c.conn != conn { + return + } + c.conn = nil + for id, ch := range c.waiting { + close(ch) + delete(c.waiting, id) + } +} + +// Close closes the connection. +func (c *Client) Close() error { + c.mu.Lock() + conn := c.conn + c.mu.Unlock() + if conn != nil { + c.drop(conn) + } + return nil +} + +// New makes an agent key pair and returns its id and public half. The +// private half stays sealed in the keys part. Until Keep, it opens no +// session, and it is deleted within PendingFor. +func (c *Client) New(ctx context.Context) (kid string, pub []byte, err error) { + resp, err := c.call(ctx, request{Op: opNew}) + return resp.Kid, resp.Pub, err +} + +// Keep keeps a key whose agent the owner approved. +func (c *Client) Keep(ctx context.Context, kid string) error { + _, err := c.call(ctx, request{Op: opKeep, Kid: kid}) + return err +} + +// Forget deletes a key and ends its sessions. +func (c *Client) Forget(ctx context.Context, kid string) error { + _, err := c.call(ctx, request{Op: opForget, Kid: kid}) + return err +} + +// Open starts a session from agent key kid to the box whose Noise static key +// is boxStatic, and returns its id and handshake message 1. +func (c *Client) Open(ctx context.Context, kid string, boxStatic []byte) (sess string, msg1 []byte, err error) { + resp, err := c.call(ctx, request{Op: opOpen, Kid: kid, RS: boxStatic}) + return resp.Sess, resp.D, err +} + +// Finish reads the box's handshake message 2. The session is then ready. +func (c *Client) Finish(ctx context.Context, sess string, msg2 []byte) error { + _, err := c.call(ctx, request{Op: opFinish, Sess: sess, D: msg2}) + return err +} + +// Seal encrypts one frame of the session for the box. +func (c *Client) Seal(ctx context.Context, sess string, frame []byte) ([]byte, error) { + resp, err := c.call(ctx, request{Op: opSeal, Sess: sess, D: frame}) + return resp.D, err +} + +// Unseal decrypts one message from the box. +func (c *Client) Unseal(ctx context.Context, sess string, msg []byte) ([]byte, error) { + resp, err := c.call(ctx, request{Op: opUnseal, Sess: sess, D: msg}) + return resp.D, err +} + +// End closes a session. +func (c *Client) End(ctx context.Context, sess string) error { + _, err := c.call(ctx, request{Op: opClose, Sess: sess}) + return err +} diff --git a/internal/keys/keys_test.go b/internal/keys/keys_test.go new file mode 100644 index 0000000..31edde4 --- /dev/null +++ b/internal/keys/keys_test.go @@ -0,0 +1,511 @@ +package keys + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/hex" + "errors" + "log/slog" + "maps" + "net" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/srcfl/ftw/go/protocol/appwire" +) + +func newTestServer(t *testing.T) (*Server, *FileSealer, *bytes.Buffer) { + t.Helper() + master := bytes.Repeat([]byte{7}, dataKeyBytes) + sealer, err := FileSealerFromKey(master) + if err != nil { + t.Fatal(err) + } + var logs bytes.Buffer + log := slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})) + s, err := NewServer(t.TempDir(), sealer, log) + if err != nil { + t.Fatal(err) + } + return s, sealer, &logs +} + +// box is the responder half of one session, as the box runs it. +type box struct { + static appwire.KeyPair +} + +func newBox(t *testing.T) box { + kp, err := appwire.GenerateKeyPair() + if err != nil { + t.Fatal(err) + } + return box{static: kp} +} + +func (b box) answer(t *testing.T, msg1 []byte) (msg2 []byte, tr *appwire.Transport, agent []byte) { + t.Helper() + hs, err := appwire.NewResponder(appwire.HandshakeOptions{StaticKey: b.static, Prologue: SessionPrologue(b.static.Public())}) + if err != nil { + t.Fatal(err) + } + payload, err := hs.ReadMessage(msg1) + if err != nil { + t.Fatalf("the box cannot read message 1: %v", err) + } + if len(payload) != 0 { + t.Fatalf("message 1 carries %d bytes; a paired device sends none", len(payload)) + } + if msg2, err = hs.WriteMessage(nil); err != nil { + t.Fatal(err) + } + split, err := hs.Split() + if err != nil { + t.Fatal(err) + } + return msg2, appwire.NewTransport(split), split.RemoteStatic +} + +var ctx = context.Background() + +// A key opens a session only once its agent is approved, and the box then +// sees the key the owner approved: the public half the keys part handed out. +func TestAKeyOpensASessionOnlyAfterApproval(t *testing.T) { + s, _, _ := newTestServer(t) + c := s.Local() + b := newBox(t) + + kid, pub, err := c.New(ctx) + if err != nil || len(pub) != appwire.DHBytes || !validKid(kid) { + t.Fatalf("new: %q %d %v", kid, len(pub), err) + } + if _, _, err := c.Open(ctx, kid, b.static.Public()); !errors.Is(err, ErrUnknownKey) { + t.Fatalf("open before approval: %v", err) + } + if err := c.Keep(ctx, kid); err != nil { + t.Fatal(err) + } + if err := c.Keep(ctx, kid); err != nil { + t.Fatalf("keep is idempotent: %v", err) + } + sess, msg1, err := c.Open(ctx, kid, b.static.Public()) + if err != nil { + t.Fatal(err) + } + msg2, boxTr, agent := b.answer(t, msg1) + if !bytes.Equal(agent, pub) { + t.Fatal("the box sees another key than the one approved") + } + if err := c.Finish(ctx, sess, msg2); err != nil { + t.Fatal(err) + } + + // Both ways, in order and out of it. + for i := range 3 { + frame := []byte{byte(i), 'a', 'g', 'e', 'n', 't'} + msg, err := c.Seal(ctx, sess, frame) + if err != nil { + t.Fatal(err) + } + got, err := boxTr.Decrypt(msg) + if err != nil || !bytes.Equal(got, frame) { + t.Fatalf("box read %q, %v", got, err) + } + reply, _ := boxTr.Encrypt([]byte("box")) + back, err := c.Unseal(ctx, sess, reply) + if err != nil || string(back) != "box" { + t.Fatalf("agent read %q, %v", back, err) + } + } + // A replayed box message is refused. + reply, _ := boxTr.Encrypt([]byte("once")) + if _, err := c.Unseal(ctx, sess, reply); err != nil { + t.Fatal(err) + } + if _, err := c.Unseal(ctx, sess, reply); err == nil { + t.Fatal("a replay was accepted") + } + + if err := c.End(ctx, sess); err != nil { + t.Fatal(err) + } + if _, err := c.Seal(ctx, sess, []byte("x")); !errors.Is(err, ErrUnknownSession) { + t.Fatalf("seal after end: %v", err) + } +} + +// Message 1 is bound to one box: another box's key cannot read it, and the +// prologue names the box, so a session cannot be moved between them. +func TestASessionIsBoundToItsBox(t *testing.T) { + s, _, _ := newTestServer(t) + c := s.Local() + kid, _, _ := c.New(ctx) + _ = c.Keep(ctx, kid) + a, other := newBox(t), newBox(t) + _, msg1, err := c.Open(ctx, kid, a.static.Public()) + if err != nil { + t.Fatal(err) + } + hs, _ := appwire.NewResponder(appwire.HandshakeOptions{StaticKey: other.static, Prologue: SessionPrologue(other.static.Public())}) + if _, err := hs.ReadMessage(msg1); err == nil { + t.Fatal("another box read message 1") + } + if _, _, err := c.Open(ctx, kid, []byte("short")); !errors.Is(err, ErrRefused) { + t.Fatalf("open with a malformed box key: %v", err) + } +} + +// A bad message 2 ends the attempt rather than leaving a half-open session. +func TestABadMessage2EndsTheAttempt(t *testing.T) { + s, _, _ := newTestServer(t) + c := s.Local() + kid, _, _ := c.New(ctx) + _ = c.Keep(ctx, kid) + b := newBox(t) + sess, _, _ := c.Open(ctx, kid, b.static.Public()) + if err := c.Finish(ctx, sess, bytes.Repeat([]byte{1}, 48)); !errors.Is(err, ErrRefused) { + t.Fatalf("finish with noise: %v", err) + } + if s.Sessions() != 0 { + t.Fatal("the failed attempt is still held") + } + // Sealing before the handshake finished is refused too. + sess, _, _ = c.Open(ctx, kid, b.static.Public()) + if _, err := c.Seal(ctx, sess, []byte("early")); !errors.Is(err, ErrRefused) { + t.Fatalf("seal before finish: %v", err) + } +} + +// Forget deletes the key and ends its sessions at once: revoking an agent +// leaves nothing in the keys part that could act for it. +func TestForgetEndsEverySessionOfTheKey(t *testing.T) { + s, _, _ := newTestServer(t) + c := s.Local() + kid, _, _ := c.New(ctx) + _ = c.Keep(ctx, kid) + b := newBox(t) + sess, msg1, _ := c.Open(ctx, kid, b.static.Public()) + msg2, _, _ := b.answer(t, msg1) + _ = c.Finish(ctx, sess, msg2) + + other, _, _ := c.New(ctx) + _ = c.Keep(ctx, other) + osess, _, _ := c.Open(ctx, other, b.static.Public()) + + if err := c.Forget(ctx, kid); err != nil { + t.Fatal(err) + } + if _, err := c.Seal(ctx, sess, []byte("x")); !errors.Is(err, ErrUnknownSession) { + t.Fatalf("seal after forget: %v", err) + } + if _, _, err := c.Open(ctx, kid, b.static.Public()); !errors.Is(err, ErrUnknownKey) { + t.Fatalf("open after forget: %v", err) + } + if _, err := os.Stat(filepath.Join(s.dir, "kept", kid)); !errors.Is(err, os.ErrNotExist) { + t.Fatal("the sealed key is still on disk") + } + if _, err := c.Seal(ctx, osess, []byte("x")); errors.Is(err, ErrUnknownSession) { + t.Fatal("forgetting one key ended another key's session") + } +} + +// No key id names a file outside the state directory. +func TestKidsAreNeverPaths(t *testing.T) { + s, _, _ := newTestServer(t) + c := s.Local() + for _, kid := range []string{"", "../kept/x", "..", "a/b", "AAAAAAAAAAAAAAAAAAAAAA==", "kept"} { + if err := c.Keep(ctx, kid); !errors.Is(err, ErrUnknownKey) { + t.Fatalf("keep %q: %v", kid, err) + } + if err := c.Forget(ctx, kid); !errors.Is(err, ErrUnknownKey) { + t.Fatalf("forget %q: %v", kid, err) + } + if _, _, err := c.Open(ctx, kid, make([]byte, 32)); !errors.Is(err, ErrUnknownKey) { + t.Fatalf("open %q: %v", kid, err) + } + } +} + +// The secret is on disk only sealed, a record moved to another kid does not +// open, and nothing the keys part logs or answers carries it. +func TestTheKeyIsSealedAndNeverLogged(t *testing.T) { + s, sealer, logs := newTestServer(t) + c := s.Local() + b := newBox(t) + kid, pub, _ := c.New(ctx) + _ = c.Keep(ctx, kid) + raw, err := os.ReadFile(filepath.Join(s.dir, "kept", kid)) + if err != nil { + t.Fatal(err) + } + var rec record + if err := decMode.Unmarshal(raw, &rec); err != nil { + t.Fatal(err) + } + dek, err := sealer.Unwrap(ctx, kid, rec.DEK) + if err != nil { + t.Fatal(err) + } + aead, _ := newGCM(dek) + secret, err := open(aead, rec.CT, keyAAD(kid, pub)) + if err != nil || len(secret) != 32 { + t.Fatalf("the test cannot unseal its own key: %v", err) + } + if bytes.Contains(raw, secret) { + t.Fatal("the secret is on disk in the clear") + } + + // The same record under another kid does not open: the kid is bound + // into both the data key and the sealed secret. + other, _, _ := c.New(ctx) + _ = c.Keep(ctx, other) + if err := os.WriteFile(filepath.Join(s.dir, "kept", other), bytes.ReplaceAll(raw, []byte(kid), []byte(other)), 0o600); err != nil { + t.Fatal(err) + } + if _, _, err := c.Open(ctx, other, b.static.Public()); err == nil { + t.Fatal("a sealed key opened under another kid") + } + + // Run every operation, then look for the secret in what was said. + sess, msg1, _ := c.Open(ctx, kid, b.static.Public()) + msg2, _, _ := b.answer(t, msg1) + _ = c.Finish(ctx, sess, msg2) + _, _ = c.Seal(ctx, sess, []byte("frame")) + _, _ = c.Unseal(ctx, sess, []byte("not a message at all, at least 24 bytes")) + _ = c.Forget(ctx, kid) + for _, form := range [][]byte{secret, []byte(hex.EncodeToString(secret)), []byte(base64.StdEncoding.EncodeToString(secret)), + []byte(base64.RawURLEncoding.EncodeToString(secret)), dek, []byte(hex.EncodeToString(dek))} { + if bytes.Contains(logs.Bytes(), form) { + t.Fatal("the log carries key material") + } + } +} + +// An error crossing the socket is one of a few fixed words. +func TestErrorsSayNothingAboutKeys(t *testing.T) { + failing := errors.New("kms said: key material 0123456789abcdef") + if got := errorText(failing); got != "failed" { + t.Fatalf("errorText = %q", got) + } + if got := errorText(errors.Join(ErrBusy, failing)); got != ErrBusy.Error() { + t.Fatalf("errorText = %q", got) + } +} + +// fakeKMS checks the encryption context the way KMS does: a blob opens only +// under the context it was made with. +type fakeKMS struct { + mu sync.Mutex + blobs map[string]struct { + plain []byte + ctx map[string]string + key string + } + calls int +} + +func (f *fakeKMS) GenerateDataKey(_ context.Context, keyID string, c map[string]string) ([]byte, []byte, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls++ + plain := bytes.Repeat([]byte{byte(f.calls)}, 32) + blob := []byte("blob-" + string(rune('a'+f.calls))) + if f.blobs == nil { + f.blobs = map[string]struct { + plain []byte + ctx map[string]string + key string + }{} + } + f.blobs[string(blob)] = struct { + plain []byte + ctx map[string]string + key string + }{bytes.Clone(plain), maps.Clone(c), keyID} + return plain, blob, nil +} + +func (f *fakeKMS) Decrypt(_ context.Context, keyID string, blob []byte, c map[string]string) ([]byte, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls++ + b, ok := f.blobs[string(blob)] + if !ok || b.key != keyID || !maps.Equal(b.ctx, c) { + return nil, errors.New("InvalidCiphertextException") + } + return bytes.Clone(b.plain), nil +} + +// With KMS, each key has its own data key, wrapped under a context that +// names the purpose and the kid; the key policy can require the purpose. +func TestKMSSealerBindsTheContext(t *testing.T) { + kms := &fakeKMS{} + sealer := KMSSealer{KMS: kms, KeyID: "alias/ftw-cloud-agent-keys"} + s, err := NewServer(t.TempDir(), sealer, nil) + if err != nil { + t.Fatal(err) + } + c := s.Local() + b := newBox(t) + kid, _, err := c.New(ctx) + if err != nil { + t.Fatal(err) + } + _ = c.Keep(ctx, kid) + for _, blob := range kms.blobs { + if blob.ctx["purpose"] != EncryptionContextPurpose || blob.ctx["kid"] != kid { + t.Fatalf("encryption context %v", blob.ctx) + } + } + before := kms.calls + sess, msg1, err := c.Open(ctx, kid, b.static.Public()) + if err != nil || sess == "" { + t.Fatalf("open through KMS: %v", err) + } + if kms.calls != before+1 { + t.Fatalf("opening a session made %d KMS calls, want 1", kms.calls-before) + } + _, _, _ = b.answer(t, msg1) + + // The wrapped data key does not open for another kid. + var wrapped []byte + for blob := range kms.blobs { + wrapped = []byte(blob) + } + if _, err := sealer.Unwrap(ctx, "AAAAAAAAAAAAAAAAAAAAAA", wrapped); err == nil { + t.Fatal("a data key opened under another kid") + } +} + +// Keys whose approval never came go after PendingFor, and idle sessions +// after IdleFor. +func TestSweepDropsStalePendingKeysAndIdleSessions(t *testing.T) { + s, _, _ := newTestServer(t) + now := time.Now() + s.now = func() time.Time { return now } + c := s.Local() + stale, _, _ := c.New(ctx) + fresh, _, _ := c.New(ctx) + kept, _, _ := c.New(ctx) + _ = c.Keep(ctx, kept) + old := now.Add(-PendingFor - time.Minute) + _ = os.Chtimes(filepath.Join(s.dir, "pending", stale), old, old) + _ = os.Chtimes(filepath.Join(s.dir, "kept", kept), old, old) + b := newBox(t) + sess, _, _ := c.Open(ctx, kept, b.static.Public()) + + now = now.Add(IdleFor + time.Second) + s.Sweep() + if _, err := os.Stat(filepath.Join(s.dir, "pending", stale)); !errors.Is(err, os.ErrNotExist) { + t.Fatal("a stale pending key is still there") + } + for _, p := range []string{filepath.Join("pending", fresh), filepath.Join("kept", kept)} { + if _, err := os.Stat(filepath.Join(s.dir, p)); err != nil { + t.Fatalf("%s went: %v", p, err) + } + } + if _, err := c.Seal(ctx, sess, []byte("x")); !errors.Is(err, ErrUnknownSession) { + t.Fatal("an idle session outlived the sweep") + } +} + +// The socket: the peer's user is checked, and the client dials again after +// ftwkeysd restarts. A session does not survive the restart. +func TestTheSocketChecksThePeerAndSurvivesARestart(t *testing.T) { + dir, err := os.MkdirTemp("", "k") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + path := filepath.Join(dir, "s") + s, _, _ := newTestServer(t) + serve := func(srv *Server) net.Listener { + ln, err := net.Listen("unix", path) + if err != nil { + t.Fatal(err) + } + go func() { _ = srv.Serve(ln) }() + return ln + } + ln := serve(s) + c := Dial(path) + defer c.Close() + kid, _, err := c.New(ctx) + if err != nil { + t.Fatalf("own user refused: %v", err) + } + _ = c.Keep(ctx, kid) + b := newBox(t) + sess, _, err := c.Open(ctx, kid, b.static.Public()) + if err != nil { + t.Fatal(err) + } + + // Restart: same state directory, new process, sessions gone. + _ = ln.Close() + _ = c.Close() + s2, err := NewServer(s.dir, s.sealer, nil) + if err != nil { + t.Fatal(err) + } + ln = serve(s2) + if _, err := c.Seal(ctx, sess, []byte("x")); !errors.Is(err, ErrUnknownSession) { + t.Fatalf("a session survived the restart: %v", err) + } + if _, _, err := c.Open(ctx, kid, b.static.Public()); err != nil { + t.Fatalf("a kept key did not survive the restart: %v", err) + } + + // Another user is turned away before a single request is read. + _ = ln.Close() + _ = c.Close() + s3, _ := NewServer(s.dir, s.sealer, nil) + s3.PeerUID = os.Getuid() + 1 + ln = serve(s3) + defer ln.Close() + if _, _, err := c.New(ctx); err == nil { + t.Fatal("a peer of another user was served") + } +} + +// Many sessions at once, each kept in order by its caller. +func TestManySessionsAtOnce(t *testing.T) { + s, _, _ := newTestServer(t) + c := s.Local() + kid, _, _ := c.New(ctx) + _ = c.Keep(ctx, kid) + var wg sync.WaitGroup + for range 16 { + wg.Add(1) + go func() { + defer wg.Done() + b := newBox(t) + sess, msg1, err := c.Open(ctx, kid, b.static.Public()) + if err != nil { + t.Error(err) + return + } + msg2, tr, _ := b.answer(t, msg1) + if err := c.Finish(ctx, sess, msg2); err != nil { + t.Error(err) + return + } + for i := range 20 { + msg, err := c.Seal(ctx, sess, []byte{byte(i)}) + if err != nil { + t.Error(err) + return + } + if got, err := tr.Decrypt(msg); err != nil || got[0] != byte(i) { + t.Errorf("frame %d: %v", i, err) + return + } + } + }() + } + wg.Wait() +} diff --git a/internal/keys/peer_darwin.go b/internal/keys/peer_darwin.go new file mode 100644 index 0000000..52533c9 --- /dev/null +++ b/internal/keys/peer_darwin.go @@ -0,0 +1,35 @@ +package keys + +import ( + "errors" + "net" + "os" + + "golang.org/x/sys/unix" +) + +// checkPeer refuses a unix connection from any user but uid, or from any +// user but this process's own when uid is negative. +func checkPeer(c *net.UnixConn, uid int) error { + if uid < 0 { + uid = os.Getuid() + } + raw, err := c.SyscallConn() + if err != nil { + return err + } + var cred *unix.Xucred + var credErr error + if err := raw.Control(func(fd uintptr) { + cred, credErr = unix.GetsockoptXucred(int(fd), unix.SOL_LOCAL, unix.LOCAL_PEERCRED) + }); err != nil { + return err + } + if credErr != nil { + return credErr + } + if int(cred.Uid) != uid { + return errors.New("keys: a peer of another user") + } + return nil +} diff --git a/internal/keys/peer_linux.go b/internal/keys/peer_linux.go new file mode 100644 index 0000000..b690437 --- /dev/null +++ b/internal/keys/peer_linux.go @@ -0,0 +1,35 @@ +package keys + +import ( + "errors" + "net" + "os" + + "golang.org/x/sys/unix" +) + +// checkPeer refuses a unix connection from any user but uid, or from any +// user but this process's own when uid is negative. +func checkPeer(c *net.UnixConn, uid int) error { + if uid < 0 { + uid = os.Getuid() + } + raw, err := c.SyscallConn() + if err != nil { + return err + } + var cred *unix.Ucred + var credErr error + if err := raw.Control(func(fd uintptr) { + cred, credErr = unix.GetsockoptUcred(int(fd), unix.SOL_SOCKET, unix.SO_PEERCRED) + }); err != nil { + return err + } + if credErr != nil { + return credErr + } + if int(cred.Uid) != uid { + return errors.New("keys: a peer of another user") + } + return nil +} diff --git a/internal/keys/peer_other.go b/internal/keys/peer_other.go new file mode 100644 index 0000000..2127653 --- /dev/null +++ b/internal/keys/peer_other.go @@ -0,0 +1,13 @@ +//go:build !linux && !darwin + +package keys + +import ( + "errors" + "net" +) + +// checkPeer cannot tell who is on the other end here, so it refuses. +func checkPeer(*net.UnixConn, int) error { + return errors.New("keys: peer credentials are not available on this system") +} diff --git a/internal/keys/seal.go b/internal/keys/seal.go new file mode 100644 index 0000000..910812b --- /dev/null +++ b/internal/keys/seal.go @@ -0,0 +1,178 @@ +package keys + +import ( + "context" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "os" + "strings" +) + +// A Sealer makes and unwraps the data keys that seal agent keys at rest: +// envelope encryption, one data key per agent key. In production the data +// keys are wrapped by one AWS KMS key that only ftwkeysd's role may use; in +// development, by a key in a local file. +type Sealer interface { + // DataKey returns a new data key and its wrapped form, bound to kid. + DataKey(ctx context.Context, kid string) (plain, wrapped []byte, err error) + // Unwrap returns the data key wrapped for kid. + Unwrap(ctx context.Context, kid string, wrapped []byte) ([]byte, error) +} + +// dataKeyBytes is an AES-256 key. +const dataKeyBytes = 32 + +// KMS is the part of AWS KMS the sealer uses. cmd/ftwkeysd adapts the AWS +// SDK to it, so this package and its tests do not link the SDK. +type KMS interface { + // GenerateDataKey asks for a new AES-256 data key under keyID with the + // encryption context, as kms:GenerateDataKey. + GenerateDataKey(ctx context.Context, keyID string, context map[string]string) (plain, blob []byte, err error) + // Decrypt unwraps blob under keyID with the same context, as + // kms:Decrypt. + Decrypt(ctx context.Context, keyID string, blob []byte, context map[string]string) ([]byte, error) +} + +// KMSSealer wraps data keys with one AWS KMS key. The encryption context +// names the purpose and the key id, so a wrapped data key opens only for the +// agent key it was made for, and the key policy can require the purpose. +type KMSSealer struct { + KMS KMS + KeyID string +} + +// EncryptionContextPurpose is the purpose in every encryption context. The +// KMS key policy allows ftwkeysd's role only with it. +const EncryptionContextPurpose = "ftw-agent-key" + +func kmsContext(kid string) map[string]string { + return map[string]string{"purpose": EncryptionContextPurpose, "kid": kid} +} + +func (s KMSSealer) DataKey(ctx context.Context, kid string) ([]byte, []byte, error) { + plain, blob, err := s.KMS.GenerateDataKey(ctx, s.KeyID, kmsContext(kid)) + if err != nil { + return nil, nil, err + } + if len(plain) != dataKeyBytes || len(blob) == 0 { + return nil, nil, errors.New("keys: KMS returned a data key of the wrong size") + } + return plain, blob, nil +} + +func (s KMSSealer) Unwrap(ctx context.Context, kid string, wrapped []byte) ([]byte, error) { + plain, err := s.KMS.Decrypt(ctx, s.KeyID, wrapped, kmsContext(kid)) + if err != nil { + return nil, err + } + if len(plain) != dataKeyBytes { + return nil, errors.New("keys: KMS returned a data key of the wrong size") + } + return plain, nil +} + +// FileSealer wraps data keys with a 32-byte key from a local file. It is for +// development and tests: whoever reads the file and the state directory has +// every agent key. +type FileSealer struct { + aead cipher.AEAD +} + +// NewFileSealer reads a key file of 32 bytes, raw or as 64 hex digits. +func NewFileSealer(path string) (*FileSealer, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("file key: %w", err) + } + key := raw + if s := strings.TrimSpace(string(raw)); len(s) == 2*dataKeyBytes { + if key, err = hex.DecodeString(s); err != nil { + return nil, fmt.Errorf("file key: %w", err) + } + } + if len(key) != dataKeyBytes { + return nil, fmt.Errorf("file key: %s holds %d bytes, want %d", path, len(key), dataKeyBytes) + } + return FileSealerFromKey(key) +} + +// FileSealerFromKey is a FileSealer around key. +func FileSealerFromKey(key []byte) (*FileSealer, error) { + aead, err := newGCM(key) + if err != nil { + return nil, err + } + return &FileSealer{aead: aead}, nil +} + +// MakeFileKey writes a new file key that only its owner can read, and never +// over an existing file. +func MakeFileKey(path string) error { + key := make([]byte, dataKeyBytes) + if _, err := rand.Read(key); err != nil { + return err + } + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return err + } + if _, err := f.Write([]byte(hex.EncodeToString(key) + "\n")); err != nil { + _ = f.Close() + return err + } + return f.Close() +} + +func fileAAD(kid string) []byte { return []byte("ftw.agent-dek.v1\x00" + kid) } + +func (s *FileSealer) DataKey(_ context.Context, kid string) ([]byte, []byte, error) { + plain := make([]byte, dataKeyBytes) + if _, err := rand.Read(plain); err != nil { + return nil, nil, err + } + wrapped, err := seal(s.aead, plain, fileAAD(kid)) + if err != nil { + return nil, nil, err + } + return plain, wrapped, nil +} + +func (s *FileSealer) Unwrap(_ context.Context, kid string, wrapped []byte) ([]byte, error) { + return open(s.aead, wrapped, fileAAD(kid)) +} + +func newGCM(key []byte) (cipher.AEAD, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + return cipher.NewGCM(block) +} + +// seal is nonce ‖ AES-GCM(plain, aad). +func seal(aead cipher.AEAD, plain, aad []byte) ([]byte, error) { + nonce := make([]byte, aead.NonceSize(), aead.NonceSize()+len(plain)+aead.Overhead()) + if _, err := rand.Read(nonce); err != nil { + return nil, err + } + return aead.Seal(nonce, nonce, plain, aad), nil +} + +func open(aead cipher.AEAD, sealed, aad []byte) ([]byte, error) { + if len(sealed) < aead.NonceSize()+aead.Overhead() { + return nil, errors.New("keys: sealed data too short") + } + n := aead.NonceSize() + return aead.Open(nil, sealed[:n], sealed[n:], aad) +} + +// wipe overwrites a secret held in a slice. Go may have copied it elsewhere, +// so this narrows how long a key lingers in memory; it does not promise +// that no copy is left. +func wipe(b []byte) { + clear(b) +} diff --git a/internal/keys/server.go b/internal/keys/server.go new file mode 100644 index 0000000..1788a73 --- /dev/null +++ b/internal/keys/server.go @@ -0,0 +1,513 @@ +package keys + +import ( + "context" + "crypto/ecdh" + "crypto/rand" + "encoding/base64" + "errors" + "io/fs" + "log/slog" + "net" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/srcfl/ftw/go/protocol/appwire" +) + +// Limits. +const ( + // MaxPending bounds the keys made for consents the owner has not + // approved yet. Each authorize request makes one, so this bounds what a + // stranger can make the keys part hold. + MaxPending = 10_000 + // PendingFor is how long an unapproved key is kept. A consent page + // lasts far less. + PendingFor = time.Hour + // MaxSessions bounds the open sessions, and IdleFor how long one lasts + // unused. ftwcloudd closes its warm sessions sooner; these catch one it + // lost track of. + MaxSessions = 4096 + IdleFor = 10 * time.Minute + // unwrapTimeout bounds one KMS call. + unwrapTimeout = 5 * time.Second +) + +// SessionPrologue is the box session's prologue, as the box spells it +// (appuplink.prologue in srcfl/ftw). The keys part sets it itself, so it +// opens sessions for this protocol and no other. +func SessionPrologue(boxStatic []byte) []byte { + return append([]byte("ftw.session.v1:"), boxStatic...) +} + +// Server is ftwkeysd: the sealed keys on disk and the open sessions in +// memory. +type Server struct { + dir string + sealer Sealer + log *slog.Logger + now func() time.Time + // PeerUID is the one user allowed to connect, checked on each accepted + // unix connection. Negative allows the server's own user only. + PeerUID int + + mu sync.Mutex + sessions map[string]*session +} + +type session struct { + kid string + mu sync.Mutex + init *appwire.InitiatorState + tr *appwire.Transport + used time.Time +} + +// NewServer keeps sealed keys under dir: pending/ until the owner approves, +// then kept/. +func NewServer(dir string, sealer Sealer, log *slog.Logger) (*Server, error) { + if log == nil { + log = slog.Default() + } + for _, d := range []string{"pending", "kept"} { + if err := os.MkdirAll(filepath.Join(dir, d), 0o700); err != nil { + return nil, err + } + } + return &Server{dir: dir, sealer: sealer, log: log, now: time.Now, PeerUID: -1, sessions: map[string]*session{}}, nil +} + +// Serve accepts connections until ln closes. Each must come from PeerUID. +func (s *Server) Serve(ln net.Listener) error { + for { + conn, err := ln.Accept() + if err != nil { + if errors.Is(err, net.ErrClosed) { + return nil + } + return err + } + if uc, ok := conn.(*net.UnixConn); ok { + if err := checkPeer(uc, s.PeerUID); err != nil { + s.log.Warn("refused a connection from another user") + _ = conn.Close() + continue + } + } + go s.serveConn(conn) + } +} + +// Local is a client wired to this server in memory, for tests and for a +// development build that runs both parts in one process. +func (s *Server) Local() *Client { + a, b := net.Pipe() + go s.serveConn(b) + return newClient(func() (net.Conn, error) { return a, nil }) +} + +func (s *Server) serveConn(conn net.Conn) { + defer conn.Close() + var wmu sync.Mutex + for { + var req request + if err := readFrame(conn, &req); err != nil { + return + } + go func() { + resp := s.handle(req) + resp.ID = req.ID + wmu.Lock() + defer wmu.Unlock() + _ = writeFrame(conn, resp) + }() + } +} + +func (s *Server) handle(req request) response { + var ( + resp response + err error + ) + switch req.Op { + case opNew: + resp.Kid, resp.Pub, err = s.newKey() + case opKeep: + err = s.keep(req.Kid) + case opForget: + err = s.forget(req.Kid) + case opOpen: + resp.Sess, resp.D, err = s.open(req.Kid, req.RS) + case opFinish: + resp.D, err = s.finish(req.Sess, req.D) + case opSeal: + err = s.with(req.Sess, func(x *session) error { + if x.tr == nil { + return ErrRefused + } + resp.D, err = x.tr.Encrypt(req.D) + return err + }) + case opUnseal: + err = s.with(req.Sess, func(x *session) error { + if x.tr == nil { + return ErrRefused + } + resp.D, err = x.tr.Decrypt(req.D) + return err + }) + case opClose: + s.end(req.Sess) + default: + err = ErrRefused + } + if err != nil { + return response{Err: errorText(err)} + } + return resp +} + +// -------------------------------------------------------------------------- +// Keys at rest +// -------------------------------------------------------------------------- + +// record is one sealed key on disk. +type record struct { + V int `cbor:"v"` + Kid string `cbor:"kid"` + Pub []byte `cbor:"pub"` + // DEK is the data key, wrapped by the sealer; CT the X25519 secret + // sealed with it. + DEK []byte `cbor:"dek"` + CT []byte `cbor:"ct"` + At int64 `cbor:"at"` +} + +func keyAAD(kid string, pub []byte) []byte { + return append([]byte("ftw.agent-key.v1\x00"+kid+"\x00"), pub...) +} + +// validKid is the shape newKey makes: 16 random bytes in base64url. Nothing +// else is a file name here. +func validKid(kid string) bool { + b, err := base64.RawURLEncoding.DecodeString(kid) + return err == nil && len(b) == 16 && base64.RawURLEncoding.EncodeToString(b) == kid +} + +func (s *Server) newKey() (string, []byte, error) { + if n, _ := countDir(filepath.Join(s.dir, "pending")); n >= MaxPending { + return "", nil, ErrBusy + } + id := make([]byte, 16) + if _, err := rand.Read(id); err != nil { + return "", nil, err + } + kid := base64.RawURLEncoding.EncodeToString(id) + priv, err := ecdh.X25519().GenerateKey(rand.Reader) + if err != nil { + return "", nil, err + } + pub := priv.PublicKey().Bytes() + secret := priv.Bytes() + defer wipe(secret) + + ctx, cancel := context.WithTimeout(context.Background(), unwrapTimeout) + defer cancel() + dek, wrapped, err := s.sealer.DataKey(ctx, kid) + if err != nil { + s.log.Warn("could not make a data key", "err", err) + return "", nil, err + } + defer wipe(dek) + aead, err := newGCM(dek) + if err != nil { + return "", nil, err + } + ct, err := seal(aead, secret, keyAAD(kid, pub)) + if err != nil { + return "", nil, err + } + raw, err := encMode.Marshal(record{V: 1, Kid: kid, Pub: pub, DEK: wrapped, CT: ct, At: s.now().UnixMilli()}) + if err != nil { + return "", nil, err + } + if err := writeFile(filepath.Join(s.dir, "pending", kid), raw); err != nil { + return "", nil, err + } + return kid, pub, nil +} + +func (s *Server) keep(kid string) error { + if !validKid(kid) { + return ErrUnknownKey + } + from, to := filepath.Join(s.dir, "pending", kid), filepath.Join(s.dir, "kept", kid) + if err := os.Rename(from, to); err != nil { + if _, statErr := os.Stat(to); statErr == nil { + return nil // kept already + } + return ErrUnknownKey + } + return syncDir(filepath.Join(s.dir, "kept")) +} + +func (s *Server) forget(kid string) error { + if !validKid(kid) { + return ErrUnknownKey + } + for _, d := range []string{"pending", "kept"} { + if err := os.Remove(filepath.Join(s.dir, d, kid)); err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + } + s.mu.Lock() + var ending []*session + for id, x := range s.sessions { + if x.kid == kid { + ending = append(ending, x) + delete(s.sessions, id) + } + } + s.mu.Unlock() + for _, x := range ending { + x.close() + } + return nil +} + +// unseal reads a kept key. A key whose approval is still pending opens no +// session. +func (s *Server) unseal(kid string) ([]byte, error) { + if !validKid(kid) { + return nil, ErrUnknownKey + } + raw, err := os.ReadFile(filepath.Join(s.dir, "kept", kid)) + if err != nil { + return nil, ErrUnknownKey + } + var rec record + if err := decMode.Unmarshal(raw, &rec); err != nil || rec.Kid != kid || len(rec.Pub) != appwire.DHBytes { + return nil, ErrUnknownKey + } + ctx, cancel := context.WithTimeout(context.Background(), unwrapTimeout) + defer cancel() + dek, err := s.sealer.Unwrap(ctx, kid, rec.DEK) + if err != nil { + s.log.Warn("could not unwrap a data key", "err", err) + return nil, err + } + defer wipe(dek) + aead, err := newGCM(dek) + if err != nil { + return nil, err + } + secret, err := open(aead, rec.CT, keyAAD(kid, rec.Pub)) + if err != nil { + return nil, ErrUnknownKey + } + return secret, nil +} + +// -------------------------------------------------------------------------- +// Sessions +// -------------------------------------------------------------------------- + +// open unseals kid and starts a session to the box whose static key is rs. +// The secret is wiped once message 1 is written; the handshake state keeps +// what it needs to read message 2. +func (s *Server) open(kid string, rs []byte) (string, []byte, error) { + if len(rs) != appwire.DHBytes { + return "", nil, ErrRefused + } + s.mu.Lock() + full := len(s.sessions) >= MaxSessions + s.mu.Unlock() + if full { + return "", nil, ErrBusy + } + secret, err := s.unseal(kid) + if err != nil { + return "", nil, err + } + kp, err := appwire.KeyPairFromSecret(secret) + wipe(secret) + if err != nil { + return "", nil, err + } + init, err := appwire.NewInitiator(appwire.InitiatorOptions{StaticKey: kp, RemoteStatic: rs, Prologue: SessionPrologue(rs)}) + if err != nil { + return "", nil, err + } + // A paired device's message 1 carries no payload: the box knows it by + // its static key. + msg1, err := init.WriteMessage(nil) + if err != nil { + return "", nil, err + } + id := make([]byte, 16) + if _, err := rand.Read(id); err != nil { + return "", nil, err + } + sess := base64.RawURLEncoding.EncodeToString(id) + s.mu.Lock() + defer s.mu.Unlock() + if len(s.sessions) >= MaxSessions { + return "", nil, ErrBusy + } + s.sessions[sess] = &session{kid: kid, init: init, used: s.now()} + return sess, msg1, nil +} + +func (s *Server) finish(sess string, msg2 []byte) ([]byte, error) { + var payload []byte + err := s.with(sess, func(x *session) error { + if x.init == nil { + return ErrRefused + } + p, err := x.init.ReadMessage(msg2) + if err != nil { + return ErrRefused + } + split, err := x.init.Split() + if err != nil { + return err + } + x.init, x.tr, payload = nil, appwire.NewTransport(split), p + return nil + }) + if errors.Is(err, ErrRefused) { + // A message 2 that does not read ends the attempt: the handshake + // state cannot take another. + s.end(sess) + } + return payload, err +} + +// with runs fn on one session under its own lock. +func (s *Server) with(sess string, fn func(*session) error) error { + s.mu.Lock() + x := s.sessions[sess] + s.mu.Unlock() + if x == nil { + return ErrUnknownSession + } + x.mu.Lock() + defer x.mu.Unlock() + x.used = s.now() + return fn(x) +} + +func (s *Server) end(sess string) { + s.mu.Lock() + x := s.sessions[sess] + delete(s.sessions, sess) + s.mu.Unlock() + if x != nil { + x.close() + } +} + +func (x *session) close() { + x.mu.Lock() + defer x.mu.Unlock() + if x.tr != nil { + x.tr.Close() + } + x.init, x.tr = nil, nil +} + +// Sweep closes idle sessions and deletes keys whose approval never came. Run +// it every minute or so. +func (s *Server) Sweep() { + now := s.now() + s.mu.Lock() + var idle []*session + for id, x := range s.sessions { + if x.mu.TryLock() { + stale := now.Sub(x.used) > IdleFor + x.mu.Unlock() + if stale { + idle = append(idle, x) + delete(s.sessions, id) + } + } + } + s.mu.Unlock() + for _, x := range idle { + x.close() + } + dir := filepath.Join(s.dir, "pending") + entries, err := os.ReadDir(dir) + if err != nil { + return + } + for _, e := range entries { + info, err := e.Info() + if err == nil && now.Sub(info.ModTime()) > PendingFor { + _ = os.Remove(filepath.Join(dir, e.Name())) + } + } +} + +// Sessions is how many sessions are open, for the log. +func (s *Server) Sessions() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.sessions) +} + +// -------------------------------------------------------------------------- +// Files +// -------------------------------------------------------------------------- + +// writeFile writes a new file whole or not at all. +func writeFile(path string, data []byte) error { + dir := filepath.Dir(path) + tmp, err := os.CreateTemp(dir, ".tmp-*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + if err := tmp.Chmod(0o600); err != nil { + _ = tmp.Close() + return err + } + if _, err := tmp.Write(data); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := os.Rename(tmp.Name(), path); err != nil { + return err + } + return syncDir(dir) +} + +func syncDir(dir string) error { + d, err := os.Open(dir) + if err != nil { + return err + } + defer d.Close() + return d.Sync() +} + +func countDir(dir string) (int, error) { + entries, err := os.ReadDir(dir) + n := 0 + for _, e := range entries { + if !strings.HasPrefix(e.Name(), ".") { + n++ + } + } + return n, err +} diff --git a/internal/keys/wire.go b/internal/keys/wire.go new file mode 100644 index 0000000..aee917b --- /dev/null +++ b/internal/keys/wire.go @@ -0,0 +1,157 @@ +// Package keys is the one part of FTW Cloud that can act on a home. It runs +// as its own process, ftwkeysd, on the same host as ftwcloudd, and answers +// only over a local unix socket. +// +// It holds each agent's X25519 Noise static key, sealed at rest (AWS KMS +// envelope encryption in production, a local file key in development). It +// unseals a key only to open one Noise session with one box, and drops the +// key once the handshake has used it. It keeps the session's transport keys +// and seals and opens each frame of it, so neither an agent's static key nor +// its session keys ever sit in ftwcloudd's memory. +// +// It has no path to a box of its own. It writes and reads Noise messages, +// and ftwcloudd carries them as tun through the box's own uplink, the same +// way it carries a phone's session. The box admits the agent only while its +// signed roster lists the agent's key, and decides what each request may do. +// +// The wire between the two processes is a sequence of frames, each a 4-byte +// big-endian length and one CBOR map: a request {id, op, ...} one way and a +// response {id, err?, ...} the other. Many requests may be in flight on one +// connection. +package keys + +import ( + "encoding/binary" + "errors" + "fmt" + "io" + + "github.com/fxamacker/cbor/v2" +) + +// Operations. +const ( + opNew = "new" // make a key pair; answers kid and pub + opKeep = "keep" // the owner approved the agent: keep its key + opForget = "forget" // delete a key and close its sessions + opOpen = "open" // unseal kid, start a session to rs; answers sess and d (message 1) + opFinish = "finish" // read message 2 in d + opSeal = "seal" // encrypt frame d of sess; answers d + opUnseal = "unseal" // decrypt message d of sess; answers d + opClose = "close" // end sess +) + +// maxFrame bounds one frame between the processes: one Noise transport +// message around the largest bulk frame, and room for the map around it. +const maxFrame = 64 << 10 + +type request struct { + ID uint64 `cbor:"id"` + Op string `cbor:"op"` + Kid string `cbor:"kid,omitempty"` + Sess string `cbor:"sess,omitempty"` + // RS is the box's Noise static key, for open. + RS []byte `cbor:"rs,omitempty"` + D []byte `cbor:"d,omitempty"` +} + +type response struct { + ID uint64 `cbor:"id"` + Err string `cbor:"err,omitempty"` + Kid string `cbor:"kid,omitempty"` + Pub []byte `cbor:"pub,omitempty"` + Sess string `cbor:"sess,omitempty"` + D []byte `cbor:"d,omitempty"` +} + +// Errors a caller can act on. Each crosses the socket as its text. +var ( + // ErrUnknownKey is a kid the keys part does not hold, or holds only + // while its approval is pending. + ErrUnknownKey = errors.New("keys: unknown key") + // ErrUnknownSession is a session the keys part does not hold: it ended, + // went idle, or ftwkeysd restarted. Open a new one. + ErrUnknownSession = errors.New("keys: unknown session") + // ErrBusy is a keys part at its limit of keys or sessions. + ErrBusy = errors.New("keys: busy") + // ErrRefused is a request the keys part will not do as asked. + ErrRefused = errors.New("keys: refused") +) + +var known = []error{ErrUnknownKey, ErrUnknownSession, ErrBusy, ErrRefused} + +func errorFrom(s string) error { + if s == "" { + return nil + } + for _, e := range known { + if e.Error() == s { + return e + } + } + return fmt.Errorf("keys: %s", s) +} + +// errorText is what crosses the socket for err: one of the known errors, or +// a fixed word. Nothing else, so no detail about a key can leave by way of an +// error message. +func errorText(err error) string { + for _, e := range known { + if errors.Is(err, e) { + return e.Error() + } + } + return "failed" +} + +var ( + encMode = mustEnc() + decMode = mustDec() +) + +func mustEnc() cbor.EncMode { + m, err := cbor.CoreDetEncOptions().EncMode() + if err != nil { + panic(err) + } + return m +} + +func mustDec() cbor.DecMode { + m, err := cbor.DecOptions{MaxArrayElements: 16, MaxMapPairs: 16, MaxNestedLevels: 4}.DecMode() + if err != nil { + panic(err) + } + return m +} + +func writeFrame(w io.Writer, v any) error { + body, err := encMode.Marshal(v) + if err != nil { + return err + } + if len(body) > maxFrame { + return fmt.Errorf("keys: a frame of %d bytes", len(body)) + } + buf := make([]byte, 4+len(body)) + binary.BigEndian.PutUint32(buf, uint32(len(body))) + copy(buf[4:], body) + _, err = w.Write(buf) + return err +} + +func readFrame(r io.Reader, v any) error { + var n [4]byte + if _, err := io.ReadFull(r, n[:]); err != nil { + return err + } + size := binary.BigEndian.Uint32(n[:]) + if size > maxFrame { + return fmt.Errorf("keys: a frame of %d bytes", size) + } + body := make([]byte, size) + if _, err := io.ReadFull(r, body); err != nil { + return err + } + return decMode.Unmarshal(body, v) +} diff --git a/scripts/check-imports.sh b/scripts/check-imports.sh index fc0315a..e2e3f5b 100755 --- a/scripts/check-imports.sh +++ b/scripts/check-imports.sh @@ -17,3 +17,13 @@ if [ -n "${reached}" ]; then fi echo "ftw-cloud: only go/protocol from srcfl/ftw ($(go list -deps ./... | grep -c '^github.com/srcfl/ftw/') packages)" + +# Only ftwkeysd talks to KMS. ftwcloudd links no AWS SDK, so a bug in the +# door cannot reach for the agent keys' data keys itself. +aws=$(go list -deps ./cmd/ftwcloudd | grep '^github.com/aws/' || true) +if [ -n "${aws}" ]; then + echo "ftwcloudd links the AWS SDK; only ftwkeysd may:" >&2 + printf '%s\n' "${aws}" >&2 + exit 1 +fi +echo "ftwcloudd: no AWS SDK" From 3952606d991b8358d6bbb920bdf823e8159c184c Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Thu, 15 Oct 2026 19:41:37 +0200 Subject: [PATCH 2/5] feat(keys): an offer key that signs consent offers, and nothing else The agent contract (srcfl/ftw#1642) has FTW Cloud sign each grant offer the consent page hands the owner's app, with a key the app pins. It lives in the keys part, apart from the agent keys: one P-256 key, made at the first start, sealed like an agent key under its own context, kept in memory. It signs only the deterministic CBOR ["ftw.offer.v1", offer]. VerifyOffer is the app's check. Co-Authored-By: Claude Opus 5.5 --- deploy/keys.md | 8 ++ internal/keys/keys_test.go | 61 ++++++++++-- internal/keys/offer.go | 186 +++++++++++++++++++++++++++++++++++++ internal/keys/server.go | 13 ++- internal/keys/wire.go | 1 + 5 files changed, 262 insertions(+), 7 deletions(-) create mode 100644 internal/keys/offer.go diff --git a/deploy/keys.md b/deploy/keys.md index 2e55618..022d68f 100644 --- a/deploy/keys.md +++ b/deploy/keys.md @@ -35,6 +35,14 @@ and turns `ftwkeysd` on. - **Revoking deletes the key.** When the box's roster drops an agent, or it expires, `ftwcloudd` asks `ftwkeysd` to forget the key, which ends its sessions too. +- **The offer key.** One P-256 key signs every grant offer the consent page + hands the owner's app, so the app can tell an offer from FTW Cloud from a + copy. It is made at the first start, sealed like an agent key (its data + key's context names `offer-key`), and kept in memory while `ftwkeysd` + runs. It signs only arrays that start with `ftw.offer.v1`. The app pins + its public half: `ftwkeysd` logs it when it makes the key, and + `ftwcloudd` serves it at `/.well-known/ftw-offer-key`. Losing the volume + makes a new offer key, which the app must then pin anew. - **Nothing logs a key.** Errors on the socket are a few fixed words. - **The volume is not backed up.** Losing it means every assistant must be approved again; no home is exposed by it. diff --git a/internal/keys/keys_test.go b/internal/keys/keys_test.go index 31edde4..cd8cb65 100644 --- a/internal/keys/keys_test.go +++ b/internal/keys/keys_test.go @@ -356,10 +356,17 @@ func TestKMSSealerBindsTheContext(t *testing.T) { t.Fatal(err) } _ = c.Keep(ctx, kid) - for _, blob := range kms.blobs { - if blob.ctx["purpose"] != EncryptionContextPurpose || blob.ctx["kid"] != kid { + var wrapped []byte + for b, blob := range kms.blobs { + if blob.ctx["purpose"] != EncryptionContextPurpose || (blob.ctx["kid"] != kid && blob.ctx["kid"] != offerKid) { t.Fatalf("encryption context %v", blob.ctx) } + if blob.ctx["kid"] == kid { + wrapped = []byte(b) + } + } + if wrapped == nil { + t.Fatal("no data key for the agent key") } before := kms.calls sess, msg1, err := c.Open(ctx, kid, b.static.Public()) @@ -372,10 +379,6 @@ func TestKMSSealerBindsTheContext(t *testing.T) { _, _, _ = b.answer(t, msg1) // The wrapped data key does not open for another kid. - var wrapped []byte - for blob := range kms.blobs { - wrapped = []byte(blob) - } if _, err := sealer.Unwrap(ctx, "AAAAAAAAAAAAAAAAAAAAAA", wrapped); err == nil { t.Fatal("a data key opened under another kid") } @@ -509,3 +512,49 @@ func TestManySessionsAtOnce(t *testing.T) { } wg.Wait() } + +// The offer key: made once, sealed, the same after a restart, and it signs +// only offers. A signature verifies with its public half and nothing else +// does. +func TestTheOfferKeySignsOffers(t *testing.T) { + s, sealer, logs := newTestServer(t) + c := s.Local() + pub, err := c.OfferKey(ctx) + if err != nil || len(pub) != 65 || pub[0] != 4 { + t.Fatalf("offer key: %x %v", pub, err) + } + offer := []byte{0xa1, 0x61, 0x76, 0x01} // {"v": 1} + sig, err := c.SignOffer(ctx, offer) + if err != nil { + t.Fatal(err) + } + if err := VerifyOffer(pub, offer, sig); err != nil { + t.Fatal(err) + } + if VerifyOffer(pub, []byte{0xa1, 0x61, 0x76, 0x02}, sig) == nil { + t.Fatal("a signature verified for another offer") + } + if _, err := c.SignOffer(ctx, nil); !errors.Is(err, ErrRefused) { + t.Fatalf("an empty offer: %v", err) + } + // A restart reads the same key back; it is not on disk in the clear. + s2, err := NewServer(s.dir, sealer, nil) + if err != nil { + t.Fatal(err) + } + pub2, _ := s2.Local().OfferKey(ctx) + if !bytes.Equal(pub, pub2) { + t.Fatal("the offer key changed across a restart") + } + raw, _ := os.ReadFile(filepath.Join(s.dir, "offer.key")) + d, _ := s.offer.Bytes() + if bytes.Contains(raw, d) || bytes.Contains(logs.Bytes(), d) || bytes.Contains(logs.Bytes(), []byte(hex.EncodeToString(d))) { + t.Fatal("the offer key's secret is on disk or in the log") + } + // Its data key is bound to the offer key, not to any agent kid. + var rec offerRecord + _ = decMode.Unmarshal(raw, &rec) + if _, err := sealer.Unwrap(ctx, "AAAAAAAAAAAAAAAAAAAAAA", rec.DEK); err == nil { + t.Fatal("the offer key's data key opens under an agent kid") + } +} diff --git a/internal/keys/offer.go b/internal/keys/offer.go new file mode 100644 index 0000000..3c469bc --- /dev/null +++ b/internal/keys/offer.go @@ -0,0 +1,186 @@ +package keys + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "errors" + "fmt" + "io/fs" + "math/big" + "os" + "path/filepath" +) + +// The offer key. FTW Cloud's consent page hands the owner's app a grant +// offer: the agent's public key and what its client asks for. The app shows +// it before Face ID, so it must know the offer came from FTW Cloud and not +// from a page that copied the look. The keys part signs each offer with one +// P-256 key that the app pins. It lives here, apart from the agent keys and +// sealed the same way, and it signs offers and nothing else: the signed +// bytes always start with OfferTag. + +// OfferTag starts every signed offer array. +const OfferTag = "ftw.offer.v1" + +// offerKid binds the offer key's data key in the sealer's context. It is no +// agent kid's shape, so no agent operation can name it. +const offerKid = "offer-key" + +const ( + opOfferPub = "offer.pub" // the offer key's public half, SEC1 uncompressed + opOfferSign = "offer.sign" // sign offer bytes d; answers r ‖ s +) + +// OfferBytes is what the offer key signs: the deterministic CBOR array +// [OfferTag, offer], where offer is the offer's own CBOR encoding. +func OfferBytes(offer []byte) ([]byte, error) { + return encMode.Marshal([]any{OfferTag, offer}) +} + +// VerifyOffer checks an offer's signature, r ‖ s, against the offer key's +// public half. The app does the same with the key it pins. +func VerifyOffer(pub, offer, sig []byte) error { + key, err := ecdsa.ParseUncompressedPublicKey(elliptic.P256(), pub) + if err != nil || len(sig) != 64 { + return errors.New("keys: not an offer signature") + } + msg, err := OfferBytes(offer) + if err != nil { + return err + } + digest := sha256.Sum256(msg) + if !ecdsa.Verify(key, digest[:], new(big.Int).SetBytes(sig[:32]), new(big.Int).SetBytes(sig[32:])) { + return errors.New("keys: the offer signature does not verify") + } + return nil +} + +// offerRecord is the sealed offer key on disk. +type offerRecord struct { + V int `cbor:"v"` + Pub []byte `cbor:"pub"` + DEK []byte `cbor:"dek"` + CT []byte `cbor:"ct"` +} + +// loadOfferKey unseals the offer key, making it on first start. It stays in +// memory: it signs every consent page, and vouches only for what a page +// shows. +func (s *Server) loadOfferKey() error { + path := filepath.Join(s.dir, "offer.key") + raw, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return s.makeOfferKey(path) + } + if err != nil { + return err + } + var rec offerRecord + if err := decMode.Unmarshal(raw, &rec); err != nil { + return fmt.Errorf("keys: offer key: %w", err) + } + ctx, cancel := context.WithTimeout(context.Background(), unwrapTimeout) + defer cancel() + dek, err := s.sealer.Unwrap(ctx, offerKid, rec.DEK) + if err != nil { + return fmt.Errorf("keys: offer key: %w", err) + } + defer wipe(dek) + aead, err := newGCM(dek) + if err != nil { + return err + } + d, err := open(aead, rec.CT, append([]byte(OfferTag+"\x00"), rec.Pub...)) + if err != nil { + return errors.New("keys: the offer key does not open") + } + defer wipe(d) + key, err := ecdsa.ParseRawPrivateKey(elliptic.P256(), d) + if err != nil { + return err + } + s.offer = key + return nil +} + +func (s *Server) makeOfferKey(path string) error { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return err + } + pub, err := key.PublicKey.Bytes() + if err != nil { + return err + } + d, err := key.Bytes() + if err != nil { + return err + } + defer wipe(d) + ctx, cancel := context.WithTimeout(context.Background(), unwrapTimeout) + defer cancel() + dek, wrapped, err := s.sealer.DataKey(ctx, offerKid) + if err != nil { + return err + } + defer wipe(dek) + aead, err := newGCM(dek) + if err != nil { + return err + } + ct, err := seal(aead, d, append([]byte(OfferTag+"\x00"), pub...)) + if err != nil { + return err + } + raw, err := encMode.Marshal(offerRecord{V: 1, Pub: pub, DEK: wrapped, CT: ct}) + if err != nil { + return err + } + if err := writeFile(path, raw); err != nil { + return err + } + s.offer = key + s.log.Info("made the offer key; pin its public half in the app", "pub", fmt.Sprintf("%x", pub)) + return nil +} + +func (s *Server) offerPub() ([]byte, error) { + if s.offer == nil { + return nil, ErrRefused + } + return s.offer.PublicKey.Bytes() +} + +func (s *Server) signOffer(offer []byte) ([]byte, error) { + if s.offer == nil || len(offer) == 0 || len(offer) > 8<<10 { + return nil, ErrRefused + } + msg, err := OfferBytes(offer) + if err != nil { + return nil, err + } + digest := sha256.Sum256(msg) + r, sv, err := ecdsa.Sign(rand.Reader, s.offer, digest[:]) + if err != nil { + return nil, err + } + sig := make([]byte, 64) + r.FillBytes(sig[:32]) + sv.FillBytes(sig[32:]) + return sig, nil +} + +// OfferKey is the offer key's public half, SEC1 uncompressed. +func (c *Client) OfferKey(ctx context.Context) ([]byte, error) { + resp, err := c.call(ctx, request{Op: opOfferPub}) + return resp.Pub, err +} + +// SignOffer signs one offer's CBOR with the offer key. +func (c *Client) SignOffer(ctx context.Context, offer []byte) ([]byte, error) { + resp, err := c.call(ctx, request{Op: opOfferSign, D: offer}) + return resp.D, err +} diff --git a/internal/keys/server.go b/internal/keys/server.go index 1788a73..69175b9 100644 --- a/internal/keys/server.go +++ b/internal/keys/server.go @@ -3,6 +3,7 @@ package keys import ( "context" "crypto/ecdh" + "crypto/ecdsa" "crypto/rand" "encoding/base64" "errors" @@ -56,6 +57,8 @@ type Server struct { mu sync.Mutex sessions map[string]*session + // offer signs consent offers (offer.go). + offer *ecdsa.PrivateKey } type session struct { @@ -77,7 +80,11 @@ func NewServer(dir string, sealer Sealer, log *slog.Logger) (*Server, error) { return nil, err } } - return &Server{dir: dir, sealer: sealer, log: log, now: time.Now, PeerUID: -1, sessions: map[string]*session{}}, nil + s := &Server{dir: dir, sealer: sealer, log: log, now: time.Now, PeerUID: -1, sessions: map[string]*session{}} + if err := s.loadOfferKey(); err != nil { + return nil, err + } + return s, nil } // Serve accepts connections until ln closes. Each must come from PeerUID. @@ -161,6 +168,10 @@ func (s *Server) handle(req request) response { }) case opClose: s.end(req.Sess) + case opOfferPub: + resp.Pub, err = s.offerPub() + case opOfferSign: + resp.D, err = s.signOffer(req.D) default: err = ErrRefused } diff --git a/internal/keys/wire.go b/internal/keys/wire.go index aee917b..9f18101 100644 --- a/internal/keys/wire.go +++ b/internal/keys/wire.go @@ -39,6 +39,7 @@ const ( opSeal = "seal" // encrypt frame d of sess; answers d opUnseal = "unseal" // decrypt message d of sess; answers d opClose = "close" // end sess + // offer.pub and offer.sign are in offer.go. ) // maxFrame bounds one frame between the processes: one Noise transport From 1c6978d879b45364fe0cd292c0d3d7407aa7c4b7 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Thu, 15 Oct 2026 20:11:31 +0200 Subject: [PATCH 3/5] feat(keys): name the offer key by a kid, and print it for pinning The contract's signed offer is {o, kid, sig}, and Core and the apps pin offer keys by kid. The kid is the first 16 characters of the base64url SHA-256 of the key's SEC1 point. ftwkeysd logs kid and key when it makes the offer key, and `ftwkeysd offerkey` prints them from the same state directory; deploy/keys.md adds the pinning step. Co-Authored-By: Claude Opus 5.5 --- cmd/ftwkeysd/main.go | 97 ++++++++++++++++++++++++++++---------- deploy/keys.md | 17 +++++-- internal/keys/keys_test.go | 7 +++ internal/keys/offer.go | 12 ++++- 4 files changed, 103 insertions(+), 30 deletions(-) diff --git a/cmd/ftwkeysd/main.go b/cmd/ftwkeysd/main.go index a266083..7c33561 100644 --- a/cmd/ftwkeysd/main.go +++ b/cmd/ftwkeysd/main.go @@ -4,11 +4,13 @@ // internal/keys and deploy/keys.md. // // ftwkeysd filekey -out dev.key make a development file key +// ftwkeysd offerkey [flags] print the offer key's kid and public half, making it if need be // ftwkeysd [flags] serve package main import ( "context" + "encoding/base64" "errors" "flag" "fmt" @@ -42,47 +44,94 @@ func main() { } return } + if len(os.Args) > 1 && os.Args[1] == "offerkey" { + if err := offerKey(os.Args[2:]); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + return + } if err := serve(os.Args[1:]); err != nil { slog.Error("ftwkeysd stopped", "err", err) os.Exit(1) } } +// sealerFlags are the flags that choose where keys are sealed. +type sealerFlags struct { + state, kmsKey, region, fileKey *string +} + +func addSealerFlags(fs *flag.FlagSet) sealerFlags { + return sealerFlags{ + state: fs.String("state", "/state/keys", "directory for the sealed keys"), + kmsKey: fs.String("kms-key", "", "AWS KMS key id, ARN or alias that wraps the data keys"), + region: fs.String("region", "eu-north-1", "AWS region of the KMS key"), + fileKey: fs.String("file-key", "", "a local file key instead of KMS (development only)"), + } +} + +func (f sealerFlags) sealer(log *slog.Logger) (keys.Sealer, error) { + switch { + case *f.kmsKey != "" && *f.fileKey != "": + return nil, errors.New("choose -kms-key or -file-key, not both") + case *f.kmsKey != "": + cfg, err := config.LoadDefaultConfig(context.Background(), config.WithRegion(*f.region)) + if err != nil { + return nil, fmt.Errorf("aws config: %w", err) + } + return keys.KMSSealer{KMS: kmsClient{kms.NewFromConfig(cfg)}, KeyID: *f.kmsKey}, nil + case *f.fileKey != "": + s, err := keys.NewFileSealer(*f.fileKey) + if err != nil { + return nil, err + } + log.Warn("agent keys sealed with a local file key; for development only") + return s, nil + } + return nil, errors.New("no sealer: set -kms-key, or -file-key in development") +} + +// offerKey prints the offer key's kid and SEC1 public half, base64url, for +// Core and the apps to pin. It makes the key on first use, in the same state +// directory the server uses. +func offerKey(args []string) error { + fs := flag.NewFlagSet("offerkey", flag.ExitOnError) + sf := addSealerFlags(fs) + _ = fs.Parse(args) + log := slog.New(slog.NewTextHandler(os.Stderr, nil)) + sealer, err := sf.sealer(log) + if err != nil { + return err + } + srv, err := keys.NewServer(*sf.state, sealer, log) + if err != nil { + return err + } + pub, err := srv.Local().OfferKey(context.Background()) + if err != nil { + return err + } + fmt.Printf("kid %s\npub %s\n", keys.OfferKid(pub), base64.RawURLEncoding.EncodeToString(pub)) + return nil +} + func serve(args []string) error { fs := flag.NewFlagSet("ftwkeysd", flag.ExitOnError) socket := fs.String("socket", "/run/ftw-cloud-keys/keys.sock", "unix socket to answer on") - state := fs.String("state", "/state/keys", "directory for the sealed keys") - kmsKey := fs.String("kms-key", "", "AWS KMS key id, ARN or alias that wraps the data keys") - region := fs.String("region", "eu-north-1", "AWS region of the KMS key") - fileKey := fs.String("file-key", "", "a local file key instead of KMS (development only)") + sf := addSealerFlags(fs) peerUID := fs.Int("peer-uid", -1, "the one user allowed to connect; -1 is this process's own") socketMode := fs.Uint("socket-mode", 0o660, "permissions of the socket file") _ = fs.Parse(args) harden() log := slog.New(slog.NewTextHandler(os.Stderr, nil)) - var sealer keys.Sealer - switch { - case *kmsKey != "" && *fileKey != "": - return errors.New("choose -kms-key or -file-key, not both") - case *kmsKey != "": - cfg, err := config.LoadDefaultConfig(context.Background(), config.WithRegion(*region)) - if err != nil { - return fmt.Errorf("aws config: %w", err) - } - sealer = keys.KMSSealer{KMS: kmsClient{kms.NewFromConfig(cfg)}, KeyID: *kmsKey} - case *fileKey != "": - s, err := keys.NewFileSealer(*fileKey) - if err != nil { - return err - } - log.Warn("agent keys sealed with a local file key; for development only") - sealer = s - default: - return errors.New("no sealer: set -kms-key, or -file-key in development") + sealer, err := sf.sealer(log) + if err != nil { + return err } - srv, err := keys.NewServer(*state, sealer, log) + srv, err := keys.NewServer(*sf.state, sealer, log) if err != nil { return err } diff --git a/deploy/keys.md b/deploy/keys.md index 022d68f..e1fdf39 100644 --- a/deploy/keys.md +++ b/deploy/keys.md @@ -39,10 +39,12 @@ and turns `ftwkeysd` on. hands the owner's app, so the app can tell an offer from FTW Cloud from a copy. It is made at the first start, sealed like an agent key (its data key's context names `offer-key`), and kept in memory while `ftwkeysd` - runs. It signs only arrays that start with `ftw.offer.v1`. The app pins - its public half: `ftwkeysd` logs it when it makes the key, and - `ftwcloudd` serves it at `/.well-known/ftw-offer-key`. Losing the volume - makes a new offer key, which the app must then pin anew. + runs. It signs only arrays that start with `ftw.offer.v1`. Core and the + apps pin it by its kid, the first 16 characters of the base64url SHA-256 + of its public half. `ftwkeysd` logs both when it makes the key, + `ftwkeysd offerkey` prints them, and `ftwcloudd` serves them at + `/.well-known/ftw-offer-key`. Losing the volume makes a new offer key, + which Core and the apps must then pin beside the old one. - **Nothing logs a key.** Errors on the socket are a few fixed words. - **The volume is not backed up.** Losing it means every assistant must be approved again; no home is exposed by it. @@ -120,7 +122,12 @@ cannot read the apps' sessions with their boxes. `/etc/ftw-cloud/compose.env`. 5. **Deploy** as usual (`ftw-cloud-deploy sha256:`). Compose starts - `ftwkeysd` beside `ftwcloudd`. Its log says `ftwkeysd listening`. + `ftwkeysd` beside `ftwcloudd`. Its log says `ftwkeysd listening`, and on + its first start, `made the offer key` with the kid and public half. + +6. **Pin the offer key** in Core and the apps (srcfl/ftw `docs/agents.md`, + "The grant offer"): until a build pins it, no offer opens. Read it from + `https://cloud.ftw.energy/.well-known/ftw-offer-key` or the log. To turn assistants off again, remove `COMPOSE_PROFILES=agents` and deploy. Approved agents stay sealed in the volume and work again when it comes back, diff --git a/internal/keys/keys_test.go b/internal/keys/keys_test.go index cd8cb65..4b11b4a 100644 --- a/internal/keys/keys_test.go +++ b/internal/keys/keys_test.go @@ -558,3 +558,10 @@ func TestTheOfferKeySignsOffers(t *testing.T) { t.Fatal("the offer key's data key opens under an agent kid") } } + +func TestOfferKidNamesTheKey(t *testing.T) { + a, b := bytes.Repeat([]byte{4}, 65), bytes.Repeat([]byte{5}, 65) + if len(OfferKid(a)) != 16 || OfferKid(a) == OfferKid(b) || OfferKid(a) != OfferKid(bytes.Clone(a)) { + t.Fatal("kid") + } +} diff --git a/internal/keys/offer.go b/internal/keys/offer.go index 3c469bc..1dce650 100644 --- a/internal/keys/offer.go +++ b/internal/keys/offer.go @@ -6,6 +6,7 @@ import ( "crypto/elliptic" "crypto/rand" "crypto/sha256" + "encoding/base64" "errors" "fmt" "io/fs" @@ -34,6 +35,14 @@ const ( opOfferSign = "offer.sign" // sign offer bytes d; answers r ‖ s ) +// OfferKid names an offer key by its public half: the first 16 characters +// of the base64url SHA-256 of the SEC1 point. The app and the box pin keys +// by it, so a new key can be added beside the old one. +func OfferKid(pub []byte) string { + sum := sha256.Sum256(pub) + return base64.RawURLEncoding.EncodeToString(sum[:])[:16] +} + // OfferBytes is what the offer key signs: the deterministic CBOR array // [OfferTag, offer], where offer is the offer's own CBOR encoding. func OfferBytes(offer []byte) ([]byte, error) { @@ -143,7 +152,8 @@ func (s *Server) makeOfferKey(path string) error { return err } s.offer = key - s.log.Info("made the offer key; pin its public half in the app", "pub", fmt.Sprintf("%x", pub)) + s.log.Info("made the offer key; pin it in Core and the apps", "kid", OfferKid(pub), + "pub", base64.RawURLEncoding.EncodeToString(pub)) return nil } From 5a08a11e50e554755d7172a117b0b42a93a2a77f Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Thu, 15 Oct 2026 22:49:06 +0200 Subject: [PATCH 4/5] fix(keys): review of #18: sign only offers it can vouch for, the offer key in KMS, a role of its own - signOffer decodes the GrantOffer strictly and signs only version 1, the agent role, at most OfferTTLMs ahead, and a key that is a pending kid's. The pin of the agent contract (e2559632) moves here from #19 for it. - The offer key is an asymmetric KMS key (ECC_NIST_P256, SIGN_VERIFY) that only the keys role may sign with. A local offer key and the file sealer exist only in a development build (-tags ftwclouddev). - ftwkeysd leaves the host network for a bridge of its own and uses the ftw-cloud-keys role, which may use the two KMS keys and nothing else. The host renews its credentials into tmpfs every 15 minutes. - Key policies: kms:CreateAlias instead of kms:Create*, an explicit deny on kms:CreateGrant, and keys.md says plainly that PutKeyPolicy lets an admin take use. - ftwcloudd runs as 65530, not distroless's shared nonroot. ftw-cloud-own gives its files to the user its image runs as at every start, and ftw-cloud-deploy keeps the lines keys.sh adds to compose.env. - Pin the offer key from ftwkeysd on the host through SSM, not from the public endpoint. - The socket refuses a connection that is not a unix socket's, and a message 2 that does not split ends its session. Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 9 +- cmd/ftwkeysd/dev_off.go | 13 ++ cmd/ftwkeysd/dev_on.go | 39 ++++ cmd/ftwkeysd/main.go | 210 ++++++++++++++++----- cmd/ftwkeysd/main_test.go | 61 ++++++ deploy/apns.sh | 1 + deploy/bootstrap.sh | 33 +++- deploy/compose.yml | 24 ++- deploy/keys-host-policy.json | 9 +- deploy/keys-kms-policy.json | 15 +- deploy/keys-offer-kms-policy.json | 52 ++++++ deploy/keys-role-policy.json | 20 ++ deploy/keys-role-trust.json | 11 ++ deploy/keys.md | 165 ++++++++++++----- deploy/keys.sh | 96 ++++++++-- deploy/noise.sh | 1 + go.mod | 2 +- go.sum | 4 +- internal/keys/keys_test.go | 171 ++++++++++++++++- internal/keys/offer.go | 299 ++++++++++++++++++++++-------- internal/keys/server.go | 48 +++-- 21 files changed, 1047 insertions(+), 236 deletions(-) create mode 100644 cmd/ftwkeysd/dev_off.go create mode 100644 cmd/ftwkeysd/dev_on.go create mode 100644 cmd/ftwkeysd/main_test.go create mode 100644 deploy/keys-offer-kms-policy.json create mode 100644 deploy/keys-role-policy.json create mode 100644 deploy/keys-role-trust.json diff --git a/Dockerfile b/Dockerfile index 88dafd6..e78cbd3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,10 +30,13 @@ COPY --from=build /out/ftwcloudd /usr/local/bin/ftwcloudd # The keys part, run as its own container and user (deploy/keys.md). COPY --from=build /out/ftwkeysd /usr/local/bin/ftwkeysd COPY --from=build /out/share/source.tar.gz /usr/share/ftw-cloud/source.tar.gz -COPY --from=build --chown=nonroot:nonroot /out/state /state +# ftwcloudd runs as a user of its own, 65530, not distroless's shared +# nonroot. The host gives its files to the user the image names +# (ftw-cloud-own in deploy/bootstrap.sh). +COPY --from=build --chown=65530:65530 /out/state /state # Owned by ftwkeysd's user, so a new volume mounted here starts out its own # and ftwcloudd cannot read it. -COPY --from=build --chown=65533:65532 /out/keys /keys -USER nonroot +COPY --from=build --chown=65533:65533 /out/keys /keys +USER 65530:65530 EXPOSE 443 ENTRYPOINT ["/usr/local/bin/ftwcloudd"] diff --git a/cmd/ftwkeysd/dev_off.go b/cmd/ftwkeysd/dev_off.go new file mode 100644 index 0000000..d8a5116 --- /dev/null +++ b/cmd/ftwkeysd/dev_off.go @@ -0,0 +1,13 @@ +//go:build !ftwclouddev + +package main + +import "flag" + +// A production build has no file key and no local offer key: both keys live +// in KMS. +const devBuild = false + +func addFileKeyFlag(*flag.FlagSet) *string { return new(string) } + +func devCommand(string, []string) bool { return false } diff --git a/cmd/ftwkeysd/dev_on.go b/cmd/ftwkeysd/dev_on.go new file mode 100644 index 0000000..52fa23d --- /dev/null +++ b/cmd/ftwkeysd/dev_on.go @@ -0,0 +1,39 @@ +//go:build ftwclouddev + +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/srcfl/ftw-cloud/internal/keys" +) + +// A development build seals keys with a local file key and signs offers with +// a local key. Whoever reads the file and the state directory has every +// agent key, so a production build has neither. +const devBuild = true + +func addFileKeyFlag(fs *flag.FlagSet) *string { + return fs.String("file-key", "", "a local file key instead of KMS (development only)") +} + +// devCommand runs "ftwkeysd filekey -out dev.key". +func devCommand(name string, args []string) bool { + if name != "filekey" { + return false + } + fs := flag.NewFlagSet("filekey", flag.ExitOnError) + out := fs.String("out", "", "file to write the key to") + _ = fs.Parse(args) + if *out == "" { + fmt.Fprintln(os.Stderr, "filekey: -out is required") + os.Exit(1) + } + if err := keys.MakeFileKey(*out); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + return true +} diff --git a/cmd/ftwkeysd/main.go b/cmd/ftwkeysd/main.go index 7c33561..9bade86 100644 --- a/cmd/ftwkeysd/main.go +++ b/cmd/ftwkeysd/main.go @@ -3,14 +3,17 @@ // as another user, and answers only on a local unix socket. See // internal/keys and deploy/keys.md. // -// ftwkeysd filekey -out dev.key make a development file key -// ftwkeysd offerkey [flags] print the offer key's kid and public half, making it if need be +// ftwkeysd offerkey [flags] print the offer key's kid and public half // ftwkeysd [flags] serve +// +// A development build (-tags ftwclouddev) also takes -file-key and has +// "ftwkeysd filekey -out dev.key"; a production build has neither. package main import ( "context" "encoding/base64" + "encoding/json" "errors" "flag" "fmt" @@ -30,18 +33,7 @@ import ( ) func main() { - if len(os.Args) > 1 && os.Args[1] == "filekey" { - fs := flag.NewFlagSet("filekey", flag.ExitOnError) - out := fs.String("out", "", "file to write the key to") - _ = fs.Parse(os.Args[2:]) - if *out == "" { - fmt.Fprintln(os.Stderr, "filekey: -out is required") - os.Exit(1) - } - if err := keys.MakeFileKey(*out); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } + if len(os.Args) > 1 && devCommand(os.Args[1], os.Args[2:]) { return } if len(os.Args) > 1 && os.Args[1] == "offerkey" { @@ -57,60 +49,129 @@ func main() { } } -// sealerFlags are the flags that choose where keys are sealed. -type sealerFlags struct { - state, kmsKey, region, fileKey *string +// keyFlags are the flags that choose where keys are sealed and what signs +// offers. +type keyFlags struct { + state, kmsKey, offerKMSKey, region, awsCreds, fileKey *string } -func addSealerFlags(fs *flag.FlagSet) sealerFlags { - return sealerFlags{ - state: fs.String("state", "/state/keys", "directory for the sealed keys"), - kmsKey: fs.String("kms-key", "", "AWS KMS key id, ARN or alias that wraps the data keys"), - region: fs.String("region", "eu-north-1", "AWS region of the KMS key"), - fileKey: fs.String("file-key", "", "a local file key instead of KMS (development only)"), +func addKeyFlags(fs *flag.FlagSet) keyFlags { + return keyFlags{ + state: fs.String("state", "/state/keys", "directory for the sealed keys"), + kmsKey: fs.String("kms-key", "", "AWS KMS key id, ARN or alias that wraps the data keys"), + offerKMSKey: fs.String("offer-kms-key", "", "AWS KMS key id, ARN or alias of the offer key (ECC_NIST_P256, SIGN_VERIFY)"), + region: fs.String("region", "eu-north-1", "AWS region of the KMS keys"), + awsCreds: fs.String("aws-credentials", "", + "a JSON file of the keys role's short-lived credentials, as aws sts assume-role answers them; the host refreshes it"), + fileKey: addFileKeyFlag(fs), } } -func (f sealerFlags) sealer(log *slog.Logger) (keys.Sealer, error) { +// aws is the SDK's configuration, made once. +func (f keyFlags) aws(ctx context.Context) (aws.Config, error) { + opts := []func(*config.LoadOptions) error{config.WithRegion(*f.region)} + if *f.awsCreds != "" { + opts = append(opts, config.WithCredentialsProvider(aws.NewCredentialsCache(fileCredentials(*f.awsCreds), + func(o *aws.CredentialsCacheOptions) { o.ExpiryWindow = 5 * time.Minute }))) + } + cfg, err := config.LoadDefaultConfig(ctx, opts...) + if err != nil { + return aws.Config{}, fmt.Errorf("aws config: %w", err) + } + return cfg, nil +} + +// options builds the sealer and the offer key. A production build needs both +// in KMS. +func (f keyFlags) options(ctx context.Context, log *slog.Logger) (keys.Sealer, []keys.Option, error) { + var cfg *aws.Config + awsCfg := func() (aws.Config, error) { + if cfg == nil { + c, err := f.aws(ctx) + if err != nil { + return aws.Config{}, err + } + cfg = &c + } + return *cfg, nil + } + + var sealer keys.Sealer switch { case *f.kmsKey != "" && *f.fileKey != "": - return nil, errors.New("choose -kms-key or -file-key, not both") + return nil, nil, errors.New("choose -kms-key or -file-key, not both") case *f.kmsKey != "": - cfg, err := config.LoadDefaultConfig(context.Background(), config.WithRegion(*f.region)) + c, err := awsCfg() if err != nil { - return nil, fmt.Errorf("aws config: %w", err) + return nil, nil, err } - return keys.KMSSealer{KMS: kmsClient{kms.NewFromConfig(cfg)}, KeyID: *f.kmsKey}, nil + sealer = keys.KMSSealer{KMS: kmsClient{kms.NewFromConfig(c)}, KeyID: *f.kmsKey} case *f.fileKey != "": s, err := keys.NewFileSealer(*f.fileKey) if err != nil { - return nil, err + return nil, nil, err } log.Warn("agent keys sealed with a local file key; for development only") - return s, nil + sealer = s + default: + return nil, nil, errors.New("no sealer: set -kms-key") } - return nil, errors.New("no sealer: set -kms-key, or -file-key in development") + + var opts []keys.Option + switch { + case *f.offerKMSKey != "": + c, err := awsCfg() + if err != nil { + return nil, nil, err + } + tctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + k, err := keys.NewKMSOfferKey(tctx, kmsClient{kms.NewFromConfig(c)}, *f.offerKMSKey) + if err != nil { + return nil, nil, err + } + opts = append(opts, keys.WithOfferKey(k)) + case devBuild: + log.Warn("offers signed by a local key; for development only") + default: + return nil, nil, errors.New("no offer key: set -offer-kms-key") + } + return sealer, opts, nil } // offerKey prints the offer key's kid and SEC1 public half, base64url, for -// Core and the apps to pin. It makes the key on first use, in the same state -// directory the server uses. +// Core and the apps to pin. Run it on the host (deploy/keys.md), so that what +// is pinned comes from KMS and not from a public endpoint. A development +// build prints its local key instead. func offerKey(args []string) error { fs := flag.NewFlagSet("offerkey", flag.ExitOnError) - sf := addSealerFlags(fs) + kf := addKeyFlags(fs) _ = fs.Parse(args) - log := slog.New(slog.NewTextHandler(os.Stderr, nil)) - sealer, err := sf.sealer(log) - if err != nil { - return err - } - srv, err := keys.NewServer(*sf.state, sealer, log) - if err != nil { - return err - } - pub, err := srv.Local().OfferKey(context.Background()) - if err != nil { - return err + ctx := context.Background() + var pub []byte + if *kf.offerKMSKey != "" { + cfg, err := kf.aws(ctx) + if err != nil { + return err + } + k, err := keys.NewKMSOfferKey(ctx, kmsClient{kms.NewFromConfig(cfg)}, *kf.offerKMSKey) + if err != nil { + return err + } + pub = k.Public() + } else { + log := slog.New(slog.NewTextHandler(os.Stderr, nil)) + sealer, opts, err := kf.options(ctx, log) + if err != nil { + return err + } + srv, err := keys.NewServer(*kf.state, sealer, log, opts...) + if err != nil { + return err + } + if pub, err = srv.Local().OfferKey(ctx); err != nil { + return err + } } fmt.Printf("kid %s\npub %s\n", keys.OfferKid(pub), base64.RawURLEncoding.EncodeToString(pub)) return nil @@ -119,19 +180,19 @@ func offerKey(args []string) error { func serve(args []string) error { fs := flag.NewFlagSet("ftwkeysd", flag.ExitOnError) socket := fs.String("socket", "/run/ftw-cloud-keys/keys.sock", "unix socket to answer on") - sf := addSealerFlags(fs) + kf := addKeyFlags(fs) peerUID := fs.Int("peer-uid", -1, "the one user allowed to connect; -1 is this process's own") socketMode := fs.Uint("socket-mode", 0o660, "permissions of the socket file") _ = fs.Parse(args) harden() log := slog.New(slog.NewTextHandler(os.Stderr, nil)) - sealer, err := sf.sealer(log) + sealer, opts, err := kf.options(context.Background(), log) if err != nil { return err } - srv, err := keys.NewServer(*sf.state, sealer, log) + srv, err := keys.NewServer(*kf.state, sealer, log, opts...) if err != nil { return err } @@ -171,11 +232,36 @@ func serve(args []string) error { } } }() - log.Info("ftwkeysd listening", "socket", *socket) + pub, _ := srv.Local().OfferKey(ctx) + log.Info("ftwkeysd listening", "socket", *socket, "offer_kid", keys.OfferKid(pub)) return srv.Serve(ln) } -// kmsClient adapts the AWS SDK to keys.KMS. +// fileCredentials reads the keys role's credentials from the file the host +// refreshes, each time the SDK's cache runs out. +type fileCredentials string + +func (p fileCredentials) Retrieve(context.Context) (aws.Credentials, error) { + raw, err := os.ReadFile(string(p)) + if err != nil { + return aws.Credentials{}, fmt.Errorf("aws credentials: %w", err) + } + var c struct { + AccessKeyID string `json:"AccessKeyId"` + SecretAccessKey string `json:"SecretAccessKey"` + SessionToken string `json:"SessionToken"` + Expiration time.Time `json:"Expiration"` + } + if err := json.Unmarshal(raw, &c); err != nil || c.AccessKeyID == "" || c.SecretAccessKey == "" || c.Expiration.IsZero() { + return aws.Credentials{}, errors.New("aws credentials: the file is not an assume-role answer") + } + return aws.Credentials{ + AccessKeyID: c.AccessKeyID, SecretAccessKey: c.SecretAccessKey, SessionToken: c.SessionToken, + Source: "ftwkeysd-file", CanExpire: true, Expires: c.Expiration, + }, nil +} + +// kmsClient adapts the AWS SDK to keys.KMS and keys.KMSSigner. type kmsClient struct{ c *kms.Client } func (k kmsClient) GenerateDataKey(ctx context.Context, keyID string, ec map[string]string) ([]byte, []byte, error) { @@ -197,3 +283,25 @@ func (k kmsClient) Decrypt(ctx context.Context, keyID string, blob []byte, ec ma } return out.Plaintext, nil } + +func (k kmsClient) PublicKey(ctx context.Context, keyID string) ([]byte, error) { + out, err := k.c.GetPublicKey(ctx, &kms.GetPublicKeyInput{KeyId: aws.String(keyID)}) + if err != nil { + return nil, err + } + if out.KeySpec != types.KeySpecEccNistP256 || out.KeyUsage != types.KeyUsageTypeSignVerify { + return nil, fmt.Errorf("the offer key is %s for %s, want ECC_NIST_P256 for SIGN_VERIFY", out.KeySpec, out.KeyUsage) + } + return out.PublicKey, nil +} + +func (k kmsClient) Sign(ctx context.Context, keyID string, digest []byte) ([]byte, error) { + out, err := k.c.Sign(ctx, &kms.SignInput{ + KeyId: aws.String(keyID), Message: digest, + MessageType: types.MessageTypeDigest, SigningAlgorithm: types.SigningAlgorithmSpecEcdsaSha256, + }) + if err != nil { + return nil, err + } + return out.Signature, nil +} diff --git a/cmd/ftwkeysd/main_test.go b/cmd/ftwkeysd/main_test.go new file mode 100644 index 0000000..9150de2 --- /dev/null +++ b/cmd/ftwkeysd/main_test.go @@ -0,0 +1,61 @@ +package main + +import ( + "context" + "flag" + "io" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// A production build has no file key and no local offer key: both keys must +// be in KMS. +func TestAProductionBuildNeedsBothKeysInKMS(t *testing.T) { + if devBuild { + t.Skip("a development build") + } + parse := func(args ...string) (keyFlags, error) { + fs := flag.NewFlagSet("ftwkeysd", flag.ContinueOnError) + fs.SetOutput(io.Discard) + kf := addKeyFlags(fs) + return kf, fs.Parse(args) + } + if _, err := parse("-file-key=dev.key"); err == nil { + t.Fatal("a production build takes -file-key") + } + if devCommand("filekey", []string{"-out", filepath.Join(t.TempDir(), "k")}) { + t.Fatal("a production build makes file keys") + } + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + kf, _ := parse() + if _, _, err := kf.options(context.Background(), log); err == nil || !strings.Contains(err.Error(), "-kms-key") { + t.Fatalf("no sealer: %v", err) + } + kf, _ = parse("-kms-key=alias/ftw-cloud-agent-keys") + if _, _, err := kf.options(context.Background(), log); err == nil || !strings.Contains(err.Error(), "-offer-kms-key") { + t.Fatalf("no offer key: %v", err) + } +} + +// The keys role's credentials come from the file the host renews, and +// expire when it says. +func TestCredentialsComeFromTheHostsFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "credentials.json") + exp := time.Now().Add(time.Hour).UTC().Truncate(time.Second) + body := `{"AccessKeyId":"ASIAEXAMPLE","SecretAccessKey":"s","SessionToken":"t","Expiration":"` + exp.Format("2006-01-02T15:04:05+00:00") + `"}` + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + c, err := fileCredentials(path).Retrieve(context.Background()) + if err != nil || c.AccessKeyID != "ASIAEXAMPLE" || !c.CanExpire || !c.Expires.Equal(exp) { + t.Fatalf("%+v %v", c, err) + } + _ = os.WriteFile(path, []byte(`{"AccessKeyId":"x"}`), 0o600) + if _, err := fileCredentials(path).Retrieve(context.Background()); err == nil { + t.Fatal("a file without an expiry was taken") + } +} diff --git a/deploy/apns.sh b/deploy/apns.sh index ea8b554..c9a98c0 100755 --- a/deploy/apns.sh +++ b/deploy/apns.sh @@ -43,6 +43,7 @@ if ! get /ftw-cloud/apns/key key.p8 || ! get /ftw-cloud/apns/key-id key-id || ! exit 0 fi grep -q '^-----BEGIN PRIVATE KEY-----$' "$new/key.p8" +# ftw-cloud-own gives it to the user the image runs as, before compose starts. chown -R 65532:65532 "$new" chmod 0400 "$new"/* chmod 0500 "$new" diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index 70a6e86..a8e7506 100644 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -67,6 +67,7 @@ for try in $(seq 30); do sleep 2 done grep -q '^-----BEGIN EC PRIVATE KEY-----$' "$tmp" +# ftw-cloud-own gives it to the user the image runs as, before compose starts. chown 65532:65532 "$tmp" chmod 0400 "$tmp" mv -f "$tmp" /run/ftw-cloud/service.key @@ -101,6 +102,7 @@ Wants=network-online.target Type=oneshot RemainAfterExit=yes WorkingDirectory=/opt/ftw-cloud/deploy +ExecStartPre=/usr/local/sbin/ftw-cloud-own ExecStart=/usr/bin/docker compose --env-file /etc/ftw-cloud/compose.env up -d ExecStop=/usr/bin/docker compose --env-file /etc/ftw-cloud/compose.env down @@ -108,6 +110,32 @@ ExecStop=/usr/bin/docker compose --env-file /etc/ftw-cloud/compose.env down WantedBy=multi-user.target UNIT +# ftwcloudd's files belong to the user its image runs as: the state volume +# and the secrets in tmpfs. Images before 65530 ran as distroless's nonroot, +# 65532, so a deploy either way, or back, finds its files its own. +cat > /usr/local/sbin/ftw-cloud-own <<'OWN' +#!/bin/bash +# Give ftwcloudd's files to the user the configured image runs as. +set -euo pipefail +image=$(sed -n 's/^FTW_CLOUD_IMAGE=//p' /etc/ftw-cloud/compose.env) +[ -n "$image" ] || exit 0 +user=$(docker image inspect -f '{{.Config.User}}' "$image") +case "$user" in + nonroot|nonroot:nonroot) user=65532:65532 ;; +esac +if ! [[ $user =~ ^[0-9]+:[0-9]+$ ]]; then + echo "$image runs as '$user', not uid:gid" >&2 + exit 1 +fi +vol=$(docker volume inspect -f '{{.Mountpoint}}' ftw-cloud_ftw-cloud-state 2>/dev/null || true) +for p in "$vol" /run/ftw-cloud/service.key /run/ftw-cloud/apns /run/ftw-cloud/noise; do + if [ -n "$p" ] && [ -e "$p" ]; then + chown -R "$user" "$p" + fi +done +OWN +chmod 0755 /usr/local/sbin/ftw-cloud-own + # One command updates the host to an image CI built, named by its digest. cat > /usr/local/sbin/ftw-cloud-deploy <<'DEPLOY' #!/bin/bash @@ -142,7 +170,10 @@ if ! [[ $rev =~ ^[0-9a-f]{40}$ ]] || ! git merge-base --is-ancestor "$rev" origi fi git checkout --quiet --detach "$rev" -printf 'FTW_CLOUD_IMAGE=%s\n' "$image" > "$env" +# Only the image line changes; what keys.sh set stays. +touch "$env" +sed -i '/^FTW_CLOUD_IMAGE=/d' "$env" +printf 'FTW_CLOUD_IMAGE=%s\n' "$image" >> "$env" # One transaction, in order: the key comes fresh from Secrets Manager, so a # new version takes effect, then ftw-cloud.service takes the container down # and starts a new one. Always a new container: a file mounted on its own, diff --git a/deploy/compose.yml b/deploy/compose.yml index 2d06f07..96c90e2 100644 --- a/deploy/compose.yml +++ b/deploy/compose.yml @@ -56,7 +56,7 @@ services: # The keys part: the only process that can act on a home, as an agent its # owner approved (deploy/keys.md). It starts with COMPOSE_PROFILES=agents - # in compose.env, once the KMS key exists. + # in compose.env, once the KMS keys and the keys role exist. ftwkeysd: image: ${FTW_CLOUD_IMAGE:?set FTW_CLOUD_IMAGE to an image digest} profiles: [agents] @@ -66,21 +66,26 @@ services: - -socket=/run/ftw-cloud-keys/keys.sock - -state=/keys - -kms-key=${FTW_CLOUD_KMS_KEY:-} + - -offer-kms-key=${FTW_CLOUD_OFFER_KEY:-} - -region=eu-north-1 + - -aws-credentials=/run/secrets/aws/credentials.json # ftwcloudd's user, the only one allowed on the socket. - - -peer-uid=65532 + - -peer-uid=65530 # Its own user, in ftwcloudd's group so that ftwcloudd may connect to # the socket and nothing more. - user: '65533:65532' - # The host's network, because the instance's credentials answer one hop - # away and the bridged containers sit two hops off (deploy/keys.md): - # this is the one container that can reach KMS. It listens on nothing - # but the unix socket. - network_mode: host + user: '65533:65530' + # A network of its own, apart from ftwcloudd's. Like every container it + # cannot reach the instance's credentials (hop limit 1); it reaches KMS + # with the keys role's short-lived credentials, which may use the two + # KMS keys and nothing else. It listens on nothing but the unix socket. + networks: [keys] volumes: # Agent keys, sealed under KMS. ftwcloudd never mounts this. - ftw-cloud-keys:/keys - /run/ftw-cloud-keys:/run/ftw-cloud-keys + # The keys role's credentials, renewed by the host every 15 minutes + # (deploy/keys.sh). + - /run/ftw-cloud-keys-aws:/run/secrets/aws:ro read_only: true ulimits: core: 0 @@ -99,6 +104,9 @@ secrets: ftw-cloud-service.key: file: /run/ftw-cloud/service.key +networks: + keys: + volumes: ftw-cloud-state: ftw-cloud-keys: diff --git a/deploy/keys-host-policy.json b/deploy/keys-host-policy.json index cd7d468..c8ca926 100644 --- a/deploy/keys-host-policy.json +++ b/deploy/keys-host-policy.json @@ -2,13 +2,10 @@ "Version": "2012-10-17", "Statement": [ { - "Sid": "SealAndOpenAgentKeys", + "Sid": "FetchCredentialsForFtwkeysd", "Effect": "Allow", - "Action": ["kms:GenerateDataKey", "kms:Decrypt"], - "Resource": "arn:aws:kms:eu-north-1:976075258475:key/KEY-ID", - "Condition": { - "StringEquals": { "kms:EncryptionContext:purpose": "ftw-agent-key" } - } + "Action": "sts:AssumeRole", + "Resource": "arn:aws:iam::976075258475:role/ftw-cloud-keys" } ] } diff --git a/deploy/keys-kms-policy.json b/deploy/keys-kms-policy.json index bcc2acf..efddf16 100644 --- a/deploy/keys-kms-policy.json +++ b/deploy/keys-kms-policy.json @@ -3,11 +3,11 @@ "Id": "ftw-cloud-agent-keys", "Statement": [ { - "Sid": "AdministerButNeverUse", + "Sid": "AdministerButNotUse", "Effect": "Allow", "Principal": { "AWS": "arn:aws:iam::976075258475:root" }, "Action": [ - "kms:Create*", + "kms:CreateAlias", "kms:Describe*", "kms:Enable*", "kms:List*", @@ -25,15 +25,22 @@ "Resource": "*" }, { - "Sid": "TheHostSealsAndOpensAgentKeys", + "Sid": "OnlyTheKeysRoleSealsAndOpensAgentKeys", "Effect": "Allow", - "Principal": { "AWS": "arn:aws:iam::976075258475:role/ftw-cloud-host" }, + "Principal": { "AWS": "arn:aws:iam::976075258475:role/ftw-cloud-keys" }, "Action": ["kms:GenerateDataKey", "kms:Decrypt"], "Resource": "*", "Condition": { "StringEquals": { "kms:EncryptionContext:purpose": "ftw-agent-key" }, "ForAllValues:StringEquals": { "kms:EncryptionContextKeys": ["purpose", "kid"] } } + }, + { + "Sid": "NoGrants", + "Effect": "Deny", + "Principal": { "AWS": "*" }, + "Action": "kms:CreateGrant", + "Resource": "*" } ] } diff --git a/deploy/keys-offer-kms-policy.json b/deploy/keys-offer-kms-policy.json new file mode 100644 index 0000000..2cfa293 --- /dev/null +++ b/deploy/keys-offer-kms-policy.json @@ -0,0 +1,52 @@ +{ + "Version": "2012-10-17", + "Id": "ftw-cloud-offer-key", + "Statement": [ + { + "Sid": "AdministerButNotSign", + "Effect": "Allow", + "Principal": { "AWS": "arn:aws:iam::976075258475:root" }, + "Action": [ + "kms:CreateAlias", + "kms:Describe*", + "kms:Enable*", + "kms:List*", + "kms:Put*", + "kms:Update*", + "kms:Revoke*", + "kms:Disable*", + "kms:Get*", + "kms:Delete*", + "kms:TagResource", + "kms:UntagResource", + "kms:ScheduleKeyDeletion", + "kms:CancelKeyDeletion" + ], + "Resource": "*" + }, + { + "Sid": "OnlyTheKeysRoleSignsOffers", + "Effect": "Allow", + "Principal": { "AWS": "arn:aws:iam::976075258475:role/ftw-cloud-keys" }, + "Action": "kms:Sign", + "Resource": "*", + "Condition": { + "StringEquals": { "kms:SigningAlgorithm": "ECDSA_SHA_256", "kms:MessageType": "DIGEST" } + } + }, + { + "Sid": "TheKeysRoleReadsThePublicHalf", + "Effect": "Allow", + "Principal": { "AWS": "arn:aws:iam::976075258475:role/ftw-cloud-keys" }, + "Action": "kms:GetPublicKey", + "Resource": "*" + }, + { + "Sid": "NoGrants", + "Effect": "Deny", + "Principal": { "AWS": "*" }, + "Action": "kms:CreateGrant", + "Resource": "*" + } + ] +} diff --git a/deploy/keys-role-policy.json b/deploy/keys-role-policy.json new file mode 100644 index 0000000..6d506c1 --- /dev/null +++ b/deploy/keys-role-policy.json @@ -0,0 +1,20 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "SealAndOpenAgentKeys", + "Effect": "Allow", + "Action": ["kms:GenerateDataKey", "kms:Decrypt"], + "Resource": "arn:aws:kms:eu-north-1:976075258475:key/AGENT-KEY-ID", + "Condition": { + "StringEquals": { "kms:EncryptionContext:purpose": "ftw-agent-key" } + } + }, + { + "Sid": "SignOffers", + "Effect": "Allow", + "Action": ["kms:Sign", "kms:GetPublicKey"], + "Resource": "arn:aws:kms:eu-north-1:976075258475:key/OFFER-KEY-ID" + } + ] +} diff --git a/deploy/keys-role-trust.json b/deploy/keys-role-trust.json new file mode 100644 index 0000000..ead3c7b --- /dev/null +++ b/deploy/keys-role-trust.json @@ -0,0 +1,11 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "TheHostFetchesCredentialsForFtwkeysd", + "Effect": "Allow", + "Principal": { "AWS": "arn:aws:iam::976075258475:role/ftw-cloud-host" }, + "Action": "sts:AssumeRole" + } + ] +} diff --git a/deploy/keys.md b/deploy/keys.md index e1fdf39..4c1075d 100644 --- a/deploy/keys.md +++ b/deploy/keys.md @@ -13,9 +13,9 @@ and turns `ftwkeysd` on. | | `ftwcloudd` | `ftwkeysd` | |---|---|---| -| User | 65532 (`nonroot`) | 65533, group 65532 | -| Network | bridge, 443 published | the host's, listens on nothing | -| Reaches | boxes, phones, assistants | KMS, through the instance role | +| User | 65530, its own (not distroless's shared `nonroot`) | 65533, group 65530 | +| Network | the default bridge, 443 published | a bridge of its own, listens on nothing | +| Reaches | boxes, phones, assistants | KMS, as the `ftw-cloud-keys` role | | State | `ftw-cloud-state` (registry, OAuth token hashes) | `ftw-cloud-keys` (sealed agent keys) | | Socket | connects to `/run/ftw-cloud-keys/keys.sock` | owns it, mode 0660, checks the peer's uid | @@ -34,34 +34,44 @@ and turns `ftwkeysd` on. deleted after an hour. - **Revoking deletes the key.** When the box's roster drops an agent, or it expires, `ftwcloudd` asks `ftwkeysd` to forget the key, which ends its - sessions too. -- **The offer key.** One P-256 key signs every grant offer the consent page - hands the owner's app, so the app can tell an offer from FTW Cloud from a - copy. It is made at the first start, sealed like an agent key (its data - key's context names `offer-key`), and kept in memory while `ftwkeysd` - runs. It signs only arrays that start with `ftw.offer.v1`. Core and the - apps pin it by its kid, the first 16 characters of the base64url SHA-256 - of its public half. `ftwkeysd` logs both when it makes the key, - `ftwkeysd offerkey` prints them, and `ftwcloudd` serves them at - `/.well-known/ftw-offer-key`. Losing the volume makes a new offer key, - which Core and the apps must then pin beside the old one. + sessions too. `ftwcloudd` keeps asking, across restarts, until `ftwkeysd` + has done it. +- **A second KMS key signs offers.** The consent page hands the owner's app a + grant offer, and the app must tell an offer from FTW Cloud from a copy. An + asymmetric KMS key (`ECC_NIST_P256`, `SIGN_VERIFY`) signs every offer with + `kms:Sign`; its private half never leaves KMS. `ftwkeysd` signs only an + offer it can vouch for: a well-formed offer for the agent role, good for at + most ten minutes, whose key is one that `ftwkeysd` made and that waits for + approval. So `ftwcloudd` cannot get an owner to approve a key `ftwcloudd` + holds. Core and the apps pin the offer key by its kid, the first 16 + characters of the base64url SHA-256 of its public half. - **Nothing logs a key.** Errors on the socket are a few fixed words. - **The volume is not backed up.** Losing it means every assistant must be approved again; no home is exposed by it. -### Only `ftwkeysd` reaches KMS +A production build of `ftwkeysd` has no file key and no local offer key. A +development build (`-tags ftwclouddev`) has both; see Development below. -The instance role can call KMS, so the separation rests on who can reach the -role's credentials. The instance metadata service answers IMDSv2 token -requests one network hop away when the hop limit is 1. A container on Docker's -bridge network is two hops away and gets no credentials; `ftwkeysd` uses the -host's network and gets them. `ftwcloudd` needs no AWS credentials: the host -fetches its secrets before Docker starts. +### Only `ftwkeysd` reaches KMS, and it reaches nothing else -The KMS key's policy lets only the host role use it, and only with the -`ftw-agent-key` purpose. The account root can administer the key but not use -it, so another role in the shared account cannot be granted decrypt by IAM -alone. +The instance role reads the service key and the parameters at boot, so no +container may hold its credentials. The instance metadata service answers +IMDSv2 token requests one network hop away when the hop limit is 1; every +container sits on a Docker bridge, two hops away, and gets nothing. + +`ftwkeysd` uses a role of its own, `ftw-cloud-keys`, which may use the two KMS +keys and nothing else. The host assumes it every 15 minutes and leaves the +hour-long credentials in tmpfs, readable only by `ftwkeysd`'s user, which +mounts them read-only. + +Each KMS key's policy lets only `ftw-cloud-keys` use it: the agent key only +with the `ftw-agent-key` purpose, the offer key only to sign SHA-256 digests +and read its public half. An explicit deny on `kms:CreateGrant` means no grant +opens either key beside its policy. The account root may administer the keys +but not use them, so another role in the shared account gains nothing from +IAM alone. This guards against mistakes; it is not a wall: whoever may put a +key's policy (`kms:PutKeyPolicy`) can write themselves in, and that change +shows in CloudTrail. If the cloud is fully broken into, the attacker can act as the existing agents, within the limits their boxes enforce, until the owners revoke them @@ -70,30 +80,60 @@ cannot read the apps' sessions with their boxes. ## Turning it on -1. **The KMS key** (once, from a workstation with admin rights): +0. **The host's tooling.** This image runs `ftwcloudd` as 65530, and + `ftw-cloud-deploy` now keeps the lines `keys.sh` adds to `compose.env`. + Re-run [`bootstrap.sh`](bootstrap.sh) through SSM before deploying it; it + installs `ftw-cloud-own`, which gives `ftwcloudd`'s state and secrets to + the user its image runs as at every start, so deploying this image, or + going back to an older one, finds its files its own. + +1. **The keys role** (once, from a workstation with admin rights): ```bash - key=$(aws kms create-key --region eu-north-1 \ + aws iam create-role --role-name ftw-cloud-keys \ + --assume-role-policy-document file://deploy/keys-role-trust.json \ + --max-session-duration 3600 --tags Key=Project,Value=ftw-cloud + aws iam put-role-policy --role-name ftw-cloud-host --policy-name ftw-cloud-keys-assume \ + --policy-document file://deploy/keys-host-policy.json + ``` + +2. **The two KMS keys:** + + ```bash + agent=$(aws kms create-key --region eu-north-1 \ --description 'FTW Cloud agent keys (ftwkeysd)' \ --key-spec SYMMETRIC_DEFAULT --key-usage ENCRYPT_DECRYPT \ --policy file://deploy/keys-kms-policy.json \ --tags TagKey=Project,TagValue=ftw-cloud \ --query KeyMetadata.Arn --output text) aws kms create-alias --region eu-north-1 --alias-name alias/ftw-cloud-agent-keys \ - --target-key-id "$key" - aws kms enable-key-rotation --region eu-north-1 --key-id "$key" - echo "$key" + --target-key-id "$agent" + aws kms enable-key-rotation --region eu-north-1 --key-id "$agent" + + offer=$(aws kms create-key --region eu-north-1 \ + --description 'FTW Cloud grant offers (ftwkeysd)' \ + --key-spec ECC_NIST_P256 --key-usage SIGN_VERIFY \ + --policy file://deploy/keys-offer-kms-policy.json \ + --tags TagKey=Project,TagValue=ftw-cloud \ + --query KeyMetadata.Arn --output text) + aws kms create-alias --region eu-north-1 --alias-name alias/ftw-cloud-offer-key \ + --target-key-id "$offer" + echo "$agent $offer" ``` -2. **The role's policy.** Put the key's id into - [`keys-host-policy.json`](keys-host-policy.json) in place of `KEY-ID`, then: + An asymmetric key does not rotate. A new offer key is a new key, pinned + beside the old one until every build has it. + +3. **The role's policy.** Put the two keys' ids into + [`keys-role-policy.json`](keys-role-policy.json) in place of `AGENT-KEY-ID` + and `OFFER-KEY-ID`, then: ```bash - aws iam put-role-policy --role-name ftw-cloud-host --policy-name ftw-cloud-agent-keys \ - --policy-document file://deploy/keys-host-policy.json + aws iam put-role-policy --role-name ftw-cloud-keys --policy-name ftw-cloud-keys \ + --policy-document file://deploy/keys-role-policy.json ``` -3. **One hop to the metadata service**, so no bridged container gets the +4. **One hop to the metadata service**, so no container gets the instance role's credentials: ```bash @@ -107,27 +147,49 @@ cannot read the apps' sessions with their boxes. http://169.254.169.254/latest/api/token -H 'X-aws-ec2-metadata-token-ttl-seconds: 60'` must time out or fail. -4. **The host.** Through SSM, with the key's ARN: +5. **The host.** Through SSM, with both keys' ARNs: ```bash aws ssm send-command --region eu-north-1 --instance-ids i-0ecac4cbc6b5f8adc \ --document-name AWS-RunShellScript \ - --parameters "$(jq -n --rawfile s deploy/keys.sh --arg k "$key" \ - '{commands: ["set -- \($k)", $s]}')" + --parameters "$(jq -n --rawfile s deploy/keys.sh --arg a "$agent" --arg o "$offer" \ + '{commands: ["set -- \($a) \($o)", $s]}')" ``` SSM runs the lines as one script, so `set --` hands the script its - argument. It makes `/run/ftw-cloud-keys` at every boot and sets - `FTW_CLOUD_KMS_KEY` and `COMPOSE_PROFILES=agents` in + arguments. It makes `/run/ftw-cloud-keys` at every boot, installs the + timer that renews the keys role's credentials in + `/run/ftw-cloud-keys-aws`, and sets `FTW_CLOUD_KMS_KEY`, + `FTW_CLOUD_OFFER_KEY` and `COMPOSE_PROFILES=agents` in `/etc/ftw-cloud/compose.env`. -5. **Deploy** as usual (`ftw-cloud-deploy sha256:`). Compose starts - `ftwkeysd` beside `ftwcloudd`. Its log says `ftwkeysd listening`, and on - its first start, `made the offer key` with the kid and public half. +6. **Deploy** as usual (`ftw-cloud-deploy sha256:`). Compose starts + `ftwkeysd` beside `ftwcloudd`. Its log says `ftwkeysd listening` with the + offer key's kid. + +7. **Pin the offer key** in Core and the apps (srcfl/ftw `docs/agents.md`, + "The grant offer"): until a build pins it, no offer opens. Take the kid and + public half from `ftwkeysd` on the host, through SSM, not from the public + `/.well-known/ftw-offer-key`: + + ```bash + id=$(aws ssm send-command --region eu-north-1 --instance-ids i-0ecac4cbc6b5f8adc \ + --document-name AWS-RunShellScript \ + --parameters 'commands=["cd /opt/ftw-cloud/deploy && docker compose --env-file /etc/ftw-cloud/compose.env exec -T ftwkeysd /usr/local/bin/ftwkeysd offerkey -offer-kms-key=alias/ftw-cloud-offer-key -aws-credentials=/run/secrets/aws/credentials.json"]' \ + --query Command.CommandId --output text) + aws ssm get-command-invocation --region eu-north-1 --instance-id i-0ecac4cbc6b5f8adc \ + --command-id "$id" --query StandardOutputContent --output text + ``` + + It prints `kid …` and `pub …`. KMS gives the same public half to an + admin, as a cross-check that needs no host: -6. **Pin the offer key** in Core and the apps (srcfl/ftw `docs/agents.md`, - "The grant offer"): until a build pins it, no offer opens. Read it from - `https://cloud.ftw.energy/.well-known/ftw-offer-key` or the log. + ```bash + aws kms get-public-key --region eu-north-1 --key-id alias/ftw-cloud-offer-key \ + --query PublicKey --output text | python3 -c 'import base64,hashlib,sys + p=base64.b64decode(sys.stdin.read())[-65:]; b=lambda x: base64.urlsafe_b64encode(x).rstrip(b"=").decode() + print("kid", b(hashlib.sha256(p).digest())[:16]); print("pub", b(p))' + ``` To turn assistants off again, remove `COMPOSE_PROFILES=agents` and deploy. Approved agents stay sealed in the volume and work again when it comes back, @@ -136,9 +198,12 @@ unless their owners revoked them meanwhile. ## Development ```bash -go run ./cmd/ftwkeysd filekey -out dev-keys.key -go run ./cmd/ftwkeysd -socket /tmp/ftwkeys/keys.sock -state /tmp/ftwkeys/state -file-key dev-keys.key +go run -tags ftwclouddev ./cmd/ftwkeysd filekey -out dev-keys.key +go run -tags ftwclouddev ./cmd/ftwkeysd -socket /tmp/ftwkeys/keys.sock \ + -state /tmp/ftwkeys/state -file-key dev-keys.key ``` -A file key is for development only: whoever reads it and the state directory -has every agent key. +A development build seals keys with the file key and signs offers with a +local key in the state directory. Whoever reads the file and the state +directory has every agent key, so a production build refuses both: it needs +`-kms-key` and `-offer-kms-key`. diff --git a/deploy/keys.sh b/deploy/keys.sh index ad8a8d1..f6d6677 100755 --- a/deploy/keys.sh +++ b/deploy/keys.sh @@ -1,27 +1,97 @@ #!/bin/bash # Prepare the FTW Cloud host for ftwkeysd, the keys part. Run as root on the -# host through SSM, after the KMS key exists (deploy/keys.md). Safe to re-run. +# host through SSM, after the two KMS keys and the keys role exist +# (deploy/keys.md). Safe to re-run. # -# It makes the socket's directory at every boot, owned by ftwkeysd's user -# and readable by ftwcloudd's group, and turns ftwkeysd on in compose.env. -# It touches no AWS resource. +# It makes the socket's directory at every boot, keeps short-lived +# credentials of the keys role in tmpfs for ftwkeysd, and turns ftwkeysd on +# in compose.env. It touches no AWS resource. set -euo pipefail -key_arn="${1:?usage: keys.sh }" -case "$key_arn" in - arn:aws:kms:eu-north-1:*:key/*) ;; - *) echo "not a KMS key ARN in eu-north-1: $key_arn" >&2; exit 1 ;; -esac +agent_key="${1:?usage: keys.sh }" +offer_key="${2:?usage: keys.sh }" +for arn in "$agent_key" "$offer_key"; do + case "$arn" in + arn:aws:kms:eu-north-1:*:key/*) ;; + *) echo "not a KMS key ARN in eu-north-1: $arn" >&2; exit 1 ;; + esac +done -# ftwkeysd runs as 65533 in ftwcloudd's group, 65532. The directory is the +# ftwkeysd runs as 65533 in ftwcloudd's group, 65530. The directory is the # only way to the socket: ftwcloudd may enter it and connect, and nothing # else on the host but root may. cat > /etc/tmpfiles.d/ftw-cloud-keys.conf <<'TMPFILES' -d /run/ftw-cloud-keys 0750 65533 65532 - +d /run/ftw-cloud-keys 0750 65533 65530 - +d /run/ftw-cloud-keys-aws 0700 65533 65533 - TMPFILES systemd-tmpfiles --create /etc/tmpfiles.d/ftw-cloud-keys.conf +# No container reaches the instance's credentials (hop limit 1), and the +# instance role may do more than ftwkeysd needs. So the host assumes the +# keys role, which may only use the two KMS keys, and leaves its credentials +# for ftwkeysd in tmpfs. They last an hour and are renewed every 15 minutes. +cat > /usr/local/sbin/ftw-cloud-keys-aws <<'FETCH' +#!/bin/bash +# The keys role's credentials into tmpfs for ftwkeysd. +set -euo pipefail +export AWS_PAGER="" +umask 077 +dir=/run/ftw-cloud-keys-aws +install -d -m 0700 -o 65533 -g 65533 "$dir" +tmp=$(mktemp "$dir/.credentials.XXXXXX") +trap 'rm -f "$tmp"' EXIT +# At boot the network and the instance's credentials may lag a little. +for try in $(seq 30); do + if aws sts assume-role --region eu-north-1 \ + --role-arn arn:aws:iam::976075258475:role/ftw-cloud-keys \ + --role-session-name ftwkeysd --duration-seconds 3600 \ + --query Credentials --output json >"$tmp"; then + break + fi + [ "$try" -lt 30 ] || exit 1 + sleep 2 +done +chown 65533:65533 "$tmp" +chmod 0400 "$tmp" +mv -f "$tmp" "$dir/credentials.json" +trap - EXIT +FETCH +chmod 0755 /usr/local/sbin/ftw-cloud-keys-aws + +cat > /etc/systemd/system/ftw-cloud-keys-aws.service <<'UNIT' +[Unit] +Description=FTW Cloud keys role credentials into tmpfs for ftwkeysd +Wants=network-online.target +After=network-online.target +Before=docker.service + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/ftw-cloud-keys-aws + +[Install] +WantedBy=docker.service +UNIT + +cat > /etc/systemd/system/ftw-cloud-keys-aws.timer <<'UNIT' +[Unit] +Description=Renew the FTW Cloud keys role credentials + +[Timer] +OnActiveSec=15min +OnUnitActiveSec=15min +AccuracySec=1min + +[Install] +WantedBy=timers.target +UNIT + +systemctl daemon-reload +systemctl enable ftw-cloud-keys-aws.service +systemctl enable --now ftw-cloud-keys-aws.timer +systemctl start ftw-cloud-keys-aws.service + env=/etc/ftw-cloud/compose.env touch "$env" -sed -i '/^FTW_CLOUD_KMS_KEY=/d; /^COMPOSE_PROFILES=/d' "$env" -printf 'FTW_CLOUD_KMS_KEY=%s\nCOMPOSE_PROFILES=agents\n' "$key_arn" >> "$env" +sed -i '/^FTW_CLOUD_KMS_KEY=/d; /^FTW_CLOUD_OFFER_KEY=/d; /^COMPOSE_PROFILES=/d' "$env" +printf 'FTW_CLOUD_KMS_KEY=%s\nFTW_CLOUD_OFFER_KEY=%s\nCOMPOSE_PROFILES=agents\n' "$agent_key" "$offer_key" >> "$env" echo "ftwkeysd is on; the next ftw-cloud-deploy (or a restart of ftw-cloud.service) starts it" diff --git a/deploy/noise.sh b/deploy/noise.sh index 0b62b70..7ba2c5b 100755 --- a/deploy/noise.sh +++ b/deploy/noise.sh @@ -35,6 +35,7 @@ for try in $(seq 30); do sleep 2 done grep -q '^-----BEGIN PRIVATE KEY-----$' "$new/noise.key" +# ftw-cloud-own gives it to the user the image runs as, before compose starts. chown -R 65532:65532 "$new" chmod 0400 "$new/noise.key" chmod 0500 "$new" diff --git a/go.mod b/go.mod index 60a74e4..66184cb 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/kms v1.61.4 github.com/fxamacker/cbor/v2 v2.9.4 github.com/gorilla/websocket v1.5.3 - github.com/srcfl/ftw/go/protocol v0.0.0-20261015164244-f5cdb92058c1 + github.com/srcfl/ftw/go/protocol v0.0.0-20261015184016-e25596322c66 golang.org/x/crypto v0.57.0 golang.org/x/sys v0.48.0 modernc.org/sqlite v1.59.0 diff --git a/go.sum b/go.sum index 68aa95a..1c9ac7b 100644 --- a/go.sum +++ b/go.sum @@ -46,8 +46,8 @@ github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOF github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/srcfl/ftw/go/protocol v0.0.0-20261015164244-f5cdb92058c1 h1:JLItMdT1tX3ejgmW80VXcwZ77wBPY7WT2ewMGfPLDIc= -github.com/srcfl/ftw/go/protocol v0.0.0-20261015164244-f5cdb92058c1/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= +github.com/srcfl/ftw/go/protocol v0.0.0-20261015184016-e25596322c66 h1:ZFXkeVj4a828TZ5U9c4q7iC2IzLMf9pLSmcKS3gBpsk= +github.com/srcfl/ftw/go/protocol v0.0.0-20261015184016-e25596322c66/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= diff --git a/internal/keys/keys_test.go b/internal/keys/keys_test.go index 4b11b4a..a8f9b8c 100644 --- a/internal/keys/keys_test.go +++ b/internal/keys/keys_test.go @@ -3,6 +3,10 @@ package keys import ( "bytes" "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" "encoding/base64" "encoding/hex" "errors" @@ -15,6 +19,7 @@ import ( "testing" "time" + "github.com/srcfl/ftw/go/protocol" "github.com/srcfl/ftw/go/protocol/appwire" ) @@ -513,30 +518,92 @@ func TestManySessionsAtOnce(t *testing.T) { wg.Wait() } -// The offer key: made once, sealed, the same after a restart, and it signs -// only offers. A signature verifies with its public half and nothing else -// does. -func TestTheOfferKeySignsOffers(t *testing.T) { +// offerFor is a well-formed offer for key pub, as FTW Cloud makes one. +func offerFor(t *testing.T, pub []byte, edit func(*protocol.GrantOffer)) []byte { + t.Helper() + o := protocol.GrantOffer{ + V: protocol.GrantOfferVersion, Key: pub, Role: protocol.RoleAgent, + Client: "Claude", ClientID: "https://claude.ai/oauth/mcp-oauth-client-metadata", ClientHost: "claude.ai", + Scopes: []string{"ftw.live.read"}, TTLS: 3600, Ticket: "t", Code: "ABCD-EFGH", + ExpMs: time.Now().Add(5 * time.Minute).UnixMilli(), + } + if edit != nil { + edit(&o) + } + raw, err := encMode.Marshal(o) + if err != nil { + t.Fatal(err) + } + return raw +} + +// The offer key: made once, sealed, the same after a restart. It signs an +// offer only for a key that the keys part made and that waits for approval, +// for the agent role and for no longer than an offer may live. What it signs +// opens with the box's own check. +func TestTheOfferKeySignsOnlyOffersItCanVouchFor(t *testing.T) { s, sealer, logs := newTestServer(t) c := s.Local() pub, err := c.OfferKey(ctx) if err != nil || len(pub) != 65 || pub[0] != 4 { t.Fatalf("offer key: %x %v", pub, err) } - offer := []byte{0xa1, 0x61, 0x76, 0x01} // {"v": 1} - sig, err := c.SignOffer(ctx, offer) + kid, agent, _ := c.New(ctx) + other, otherPub, _ := c.New(ctx) + offer := offerFor(t, agent, nil) + sig, err := c.SignOffer(ctx, kid, offer) if err != nil { t.Fatal(err) } if err := VerifyOffer(pub, offer, sig); err != nil { t.Fatal(err) } - if VerifyOffer(pub, []byte{0xa1, 0x61, 0x76, 0x02}, sig) == nil { + so := protocol.SignedOffer{O: offer, Kid: OfferKid(pub), Sig: sig} + if _, err := protocol.OpenOffer(map[string][]byte{OfferKid(pub): pub}, so, time.Now().UnixMilli()); err != nil { + t.Fatalf("the box does not open the offer: %v", err) + } + if VerifyOffer(pub, offerFor(t, agent, func(o *protocol.GrantOffer) { o.Code = "OTHER" }), sig) == nil { t.Fatal("a signature verified for another offer") } - if _, err := c.SignOffer(ctx, nil); !errors.Is(err, ErrRefused) { - t.Fatalf("an empty offer: %v", err) + + type extra struct { + protocol.GrantOffer + X int `cbor:"x"` + } + withExtra, _ := encMode.Marshal(extra{GrantOffer: protocol.GrantOffer{ + V: 1, Key: agent, Role: protocol.RoleAgent, ExpMs: time.Now().Add(time.Minute).UnixMilli()}, X: 1}) + refused := map[string]struct { + kid string + offer []byte + }{ + "empty": {kid, nil}, + "not CBOR": {kid, []byte("{}")}, + "an unknown field": {kid, withExtra}, + "another pending key": {other, offer}, + "another key's offer": {kid, offerFor(t, otherPub, nil)}, + "a made-up key": {kid, offerFor(t, bytes.Repeat([]byte{1}, 32), nil)}, + "no kid": {"", offer}, + "the support role": {kid, offerFor(t, agent, func(o *protocol.GrantOffer) { o.Role = protocol.RoleSupport })}, + "the owner role": {kid, offerFor(t, agent, func(o *protocol.GrantOffer) { o.Role = protocol.RoleOwner })}, + "another version": {kid, offerFor(t, agent, func(o *protocol.GrantOffer) { o.V = 2 })}, + "an expired offer": {kid, offerFor(t, agent, func(o *protocol.GrantOffer) { o.ExpMs = time.Now().UnixMilli() - 1 })}, + "an offer for too long": {kid, offerFor(t, agent, func(o *protocol.GrantOffer) { + o.ExpMs = time.Now().UnixMilli() + protocol.OfferTTLMs + 60_000 + })}, + } + for name, r := range refused { + if _, err := c.SignOffer(ctx, r.kid, r.offer); !errors.Is(err, ErrRefused) { + t.Errorf("%s: %v", name, err) + } } + // A key the owner approved is no longer offered. + if err := c.Keep(ctx, kid); err != nil { + t.Fatal(err) + } + if _, err := c.SignOffer(ctx, kid, offer); !errors.Is(err, ErrRefused) { + t.Fatalf("a kept key's offer: %v", err) + } + // A restart reads the same key back; it is not on disk in the clear. s2, err := NewServer(s.dir, sealer, nil) if err != nil { @@ -547,7 +614,7 @@ func TestTheOfferKeySignsOffers(t *testing.T) { t.Fatal("the offer key changed across a restart") } raw, _ := os.ReadFile(filepath.Join(s.dir, "offer.key")) - d, _ := s.offer.Bytes() + d, _ := s.offer.(*localOfferKey).key.Bytes() if bytes.Contains(raw, d) || bytes.Contains(logs.Bytes(), d) || bytes.Contains(logs.Bytes(), []byte(hex.EncodeToString(d))) { t.Fatal("the offer key's secret is on disk or in the log") } @@ -559,6 +626,90 @@ func TestTheOfferKeySignsOffers(t *testing.T) { } } +// fakeSigner stands in for an asymmetric KMS key. +type fakeSigner struct { + key *ecdsa.PrivateKey + wrong bool + calls int +} + +func (f *fakeSigner) PublicKey(_ context.Context, keyID string) ([]byte, error) { + if keyID != "alias/ftw-cloud-offer-key" { + return nil, errors.New("NotFoundException") + } + return x509.MarshalPKIXPublicKey(&f.key.PublicKey) +} + +func (f *fakeSigner) Sign(_ context.Context, _ string, digest []byte) ([]byte, error) { + f.calls++ + if f.wrong { + digest = bytes.Repeat([]byte{1}, 32) + } + return ecdsa.SignASN1(rand.Reader, f.key, digest) +} + +// In production the offer key is in KMS: the keys part asks KMS to sign, +// checks the answer and never holds the private half. +func TestTheOfferKeyCanLiveInKMS(t *testing.T) { + priv, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + signer := &fakeSigner{key: priv} + k, err := NewKMSOfferKey(ctx, signer, "alias/ftw-cloud-offer-key") + if err != nil { + t.Fatal(err) + } + sealer, _ := FileSealerFromKey(bytes.Repeat([]byte{7}, dataKeyBytes)) + s, err := NewServer(t.TempDir(), sealer, nil, WithOfferKey(k)) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(s.dir, "offer.key")); !errors.Is(err, os.ErrNotExist) { + t.Fatal("a local offer key was made beside the KMS one") + } + c := s.Local() + pub, _ := c.OfferKey(ctx) + want, _ := priv.PublicKey.Bytes() + if !bytes.Equal(pub, want) { + t.Fatal("the public half is not KMS's") + } + kid, agent, _ := c.New(ctx) + offer := offerFor(t, agent, nil) + sig, err := c.SignOffer(ctx, kid, offer) + if err != nil || VerifyOffer(pub, offer, sig) != nil || signer.calls != 1 { + t.Fatalf("signed through KMS: %v (%d calls)", err, signer.calls) + } + // A refused offer never reaches KMS, and a wrong answer from KMS is not + // passed on. + _, _ = c.SignOffer(ctx, kid, offerFor(t, agent, func(o *protocol.GrantOffer) { o.Role = protocol.RoleSupport })) + if signer.calls != 1 { + t.Fatal("a refused offer reached KMS") + } + signer.wrong = true + if _, err := c.SignOffer(ctx, kid, offer); err == nil { + t.Fatal("a signature that does not verify was passed on") + } + if _, err := NewKMSOfferKey(ctx, signer, "alias/another"); err == nil { + t.Fatal("a missing key was taken") + } +} + +// Only a unix socket's peer is served. +func TestServeRefusesAConnectionThatIsNotUnix(t *testing.T) { + s, _, _ := newTestServer(t) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + go func() { _ = s.Serve(ln) }() + c := newClient(func() (net.Conn, error) { return net.Dial("tcp", ln.Addr().String()) }) + defer c.Close() + tctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + if _, _, err := c.New(tctx); err == nil { + t.Fatal("a TCP peer was served") + } +} + func TestOfferKidNamesTheKey(t *testing.T) { a, b := bytes.Repeat([]byte{4}, 65), bytes.Repeat([]byte{5}, 65) if len(OfferKid(a)) != 16 || OfferKid(a) == OfferKid(b) || OfferKid(a) != OfferKid(bytes.Clone(a)) { diff --git a/internal/keys/offer.go b/internal/keys/offer.go index 1dce650..33f5f8b 100644 --- a/internal/keys/offer.go +++ b/internal/keys/offer.go @@ -1,53 +1,73 @@ package keys import ( + "bytes" "context" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" "crypto/sha256" + "crypto/x509" + "encoding/asn1" "encoding/base64" "errors" "fmt" "io/fs" + "log/slog" "math/big" "os" "path/filepath" + + "github.com/fxamacker/cbor/v2" + "github.com/srcfl/ftw/go/protocol" ) // The offer key. FTW Cloud's consent page hands the owner's app a grant // offer: the agent's public key and what its client asks for. The app shows // it before Face ID, so it must know the offer came from FTW Cloud and not // from a page that copied the look. The keys part signs each offer with one -// P-256 key that the app pins. It lives here, apart from the agent keys and -// sealed the same way, and it signs offers and nothing else: the signed -// bytes always start with OfferTag. +// P-256 key that Core and the apps pin. +// +// In production the key is an AWS KMS key of its own (ECC_NIST_P256, +// SIGN_VERIFY): its private half never leaves KMS, and only ftwkeysd's role +// may sign with it. In development and tests it is a local P-256 key, sealed +// in the state directory like an agent key. +// +// The keys part signs only an offer it can vouch for: a well-formed +// protocol.GrantOffer for the agent role, short-lived, whose key is one it +// made and that is still waiting for approval. So ftwcloudd cannot get an +// owner to approve a key that ftwcloudd holds. // OfferTag starts every signed offer array. -const OfferTag = "ftw.offer.v1" +const OfferTag = protocol.OfferTag -// offerKid binds the offer key's data key in the sealer's context. It is no -// agent kid's shape, so no agent operation can name it. +// offerKid binds the local offer key's data key in the sealer's context. It +// is no agent kid's shape, so no agent operation can name it. const offerKid = "offer-key" const ( opOfferPub = "offer.pub" // the offer key's public half, SEC1 uncompressed - opOfferSign = "offer.sign" // sign offer bytes d; answers r ‖ s + opOfferSign = "offer.sign" // sign offer d for pending key kid; answers r ‖ s ) -// OfferKid names an offer key by its public half: the first 16 characters -// of the base64url SHA-256 of the SEC1 point. The app and the box pin keys -// by it, so a new key can be added beside the old one. -func OfferKid(pub []byte) string { - sum := sha256.Sum256(pub) - return base64.RawURLEncoding.EncodeToString(sum[:])[:16] +// maxOffer bounds one offer's CBOR. +const maxOffer = 8 << 10 + +// An OfferSigner holds the offer key. +type OfferSigner interface { + // Public is the key's public half, SEC1 uncompressed. + Public() []byte + // Sign signs a SHA-256 digest and answers r ‖ s. + Sign(ctx context.Context, digest []byte) ([]byte, error) } +// OfferKid names an offer key by its public half (protocol.OfferKid). Core +// and the apps pin keys by it, so a new key can be added beside the old one. +func OfferKid(pub []byte) string { return protocol.OfferKid(pub) } + // OfferBytes is what the offer key signs: the deterministic CBOR array // [OfferTag, offer], where offer is the offer's own CBOR encoding. -func OfferBytes(offer []byte) ([]byte, error) { - return encMode.Marshal([]any{OfferTag, offer}) -} +func OfferBytes(offer []byte) ([]byte, error) { return protocol.OfferBytes(offer) } // VerifyOffer checks an offer's signature, r ‖ s, against the offer key's // public half. The app does the same with the key it pins. @@ -67,7 +87,165 @@ func VerifyOffer(pub, offer, sig []byte) error { return nil } -// offerRecord is the sealed offer key on disk. +// offerDec reads an offer as the box does: a key it does not know, or a key +// twice, is refused. +var offerDec = func() cbor.DecMode { + m, err := cbor.DecOptions{ + DupMapKey: cbor.DupMapKeyEnforcedAPF, + ExtraReturnErrors: cbor.ExtraDecErrorUnknownField, + IndefLength: cbor.IndefLengthForbidden, + MaxArrayElements: 64, + MaxMapPairs: 32, + MaxNestedLevels: 4, + }.DecMode() + if err != nil { + panic(err) + } + return m +}() + +// signOffer signs offer if the keys part can vouch for it: a GrantOffer of +// this version, for the agent role, good for no longer than OfferTTLMs, whose +// key is kid's and kid is still pending. +func (s *Server) signOffer(kid string, offer []byte) ([]byte, error) { + if s.offer == nil || !validKid(kid) || len(offer) == 0 || len(offer) > maxOffer { + return nil, ErrRefused + } + var o protocol.GrantOffer + if err := offerDec.Unmarshal(offer, &o); err != nil { + return nil, ErrRefused + } + now := s.now().UnixMilli() + if o.V != protocol.GrantOfferVersion || o.Role != protocol.RoleAgent || + o.ExpMs <= now || o.ExpMs > now+protocol.OfferTTLMs { + return nil, ErrRefused + } + pub, err := s.pendingPub(kid) + if err != nil || !bytes.Equal(pub, o.Key) { + return nil, ErrRefused + } + msg, err := OfferBytes(offer) + if err != nil { + return nil, err + } + digest := sha256.Sum256(msg) + ctx, cancel := context.WithTimeout(context.Background(), unwrapTimeout) + defer cancel() + sig, err := s.offer.Sign(ctx, digest[:]) + if err != nil { + s.log.Warn("could not sign an offer", "err", err) + return nil, err + } + return sig, nil +} + +// pendingPub is the public half of a key still waiting for approval. +func (s *Server) pendingPub(kid string) ([]byte, error) { + raw, err := os.ReadFile(filepath.Join(s.dir, "pending", kid)) + if err != nil { + return nil, ErrUnknownKey + } + var rec record + if err := decMode.Unmarshal(raw, &rec); err != nil || rec.Kid != kid { + return nil, ErrUnknownKey + } + return rec.Pub, nil +} + +func (s *Server) offerPub() ([]byte, error) { + if s.offer == nil { + return nil, ErrRefused + } + return bytes.Clone(s.offer.Public()), nil +} + +// OfferKey is the offer key's public half, SEC1 uncompressed. +func (c *Client) OfferKey(ctx context.Context) ([]byte, error) { + resp, err := c.call(ctx, request{Op: opOfferPub}) + return resp.Pub, err +} + +// SignOffer signs one offer's CBOR with the offer key. The offer's key must +// be the public half of kid, a key New made that is not yet kept. +func (c *Client) SignOffer(ctx context.Context, kid string, offer []byte) ([]byte, error) { + resp, err := c.call(ctx, request{Op: opOfferSign, Kid: kid, D: offer}) + return resp.D, err +} + +// -------------------------------------------------------------------------- +// The offer key in KMS +// -------------------------------------------------------------------------- + +// KMSSigner is the part of AWS KMS the offer key uses. cmd/ftwkeysd adapts +// the AWS SDK to it. +type KMSSigner interface { + // PublicKey answers the key's public half as DER SubjectPublicKeyInfo, as + // kms:GetPublicKey. It refuses a key that is not ECC_NIST_P256 for + // signing. + PublicKey(ctx context.Context, keyID string) ([]byte, error) + // Sign signs a SHA-256 digest with ECDSA_SHA_256, as kms:Sign with + // MessageType DIGEST, and answers the DER signature. + Sign(ctx context.Context, keyID string, digest []byte) ([]byte, error) +} + +type kmsOfferKey struct { + kms KMSSigner + id string + key *ecdsa.PublicKey + pub []byte +} + +// NewKMSOfferKey is the offer key held in KMS under keyID. It reads the +// public half once. +func NewKMSOfferKey(ctx context.Context, k KMSSigner, keyID string) (OfferSigner, error) { + der, err := k.PublicKey(ctx, keyID) + if err != nil { + return nil, fmt.Errorf("keys: offer key: %w", err) + } + parsed, err := x509.ParsePKIXPublicKey(der) + if err != nil { + return nil, fmt.Errorf("keys: offer key: %w", err) + } + key, ok := parsed.(*ecdsa.PublicKey) + if !ok || key.Curve != elliptic.P256() { + return nil, errors.New("keys: the offer key is not a P-256 key") + } + pub, err := key.Bytes() + if err != nil { + return nil, err + } + return &kmsOfferKey{kms: k, id: keyID, key: key, pub: pub}, nil +} + +func (k *kmsOfferKey) Public() []byte { return k.pub } + +// Sign asks KMS and checks what comes back against the public half. +func (k *kmsOfferKey) Sign(ctx context.Context, digest []byte) ([]byte, error) { + der, err := k.kms.Sign(ctx, k.id, digest) + if err != nil { + return nil, err + } + var sig struct{ R, S *big.Int } + rest, err := asn1.Unmarshal(der, &sig) + if err != nil || len(rest) != 0 || sig.R == nil || sig.S == nil || !ecdsa.Verify(k.key, digest, sig.R, sig.S) { + return nil, errors.New("keys: KMS answered a signature that does not verify") + } + out := make([]byte, 64) + sig.R.FillBytes(out[:32]) + sig.S.FillBytes(out[32:]) + return out, nil +} + +// -------------------------------------------------------------------------- +// The local offer key, for development and tests +// -------------------------------------------------------------------------- + +type localOfferKey struct { + key *ecdsa.PrivateKey + pub []byte +} + +// offerRecord is the sealed local offer key on disk. type offerRecord struct { V int `cbor:"v"` Pub []byte `cbor:"pub"` @@ -75,105 +253,92 @@ type offerRecord struct { CT []byte `cbor:"ct"` } -// loadOfferKey unseals the offer key, making it on first start. It stays in -// memory: it signs every consent page, and vouches only for what a page -// shows. -func (s *Server) loadOfferKey() error { - path := filepath.Join(s.dir, "offer.key") +// LocalOfferKey is an offer key in dir/offer.key, sealed by sealer, made on +// first use. It is for development and tests: production signs in KMS. +func LocalOfferKey(dir string, sealer Sealer, log *slog.Logger) (OfferSigner, error) { + if log == nil { + log = slog.Default() + } + path := filepath.Join(dir, "offer.key") raw, err := os.ReadFile(path) if errors.Is(err, fs.ErrNotExist) { - return s.makeOfferKey(path) + return makeLocalOfferKey(path, sealer, log) } if err != nil { - return err + return nil, err } var rec offerRecord if err := decMode.Unmarshal(raw, &rec); err != nil { - return fmt.Errorf("keys: offer key: %w", err) + return nil, fmt.Errorf("keys: offer key: %w", err) } ctx, cancel := context.WithTimeout(context.Background(), unwrapTimeout) defer cancel() - dek, err := s.sealer.Unwrap(ctx, offerKid, rec.DEK) + dek, err := sealer.Unwrap(ctx, offerKid, rec.DEK) if err != nil { - return fmt.Errorf("keys: offer key: %w", err) + return nil, fmt.Errorf("keys: offer key: %w", err) } defer wipe(dek) aead, err := newGCM(dek) if err != nil { - return err + return nil, err } d, err := open(aead, rec.CT, append([]byte(OfferTag+"\x00"), rec.Pub...)) if err != nil { - return errors.New("keys: the offer key does not open") + return nil, errors.New("keys: the offer key does not open") } defer wipe(d) key, err := ecdsa.ParseRawPrivateKey(elliptic.P256(), d) if err != nil { - return err + return nil, err } - s.offer = key - return nil + return &localOfferKey{key: key, pub: rec.Pub}, nil } -func (s *Server) makeOfferKey(path string) error { +func makeLocalOfferKey(path string, sealer Sealer, log *slog.Logger) (OfferSigner, error) { key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { - return err + return nil, err } pub, err := key.PublicKey.Bytes() if err != nil { - return err + return nil, err } d, err := key.Bytes() if err != nil { - return err + return nil, err } defer wipe(d) ctx, cancel := context.WithTimeout(context.Background(), unwrapTimeout) defer cancel() - dek, wrapped, err := s.sealer.DataKey(ctx, offerKid) + dek, wrapped, err := sealer.DataKey(ctx, offerKid) if err != nil { - return err + return nil, err } defer wipe(dek) aead, err := newGCM(dek) if err != nil { - return err + return nil, err } ct, err := seal(aead, d, append([]byte(OfferTag+"\x00"), pub...)) if err != nil { - return err + return nil, err } raw, err := encMode.Marshal(offerRecord{V: 1, Pub: pub, DEK: wrapped, CT: ct}) if err != nil { - return err + return nil, err } if err := writeFile(path, raw); err != nil { - return err + return nil, err } - s.offer = key - s.log.Info("made the offer key; pin it in Core and the apps", "kid", OfferKid(pub), + log.Info("made a local offer key, for development only", "kid", OfferKid(pub), "pub", base64.RawURLEncoding.EncodeToString(pub)) - return nil + return &localOfferKey{key: key, pub: pub}, nil } -func (s *Server) offerPub() ([]byte, error) { - if s.offer == nil { - return nil, ErrRefused - } - return s.offer.PublicKey.Bytes() -} +func (k *localOfferKey) Public() []byte { return k.pub } -func (s *Server) signOffer(offer []byte) ([]byte, error) { - if s.offer == nil || len(offer) == 0 || len(offer) > 8<<10 { - return nil, ErrRefused - } - msg, err := OfferBytes(offer) - if err != nil { - return nil, err - } - digest := sha256.Sum256(msg) - r, sv, err := ecdsa.Sign(rand.Reader, s.offer, digest[:]) +func (k *localOfferKey) Sign(_ context.Context, digest []byte) ([]byte, error) { + r, sv, err := ecdsa.Sign(rand.Reader, k.key, digest) if err != nil { return nil, err } @@ -182,15 +347,3 @@ func (s *Server) signOffer(offer []byte) ([]byte, error) { sv.FillBytes(sig[32:]) return sig, nil } - -// OfferKey is the offer key's public half, SEC1 uncompressed. -func (c *Client) OfferKey(ctx context.Context) ([]byte, error) { - resp, err := c.call(ctx, request{Op: opOfferPub}) - return resp.Pub, err -} - -// SignOffer signs one offer's CBOR with the offer key. -func (c *Client) SignOffer(ctx context.Context, offer []byte) ([]byte, error) { - resp, err := c.call(ctx, request{Op: opOfferSign, D: offer}) - return resp.D, err -} diff --git a/internal/keys/server.go b/internal/keys/server.go index 69175b9..2d8d37b 100644 --- a/internal/keys/server.go +++ b/internal/keys/server.go @@ -3,7 +3,6 @@ package keys import ( "context" "crypto/ecdh" - "crypto/ecdsa" "crypto/rand" "encoding/base64" "errors" @@ -58,7 +57,16 @@ type Server struct { mu sync.Mutex sessions map[string]*session // offer signs consent offers (offer.go). - offer *ecdsa.PrivateKey + offer OfferSigner +} + +// An Option sets up a Server. +type Option func(*Server) + +// WithOfferKey signs offers with k. Without it, the server makes a local +// offer key in its state directory, which is for development and tests. +func WithOfferKey(k OfferSigner) Option { + return func(s *Server) { s.offer = k } } type session struct { @@ -71,7 +79,7 @@ type session struct { // NewServer keeps sealed keys under dir: pending/ until the owner approves, // then kept/. -func NewServer(dir string, sealer Sealer, log *slog.Logger) (*Server, error) { +func NewServer(dir string, sealer Sealer, log *slog.Logger, opts ...Option) (*Server, error) { if log == nil { log = slog.Default() } @@ -81,13 +89,21 @@ func NewServer(dir string, sealer Sealer, log *slog.Logger) (*Server, error) { } } s := &Server{dir: dir, sealer: sealer, log: log, now: time.Now, PeerUID: -1, sessions: map[string]*session{}} - if err := s.loadOfferKey(); err != nil { - return nil, err + for _, o := range opts { + o(s) + } + if s.offer == nil { + k, err := LocalOfferKey(dir, sealer, log) + if err != nil { + return nil, err + } + s.offer = k } return s, nil } -// Serve accepts connections until ln closes. Each must come from PeerUID. +// Serve accepts connections until ln closes. Each must be a unix connection +// from PeerUID. func (s *Server) Serve(ln net.Listener) error { for { conn, err := ln.Accept() @@ -97,12 +113,16 @@ func (s *Server) Serve(ln net.Listener) error { } return err } - if uc, ok := conn.(*net.UnixConn); ok { - if err := checkPeer(uc, s.PeerUID); err != nil { - s.log.Warn("refused a connection from another user") - _ = conn.Close() - continue - } + uc, ok := conn.(*net.UnixConn) + if !ok { + s.log.Warn("refused a connection that is not a unix socket's") + _ = conn.Close() + continue + } + if err := checkPeer(uc, s.PeerUID); err != nil { + s.log.Warn("refused a connection from another user") + _ = conn.Close() + continue } go s.serveConn(conn) } @@ -171,7 +191,7 @@ func (s *Server) handle(req request) response { case opOfferPub: resp.Pub, err = s.offerPub() case opOfferSign: - resp.D, err = s.signOffer(req.D) + resp.D, err = s.signOffer(req.Kid, req.D) default: err = ErrRefused } @@ -384,7 +404,7 @@ func (s *Server) finish(sess string, msg2 []byte) ([]byte, error) { } split, err := x.init.Split() if err != nil { - return err + return ErrRefused } x.init, x.tr, payload = nil, appwire.NewTransport(split), p return nil From 0cddf80871dd1ff22fa9955b446e7906b8346316 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Thu, 15 Oct 2026 22:51:57 +0200 Subject: [PATCH 5/5] chore(keys): pin the merged agent contract, srcfl/ftw#1642 at bf5bf50 Co-Authored-By: Claude Opus 5.5 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 66184cb..580f8ca 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/kms v1.61.4 github.com/fxamacker/cbor/v2 v2.9.4 github.com/gorilla/websocket v1.5.3 - github.com/srcfl/ftw/go/protocol v0.0.0-20261015184016-e25596322c66 + github.com/srcfl/ftw/go/protocol v0.0.0-20261015203548-bf5bf50bade6 golang.org/x/crypto v0.57.0 golang.org/x/sys v0.48.0 modernc.org/sqlite v1.59.0 diff --git a/go.sum b/go.sum index 1c9ac7b..a5ebb88 100644 --- a/go.sum +++ b/go.sum @@ -46,8 +46,8 @@ github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOF github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/srcfl/ftw/go/protocol v0.0.0-20261015184016-e25596322c66 h1:ZFXkeVj4a828TZ5U9c4q7iC2IzLMf9pLSmcKS3gBpsk= -github.com/srcfl/ftw/go/protocol v0.0.0-20261015184016-e25596322c66/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= +github.com/srcfl/ftw/go/protocol v0.0.0-20261015203548-bf5bf50bade6 h1:+VacTKhwAs9LnKb0TZztvxEIVV4OXxOIzeHKDWKAw6I= +github.com/srcfl/ftw/go/protocol v0.0.0-20261015203548-bf5bf50bade6/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=