Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,12 @@ to it yet.
most 4 KiB, the answer within 10 s.
- Limits per home: 16 sessions, one per device, 256 envelopes queued per
phone, and a small message rate from each phone.
- **HTTP/3.** The plain HTTPS routes, `/v1/app/once`, `/v1/now`,
`/healthz` and `/source`, also answer over HTTP/3 on UDP 443, with the
same certificate (`internal/quicdoor`). TCP responses say so in
`Alt-Svc`, except a WebSocket upgrade's. The sockets stay on TCP. A read
on a new connection takes two round trips instead of three; under loss
the gap is larger (`scripts/netem-once.sh`, [deploy/http3.md](deploy/http3.md)).
- **Alerts (C4).** A box's signed `event` goes to its phones through APNs,
wrapped unchanged: the phone checks the signature, writes its own words and
picks the buttons from the verified kind. Every alert has the same aps,
Expand Down Expand Up @@ -261,7 +267,8 @@ box ──wss, TLS 1.3, signed frames──▶ /v1/box ──▶ store (memory)
judges as it judges the same session over the relay. The box closes the
uplink on anything else.
- **No TLS terminator in front.** Box and service bind the handshake to the
TLS session, so `ftwcloudd` ends TLS itself. A TCP load balancer is fine.
TLS session, so `ftwcloudd` ends TLS itself. A TCP load balancer is fine,
and so is a UDP one that forwards packets; nothing may end QUIC.
- **Root on the host can read memory.** The privacy notice says so.

## Run it locally
Expand Down
75 changes: 75 additions & 0 deletions cmd/ftwcloudd/health.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
package main

import (
"context"
"crypto/tls"
"errors"
"flag"
"fmt"
"io"
"net/http"
"net/url"
"time"

"github.com/quic-go/quic-go/http3"
)

// health checks /healthz over TCP and over HTTP/3, the way the container's
// health check and an operator's laptop run it:
//
// ftwcloudd health -name cloud.ftw.energy https://127.0.0.1/healthz
// ftwcloudd health https://cloud.ftw.energy/healthz
//
// Each line names the protocol that answered and how long it took. Either
// failing fails the check: a host whose UDP is shut while Alt-Svc says
// HTTP/3 is there makes every client race for nothing.
func health(args []string) error {
fs := flag.NewFlagSet("health", flag.ExitOnError)
name := fs.String("name", "", "the certificate's name, when the URL names an address")
insecure := fs.Bool("insecure", false, "accept any certificate (local tests only)")
h3 := fs.Bool("h3", true, "check HTTP/3 too")
_ = fs.Parse(args)
if fs.NArg() != 1 {
return errors.New("usage: ftwcloudd health [-name host] [-h3=false] https://host/healthz")
}
u, err := url.Parse(fs.Arg(0))
if err != nil || u.Scheme != "https" {
return fmt.Errorf("health: %q is not an https URL", fs.Arg(0))
}
tlsConf := &tls.Config{MinVersion: tls.VersionTLS13, ServerName: *name, InsecureSkipVerify: *insecure}
tcp := &http.Transport{TLSClientConfig: tlsConf.Clone(), DisableKeepAlives: true}
defer tcp.CloseIdleConnections()
clients := []*http.Client{{Transport: tcp}}
if *h3 {
quic := &http3.Transport{TLSClientConfig: tlsConf.Clone()}
defer quic.Close()
clients = append(clients, &http.Client{Transport: quic})
}
for _, c := range clients {
c.Timeout = 5 * time.Second
start := time.Now()
proto, err := get(c, u.String())
if err != nil {
return err
}
fmt.Printf("%s ok in %d ms\n", proto, time.Since(start).Milliseconds())
}
return nil
}

func get(c *http.Client, u string) (string, error) {
req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, u, nil)
if err != nil {
return "", err
}
resp, err := c.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
body, _ := io.ReadAll(io.LimitReader(resp.Body, 64))
if resp.StatusCode != http.StatusOK || string(body) != "ok\n" {
return resp.Proto, fmt.Errorf("health: %s answered %d", resp.Proto, resp.StatusCode)
}
return resp.Proto, nil
}
42 changes: 39 additions & 3 deletions cmd/ftwcloudd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
// ftwcloudd keygen -out service.key make the service key; prints its public half
// ftwcloudd keygen -noise -out noise.key make the Noise key phones dial; prints its public half
// ftwcloudd metrics print the running service's counts, from inside its container
// ftwcloudd health https://host/healthz check /healthz over TCP and HTTP/3
// ftwcloudd [flags] serve
package main

Expand Down Expand Up @@ -41,6 +42,7 @@ import (
"github.com/srcfl/ftw-cloud/internal/metrics"
"github.com/srcfl/ftw-cloud/internal/oauth"
"github.com/srcfl/ftw-cloud/internal/push"
"github.com/srcfl/ftw-cloud/internal/quicdoor"
"github.com/srcfl/ftw-cloud/internal/registry"
"github.com/srcfl/ftw-cloud/internal/server"
"github.com/srcfl/ftw-cloud/internal/shard"
Expand All @@ -53,8 +55,12 @@ import (
)

func main() {
if len(os.Args) > 1 && os.Args[1] == "keygen" {
if err := keygen(os.Args[2:]); err != nil {
if len(os.Args) > 1 && (os.Args[1] == "keygen" || os.Args[1] == "health") {
run := keygen
if os.Args[1] == "health" {
run = health
}
if err := run(os.Args[2:]); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
Expand Down Expand Up @@ -93,6 +99,7 @@ func serve(args []string) error {
metricsAddr := fs.String("metrics-listen", defaultMetricsAddr, "loopback address for /metrics; empty turns it off")
restartRate := fs.Int("restart-rate", defaultRestartRate, "sockets a second the next instance can take after a restart; sets the spread of the redial")
restartMin := fs.Duration("restart-spread-min", 15*time.Second, "the shortest spread: longer than the port stays closed while the next container starts")
h3 := fs.Bool("h3", true, "also serve HTTP/3 on the same address over UDP, and say so in Alt-Svc")
_ = fs.Parse(args)

harden()
Expand Down Expand Up @@ -226,6 +233,28 @@ func serve(args []string) error {
tlsConfig.NextProtos = append(tlsConfig.NextProtos, "acme-tls/1")
}

// HTTP/3 on UDP, beside TCP, for the plain HTTPS routes. The sockets
// stay on TCP. The UDP port is bound before Alt-Svc says it is there,
// and a port that will not bind leaves TCP serving alone: QUIC is the
// fast path, never the only one.
var door *quicdoor.Door
var udp net.PacketConn
if *h3 {
port, err := quicdoor.PortOf(*listen)
if err != nil {
return err
}
if udp, err = net.ListenPacket("udp", *listen); err != nil {
log.Error("HTTP/3 is off: UDP did not bind; TCP serves alone", "err", err)
} else {
srv.AltSvc = quicdoor.AltSvc(port)
door = quicdoor.New(srv.Handler(), log)
if door.ResetKey, err = quicdoor.ResetKeyFile(filepath.Join(*stateDir, "quic-reset.key")); err != nil {
log.Warn("no stateless resets: a client waits out its idle timeout after a restart", "err", err)
}
}
}

httpSrv := &http.Server{
Addr: *listen,
Handler: srv.Handler(),
Expand Down Expand Up @@ -303,9 +332,16 @@ func serve(args []string) error {
}
stopPush()
}()
if door != nil {
go func() {
if err := door.Serve(ctx, udp, tlsConfig); err != nil {
log.Error("HTTP/3 stopped; TCP serves on", "err", err)
}
Comment on lines +337 to +339

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Stop the service when the HTTP/3 listener fails

If Door.Serve exits with a fatal UDP or QUIC listener error, this path only logs it while the TCP server keeps emitting the unchanged Alt-Svc header. The inspected deploy/compose.yml health check can mark the container unhealthy, but the deployment documentation explicitly says that nothing restarts on that state, so clients can keep racing a dead HTTP/3 endpoint indefinitely. Propagate this error to terminate the process, or otherwise stop advertising HTTP/3.

Useful? React with 👍 / 👎.

}()
}

pub, _ := cloudwire.PublicKeyBytesOf(&key.PublicKey)
log.Info("ftwcloudd listening", "addr", *listen, "shard", shards.Self(), "shards", shards.Shards(),
log.Info("ftwcloudd listening", "addr", *listen, "h3", door != nil, "shard", shards.Self(), "shards", shards.Shards(),
"service_key", base64.RawURLEncoding.EncodeToString(pub), "noise_key", base64.RawURLEncoding.EncodeToString(noise.Public()))
err = httpSrv.ListenAndServeTLS("", "")
if err != nil && !errors.Is(err, http.ErrServerClosed) {
Expand Down
14 changes: 9 additions & 5 deletions deploy/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ the uplink (srcfl/ftw#1595), after its owner decides to.
| Instance | `i-0ecac4cbc6b5f8adc`, `t4g.small`, Amazon Linux 2023 arm64, IMDSv2 required, no key pair |
| Disk | `vol-0760cad0e8551e5e4`, 20 GB gp3, encrypted with `aws/ebs` |
| Elastic IP | `13.50.84.149` (`eipalloc-0085f3e68b6f36da4`) |
| Security group | `sg-098992ad382ecf94a` `ftw-cloud`: 443/tcp from anywhere, nothing else in |
| Security group | `sg-098992ad382ecf94a` `ftw-cloud`: 443/tcp from anywhere, and 443/udp for HTTP/3 ([http3.md](http3.md)); nothing else in |
| Instance role | `ftw-cloud-host`, with `AmazonSSMManagedInstanceCore` and the inline policy `ftw-cloud-service-key` ([`host-policy.json`](host-policy.json)): read one secret, nothing else |
| Secret | `ftw-cloud/service-key` in Secrets Manager, eu-north-1, encrypted with `aws/secretsmanager` |
| DNS | `cloud.ftw.energy` A `13.50.84.149` in the Cloudflare zone `ftw.energy`, grey cloud (record `22d28943eea9f6b8584214e1a8256ef1`) |
Expand Down Expand Up @@ -71,8 +71,8 @@ secret the same way.
pipes cores to `/bin/false`, systemd-coredump stores nothing, `compose.yml`
sets `ulimits: core: 0`, and `ftwcloudd` marks itself not dumpable.
Household readings exist only in this process's memory.
- Nothing comes in but 443. There is no SSH and no key pair; administration
is through SSM. Session logs to a separate account are not set up yet.
- Nothing comes in but 443, TCP and UDP. There is no SSH and no key pair;
administration is through SSM. Session logs to a separate account are not set up yet.
- Outbound is open. The host now needs only GitHub and `ghcr.io` (the image
and `compose.yml`), AWS endpoints (SSM, Secrets Manager), Let's Encrypt
and, from C4, APNs. Narrowing it is still to do.
Expand Down Expand Up @@ -317,9 +317,12 @@ that holds them adds itself to the drainers in `cmd/ftwcloudd`.
```bash
curl -s https://cloud.ftw.energy/healthz
curl -sI https://cloud.ftw.energy/source
go run ./cmd/ftwcloudd health https://cloud.ftw.energy/healthz
```

`ok` means DNS, the certificate and the service all work. `/source` returns
`ok` means DNS, the certificate and the service all work. The last line
checks TCP and HTTP/3 ([http3.md](http3.md)); on the host,
`docker compose ps` shows the container's own check of both. `/source` returns
the source archive, and its headers name the commit (`X-Source-Revision`) and
the image digest (`X-Source-Image`), which should match what CI built.
`/v1/app` answers 501 on a host without the Noise key, and `/v1/now` answers
Expand Down Expand Up @@ -368,7 +371,8 @@ Cloud data; the images on `ghcr.io` hold only code.
## Not here yet

The APNs key ([apns.md](apns.md)) and the Noise key phones dial
([noise.md](noise.md)) come from Parameter Store. Widget tokens need no
([noise.md](noise.md)) come from Parameter Store. HTTP/3 needs UDP 443 in the
security group ([http3.md](http3.md)). Widget tokens need no
secret: the registry holds their hashes. Statistics (C7–C8) add secrets,
egress and storage when they land. The erase test, canary homes
and the external heartbeat that pages Sourceful belong with the first homes.
8 changes: 6 additions & 2 deletions deploy/bootstrap.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ sed -i '/\sswap\s/d' /etc/fstab
cat > /etc/sysctl.d/99-ftw-cloud.conf <<'SYSCTL'
kernel.core_pattern = |/bin/false
fs.suid_dumpable = 0
# quic-go asks for 7.5 MB socket buffers for HTTP/3 (deploy/http3.md).
net.core.rmem_max = 7500000
net.core.wmem_max = 7500000
SYSCTL
sysctl --system >/dev/null
install -d /etc/systemd/coredump.conf.d
Expand Down Expand Up @@ -171,7 +174,8 @@ if ! [[ $rev =~ ^[0-9a-f]{40}$ ]] || ! git merge-base --is-ancestor "$rev" origi
fi
git checkout --quiet --detach "$rev"

# Only the image line changes; what keys.sh set stays.
# Only the image line changes; what keys.sh set, and FTW_CLOUD_H3=false set
# by hand to turn HTTP/3 off (deploy/http3.md), stay.
touch "$env"
sed -i '/^FTW_CLOUD_IMAGE=/d' "$env"
printf 'FTW_CLOUD_IMAGE=%s\n' "$image" >> "$env"
Expand All @@ -192,7 +196,7 @@ systemctl enable ftw-cloud-key.service ftw-cloud.service

# Show that the host rules hold.
swapon --show
sysctl kernel.core_pattern
sysctl kernel.core_pattern net.core.rmem_max

# A host that already runs an image from CI runs it again with the new
# tooling. A new host waits for the owner to name a digest.
Expand Down
12 changes: 12 additions & 0 deletions deploy/compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,12 @@ services:
restart: unless-stopped
ports:
- '443:443'
# HTTP/3 for the plain HTTPS routes (deploy/http3.md). The security
# group must let UDP 443 in before this runs.
- '443:443/udp'
command:
- -listen=:443
- -h3=${FTW_CLOUD_H3:-true}
- -domain=cloud.ftw.energy
- -state=/state
- -service-key=/run/secrets/ftw-cloud-service.key
Expand Down Expand Up @@ -47,6 +51,14 @@ services:
- /run/ftw-cloud-keys:/run/ftw-cloud-keys
secrets:
- ftw-cloud-service.key
# /healthz over TCP and HTTP/3 on the container's own port, with the
# certificate's name. Docker shows the result; nothing restarts on it.
healthcheck:
test: ['CMD', '/usr/local/bin/ftwcloudd', 'health', '-name', 'cloud.ftw.energy', '-h3=${FTW_CLOUD_H3:-true}', 'https://127.0.0.1/healthz']
interval: 60s
timeout: 15s
start_period: 60s
retries: 3
read_only: true
tmpfs:
- /tmp
Expand Down
Loading
Loading