diff --git a/README.md b/README.md index 39f186a..8877105 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,12 @@ to it yet. most 4 KiB, the answer within 10 s. - Limits per home: 16 sessions, one per device, 256 envelopes queued per phone, and a small message rate from each phone. +- **HTTP/3.** The plain HTTPS routes, `/v1/app/once`, `/v1/now`, + `/healthz` and `/source`, also answer over HTTP/3 on UDP 443, with the + same certificate (`internal/quicdoor`). TCP responses say so in + `Alt-Svc`, except a WebSocket upgrade's. The sockets stay on TCP. A read + on a new connection takes two round trips instead of three; under loss + the gap is larger (`scripts/netem-once.sh`, [deploy/http3.md](deploy/http3.md)). - **Alerts (C4).** A box's signed `event` goes to its phones through APNs, wrapped unchanged: the phone checks the signature, writes its own words and picks the buttons from the verified kind. Every alert has the same aps, @@ -261,7 +267,8 @@ box ──wss, TLS 1.3, signed frames──▶ /v1/box ──▶ store (memory) judges as it judges the same session over the relay. The box closes the uplink on anything else. - **No TLS terminator in front.** Box and service bind the handshake to the - TLS session, so `ftwcloudd` ends TLS itself. A TCP load balancer is fine. + TLS session, so `ftwcloudd` ends TLS itself. A TCP load balancer is fine, + and so is a UDP one that forwards packets; nothing may end QUIC. - **Root on the host can read memory.** The privacy notice says so. ## Run it locally diff --git a/cmd/ftwcloudd/health.go b/cmd/ftwcloudd/health.go new file mode 100644 index 0000000..4b049b8 --- /dev/null +++ b/cmd/ftwcloudd/health.go @@ -0,0 +1,75 @@ +package main + +import ( + "context" + "crypto/tls" + "errors" + "flag" + "fmt" + "io" + "net/http" + "net/url" + "time" + + "github.com/quic-go/quic-go/http3" +) + +// health checks /healthz over TCP and over HTTP/3, the way the container's +// health check and an operator's laptop run it: +// +// ftwcloudd health -name cloud.ftw.energy https://127.0.0.1/healthz +// ftwcloudd health https://cloud.ftw.energy/healthz +// +// Each line names the protocol that answered and how long it took. Either +// failing fails the check: a host whose UDP is shut while Alt-Svc says +// HTTP/3 is there makes every client race for nothing. +func health(args []string) error { + fs := flag.NewFlagSet("health", flag.ExitOnError) + name := fs.String("name", "", "the certificate's name, when the URL names an address") + insecure := fs.Bool("insecure", false, "accept any certificate (local tests only)") + h3 := fs.Bool("h3", true, "check HTTP/3 too") + _ = fs.Parse(args) + if fs.NArg() != 1 { + return errors.New("usage: ftwcloudd health [-name host] [-h3=false] https://host/healthz") + } + u, err := url.Parse(fs.Arg(0)) + if err != nil || u.Scheme != "https" { + return fmt.Errorf("health: %q is not an https URL", fs.Arg(0)) + } + tlsConf := &tls.Config{MinVersion: tls.VersionTLS13, ServerName: *name, InsecureSkipVerify: *insecure} + tcp := &http.Transport{TLSClientConfig: tlsConf.Clone(), DisableKeepAlives: true} + defer tcp.CloseIdleConnections() + clients := []*http.Client{{Transport: tcp}} + if *h3 { + quic := &http3.Transport{TLSClientConfig: tlsConf.Clone()} + defer quic.Close() + clients = append(clients, &http.Client{Transport: quic}) + } + for _, c := range clients { + c.Timeout = 5 * time.Second + start := time.Now() + proto, err := get(c, u.String()) + if err != nil { + return err + } + fmt.Printf("%s ok in %d ms\n", proto, time.Since(start).Milliseconds()) + } + return nil +} + +func get(c *http.Client, u string) (string, error) { + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, u, nil) + if err != nil { + return "", err + } + resp, err := c.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 64)) + if resp.StatusCode != http.StatusOK || string(body) != "ok\n" { + return resp.Proto, fmt.Errorf("health: %s answered %d", resp.Proto, resp.StatusCode) + } + return resp.Proto, nil +} diff --git a/cmd/ftwcloudd/main.go b/cmd/ftwcloudd/main.go index fff992c..316e5a9 100644 --- a/cmd/ftwcloudd/main.go +++ b/cmd/ftwcloudd/main.go @@ -5,6 +5,7 @@ // ftwcloudd keygen -out service.key make the service key; prints its public half // ftwcloudd keygen -noise -out noise.key make the Noise key phones dial; prints its public half // ftwcloudd metrics print the running service's counts, from inside its container +// ftwcloudd health https://host/healthz check /healthz over TCP and HTTP/3 // ftwcloudd [flags] serve package main @@ -41,6 +42,7 @@ import ( "github.com/srcfl/ftw-cloud/internal/metrics" "github.com/srcfl/ftw-cloud/internal/oauth" "github.com/srcfl/ftw-cloud/internal/push" + "github.com/srcfl/ftw-cloud/internal/quicdoor" "github.com/srcfl/ftw-cloud/internal/registry" "github.com/srcfl/ftw-cloud/internal/server" "github.com/srcfl/ftw-cloud/internal/shard" @@ -53,8 +55,12 @@ import ( ) func main() { - if len(os.Args) > 1 && os.Args[1] == "keygen" { - if err := keygen(os.Args[2:]); err != nil { + if len(os.Args) > 1 && (os.Args[1] == "keygen" || os.Args[1] == "health") { + run := keygen + if os.Args[1] == "health" { + run = health + } + if err := run(os.Args[2:]); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } @@ -93,6 +99,7 @@ func serve(args []string) error { metricsAddr := fs.String("metrics-listen", defaultMetricsAddr, "loopback address for /metrics; empty turns it off") restartRate := fs.Int("restart-rate", defaultRestartRate, "sockets a second the next instance can take after a restart; sets the spread of the redial") restartMin := fs.Duration("restart-spread-min", 15*time.Second, "the shortest spread: longer than the port stays closed while the next container starts") + h3 := fs.Bool("h3", true, "also serve HTTP/3 on the same address over UDP, and say so in Alt-Svc") _ = fs.Parse(args) harden() @@ -226,6 +233,28 @@ func serve(args []string) error { tlsConfig.NextProtos = append(tlsConfig.NextProtos, "acme-tls/1") } + // HTTP/3 on UDP, beside TCP, for the plain HTTPS routes. The sockets + // stay on TCP. The UDP port is bound before Alt-Svc says it is there, + // and a port that will not bind leaves TCP serving alone: QUIC is the + // fast path, never the only one. + var door *quicdoor.Door + var udp net.PacketConn + if *h3 { + port, err := quicdoor.PortOf(*listen) + if err != nil { + return err + } + if udp, err = net.ListenPacket("udp", *listen); err != nil { + log.Error("HTTP/3 is off: UDP did not bind; TCP serves alone", "err", err) + } else { + srv.AltSvc = quicdoor.AltSvc(port) + door = quicdoor.New(srv.Handler(), log) + if door.ResetKey, err = quicdoor.ResetKeyFile(filepath.Join(*stateDir, "quic-reset.key")); err != nil { + log.Warn("no stateless resets: a client waits out its idle timeout after a restart", "err", err) + } + } + } + httpSrv := &http.Server{ Addr: *listen, Handler: srv.Handler(), @@ -303,9 +332,16 @@ func serve(args []string) error { } stopPush() }() + if door != nil { + go func() { + if err := door.Serve(ctx, udp, tlsConfig); err != nil { + log.Error("HTTP/3 stopped; TCP serves on", "err", err) + } + }() + } pub, _ := cloudwire.PublicKeyBytesOf(&key.PublicKey) - log.Info("ftwcloudd listening", "addr", *listen, "shard", shards.Self(), "shards", shards.Shards(), + log.Info("ftwcloudd listening", "addr", *listen, "h3", door != nil, "shard", shards.Self(), "shards", shards.Shards(), "service_key", base64.RawURLEncoding.EncodeToString(pub), "noise_key", base64.RawURLEncoding.EncodeToString(noise.Public())) err = httpSrv.ListenAndServeTLS("", "") if err != nil && !errors.Is(err, http.ErrServerClosed) { diff --git a/deploy/README.md b/deploy/README.md index 468264f..e29f956 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -14,7 +14,7 @@ the uplink (srcfl/ftw#1595), after its owner decides to. | Instance | `i-0ecac4cbc6b5f8adc`, `t4g.small`, Amazon Linux 2023 arm64, IMDSv2 required, no key pair | | Disk | `vol-0760cad0e8551e5e4`, 20 GB gp3, encrypted with `aws/ebs` | | Elastic IP | `13.50.84.149` (`eipalloc-0085f3e68b6f36da4`) | -| Security group | `sg-098992ad382ecf94a` `ftw-cloud`: 443/tcp from anywhere, nothing else in | +| Security group | `sg-098992ad382ecf94a` `ftw-cloud`: 443/tcp from anywhere, and 443/udp for HTTP/3 ([http3.md](http3.md)); nothing else in | | Instance role | `ftw-cloud-host`, with `AmazonSSMManagedInstanceCore` and the inline policy `ftw-cloud-service-key` ([`host-policy.json`](host-policy.json)): read one secret, nothing else | | Secret | `ftw-cloud/service-key` in Secrets Manager, eu-north-1, encrypted with `aws/secretsmanager` | | DNS | `cloud.ftw.energy` A `13.50.84.149` in the Cloudflare zone `ftw.energy`, grey cloud (record `22d28943eea9f6b8584214e1a8256ef1`) | @@ -71,8 +71,8 @@ secret the same way. pipes cores to `/bin/false`, systemd-coredump stores nothing, `compose.yml` sets `ulimits: core: 0`, and `ftwcloudd` marks itself not dumpable. Household readings exist only in this process's memory. -- Nothing comes in but 443. There is no SSH and no key pair; administration - is through SSM. Session logs to a separate account are not set up yet. +- Nothing comes in but 443, TCP and UDP. There is no SSH and no key pair; + administration is through SSM. Session logs to a separate account are not set up yet. - Outbound is open. The host now needs only GitHub and `ghcr.io` (the image and `compose.yml`), AWS endpoints (SSM, Secrets Manager), Let's Encrypt and, from C4, APNs. Narrowing it is still to do. @@ -317,9 +317,12 @@ that holds them adds itself to the drainers in `cmd/ftwcloudd`. ```bash curl -s https://cloud.ftw.energy/healthz curl -sI https://cloud.ftw.energy/source +go run ./cmd/ftwcloudd health https://cloud.ftw.energy/healthz ``` -`ok` means DNS, the certificate and the service all work. `/source` returns +`ok` means DNS, the certificate and the service all work. The last line +checks TCP and HTTP/3 ([http3.md](http3.md)); on the host, +`docker compose ps` shows the container's own check of both. `/source` returns the source archive, and its headers name the commit (`X-Source-Revision`) and the image digest (`X-Source-Image`), which should match what CI built. `/v1/app` answers 501 on a host without the Noise key, and `/v1/now` answers @@ -368,7 +371,8 @@ Cloud data; the images on `ghcr.io` hold only code. ## Not here yet The APNs key ([apns.md](apns.md)) and the Noise key phones dial -([noise.md](noise.md)) come from Parameter Store. Widget tokens need no +([noise.md](noise.md)) come from Parameter Store. HTTP/3 needs UDP 443 in the +security group ([http3.md](http3.md)). Widget tokens need no secret: the registry holds their hashes. Statistics (C7–C8) add secrets, egress and storage when they land. The erase test, canary homes and the external heartbeat that pages Sourceful belong with the first homes. diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index 0842a6c..d27174d 100644 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -13,6 +13,9 @@ sed -i '/\sswap\s/d' /etc/fstab cat > /etc/sysctl.d/99-ftw-cloud.conf <<'SYSCTL' kernel.core_pattern = |/bin/false fs.suid_dumpable = 0 +# quic-go asks for 7.5 MB socket buffers for HTTP/3 (deploy/http3.md). +net.core.rmem_max = 7500000 +net.core.wmem_max = 7500000 SYSCTL sysctl --system >/dev/null install -d /etc/systemd/coredump.conf.d @@ -171,7 +174,8 @@ if ! [[ $rev =~ ^[0-9a-f]{40}$ ]] || ! git merge-base --is-ancestor "$rev" origi fi git checkout --quiet --detach "$rev" -# Only the image line changes; what keys.sh set stays. +# Only the image line changes; what keys.sh set, and FTW_CLOUD_H3=false set +# by hand to turn HTTP/3 off (deploy/http3.md), stay. touch "$env" sed -i '/^FTW_CLOUD_IMAGE=/d' "$env" printf 'FTW_CLOUD_IMAGE=%s\n' "$image" >> "$env" @@ -192,7 +196,7 @@ systemctl enable ftw-cloud-key.service ftw-cloud.service # Show that the host rules hold. swapon --show -sysctl kernel.core_pattern +sysctl kernel.core_pattern net.core.rmem_max # A host that already runs an image from CI runs it again with the new # tooling. A new host waits for the owner to name a digest. diff --git a/deploy/compose.yml b/deploy/compose.yml index 013b3b6..da0af5b 100644 --- a/deploy/compose.yml +++ b/deploy/compose.yml @@ -14,8 +14,12 @@ services: restart: unless-stopped ports: - '443:443' + # HTTP/3 for the plain HTTPS routes (deploy/http3.md). The security + # group must let UDP 443 in before this runs. + - '443:443/udp' command: - -listen=:443 + - -h3=${FTW_CLOUD_H3:-true} - -domain=cloud.ftw.energy - -state=/state - -service-key=/run/secrets/ftw-cloud-service.key @@ -47,6 +51,14 @@ services: - /run/ftw-cloud-keys:/run/ftw-cloud-keys secrets: - ftw-cloud-service.key + # /healthz over TCP and HTTP/3 on the container's own port, with the + # certificate's name. Docker shows the result; nothing restarts on it. + healthcheck: + test: ['CMD', '/usr/local/bin/ftwcloudd', 'health', '-name', 'cloud.ftw.energy', '-h3=${FTW_CLOUD_H3:-true}', 'https://127.0.0.1/healthz'] + interval: 60s + timeout: 15s + start_period: 60s + retries: 3 read_only: true tmpfs: - /tmp diff --git a/deploy/http3.md b/deploy/http3.md new file mode 100644 index 0000000..a22645a --- /dev/null +++ b/deploy/http3.md @@ -0,0 +1,114 @@ +# HTTP/3 on UDP 443 + +`ftwcloudd` serves HTTP/3 on UDP 443 beside TCP 443, with the same +certificate, for the routes that are plain HTTPS: `/v1/app/once`, `/v1/now`, +`/healthz` and `/source`. The sockets, `/v1/box` and `/v1/app`, stay on TCP. +TCP responses carry `Alt-Svc: h3=":443"; ma=86400`, except a WebSocket +upgrade's. + +What it buys, measured by `scripts/netem-once.sh` (a watch's one-shot read, +40 ms round trip): a read on a new connection takes two round trips instead +of three. The pull request that added it has the table. + +## Order + +Open the port before the image that listens on it runs, and shut it after +the image stops saying so. A client that heard `Alt-Svc` races QUIC against +TCP for up to a day; with UDP shut it loses the race every time and gains +nothing. + +### 1. Open UDP 443 in the security group (once) + +In a terminal signed in to the FTW Cloud account (976075258475): + +```bash +aws ec2 authorize-security-group-ingress --region eu-north-1 --group-id sg-098992ad382ecf94a \ + --ip-permissions 'IpProtocol=udp,FromPort=443,ToPort=443,IpRanges=[{CidrIp=0.0.0.0/0,Description=HTTP/3}]' \ + --tag-specifications 'ResourceType=security-group-rule,Tags=[{Key=Project,Value=ftw-cloud}]' +``` + +If the TCP rule also has `::/0`, add `Ipv6Ranges=[{CidrIpv6=::/0,Description=HTTP/3}]` +to match it. Check that the group now lets in 443/tcp and 443/udp and +nothing else: + +```bash +aws ec2 describe-security-group-rules --region eu-north-1 \ + --filters Name=group-id,Values=sg-098992ad382ecf94a \ + --query 'SecurityGroupRules[?!IsEgress].[IpProtocol,FromPort,ToPort,CidrIpv4,CidrIpv6]' --output table +``` + +### 2. Run bootstrap.sh again (once per host) + +It now raises the socket buffer ceiling to 7.5 MB, which quic-go asks for +(`net.core.rmem_max` and `net.core.wmem_max`). Without it HTTP/3 still works +and quic-go logs once that its buffers are small. Run it as under "Bringing +it up" in [README.md](README.md). + +### 3. Deploy + +Deploy the image built from the commit that added HTTP/3, or any later one, +as under "Updating" in [README.md](README.md). Its `compose.yml` maps +`443:443/udp`. No flag is needed: HTTP/3 is on by default. + +### 4. Check it + +From any machine with Go and a checkout of this repository: + +```bash +go run ./cmd/ftwcloudd health https://cloud.ftw.energy/healthz +``` + +It prints `HTTP/1.1 ok` and `HTTP/3.0 ok`, each with its time, and fails if +either fails. A failing HTTP/3 line from outside, with a healthy container, +means the security group or a network in between drops UDP. `curl --http3` +does the same where curl was built with HTTP/3 (macOS's own curl is not). + +The container checks itself the same way every minute, against its own +port, and `docker compose ps` on the host shows `healthy` or `unhealthy`. +Nothing restarts on it; it is there to look at after a deploy. + +## Turning it off + +Without a release: on the host, add `FTW_CLOUD_H3=false` to +`/etc/ftw-cloud/compose.env` and run `systemctl restart ftw-cloud.service`. +`ftw-cloud-deploy` keeps that line. The service then stops listening on UDP +and stops sending `Alt-Svc`. Wait a day, the `ma` in `Alt-Svc`, before you +shut UDP 443 in the security group: + +```bash +aws ec2 revoke-security-group-ingress --region eu-north-1 --group-id sg-098992ad382ecf94a \ + --ip-permissions 'IpProtocol=udp,FromPort=443,ToPort=443,IpRanges=[{CidrIp=0.0.0.0/0}]' +``` + +## Notes + +- **TLS still ends in `ftwcloudd`.** Over HTTP/3, `r.TLS` is the service's + own TLS 1.3 session with the client, and its exporter is the client's + (`TestTheExporterOverHTTP3IsTheClientsOwn`). A UDP load balancer that + forwards packets is fine; anything that ends QUIC is not. +- **Restarts.** When the process goes away, a TCP client hears from the + kernel, but a QUIC client hears nothing. So `ftwcloudd` keeps a stateless + reset key in `quic-reset.key` in the state directory, and the next + process answers an old connection's next packet with a reset: the client + knows at once, not after its idle timeout + (`TestARestartResetsTheOldProcesssConnections`). Keep the file with the + rest of `/state`; a new key only costs one idle timeout per client. +- **Floods.** Past 200 new connections a second the door asks each to prove + its address with a Retry, which costs a real client one round trip and a + spoofed one everything. A handshake gets 5 s. A UDP port that will not + bind turns HTTP/3 off and leaves TCP serving. +- **No 0-RTT.** A request in early data can be replayed, so the door refuses + early data until a route says which of its requests survive a replay. +- **No DNS record for HTTP/3.** An `HTTPS` record with `alpn=h3` would tell + every client, the sockets' too, to try QUIC on the first connection. + `Alt-Svc` reaches only clients of the plain routes. +- **Apple's URLSession** speaks HTTP/3 to this service. With + `assumesHTTP3Capable` it tries HTTP/3 on the first request, and with UDP + refused it fell back to TCP in the same time as without the flag (macOS, + loopback; a network that drops UDP silently is not measured). A + WebSocket in a session that already holds an HTTP/3 connection still + opened over HTTP/1.1. On a port other than 443 URLSession did not move to + HTTP/3 from `Alt-Svc` alone. So the watch's direct path should set + `assumesHTTP3Capable` and keep one URLSession for its requests. +- **Logs.** Neither quic-go nor its HTTP/3 server logs anything here; both + would name the peer's address. diff --git a/go.mod b/go.mod index b151fe4..db79595 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/kms v1.61.4 github.com/fxamacker/cbor/v2 v2.9.4 github.com/gorilla/websocket v1.5.3 + github.com/quic-go/quic-go v0.63.0 github.com/srcfl/ftw/go/protocol v0.0.0-20261015212100-f024251f3248 golang.org/x/crypto v0.57.0 golang.org/x/sys v0.48.0 @@ -32,6 +33,7 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/mattn/go-isatty v0.0.24 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/quic-go/qpack v0.6.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/x448/float16 v0.8.4 // indirect golang.org/x/net v0.58.0 // indirect diff --git a/go.sum b/go.sum index e187d08..68682d1 100644 --- a/go.sum +++ b/go.sum @@ -44,12 +44,24 @@ github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsRe github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4TzM7MFjy0= +github.com/quic-go/go-ossfuzz-seeds v0.1.0/go.mod h1:3IOHRbJIc+L6YKMwfDtJAM9Vj9k0YY4muhuyUYk5tbk= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.63.0 h1:LIFGHI4PFUhhw2dDD1ARHdCff143ffMHwZtbnbuJ78A= +github.com/quic-go/quic-go v0.63.0/go.mod h1:RAro2j2yN9a9EiPACLHT9IB2NXCvGQmmo/alT0yYI0w= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/srcfl/ftw/go/protocol v0.0.0-20261015212100-f024251f3248 h1:56VRLOp7pcN9FyU6ncNri5y1nYCRlG6zD+ZKko46HTA= github.com/srcfl/ftw/go/protocol v0.0.0-20261015212100-f024251f3248/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= +go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= diff --git a/internal/quicdoor/quicdoor.go b/internal/quicdoor/quicdoor.go new file mode 100644 index 0000000..8b35817 --- /dev/null +++ b/internal/quicdoor/quicdoor.go @@ -0,0 +1,223 @@ +// Package quicdoor is FTW Cloud's door over QUIC: UDP on the same port as +// the TCP listener, with the same certificate. Each connection goes to the +// protocol its TLS handshake negotiated by ALPN. Today that is HTTP/3, for +// the routes that are plain HTTPS: /v1/app/once, /v1/now, /healthz and +// /source. The sockets, /v1/box and /v1/app, stay on TCP. +// +// TLS still ends in ftwcloudd, so r.TLS over HTTP/3 is this process's own +// TLS 1.3 session, and its exporter is the one a client on this connection +// computes. +package quicdoor + +import ( + "context" + "crypto/rand" + "crypto/tls" + "errors" + "fmt" + "log/slog" + "net" + "net/http" + "os" + "strconv" + "sync" + "time" + + "github.com/quic-go/quic-go" + "github.com/quic-go/quic-go/http3" +) + +// ALPNH3 is HTTP/3. +const ALPNH3 = http3.NextProtoH3 + +// AltSvcMaxAge is how long a client may remember that HTTP/3 is here. A day, +// so a host that loses UDP costs a client at most a day of racing for it. +const AltSvcMaxAge = 24 * time.Hour + +const ( + // idleTimeout closes a quiet connection. A watch posts once per glance + // and a widget reads once per refresh, so nothing here stays open long. + idleTimeout = 30 * time.Second + // handshakeTimeout bounds the TLS handshake. A client that takes longer + // is better served by the WebSocket it races against. + handshakeTimeout = 5 * time.Second + // maxStreams bounds concurrent requests on one connection. + maxStreams = 32 +) + +// retryRate is how many new connections a second the door takes on trust. +// Past it, each must prove its address with a Retry first (RFC 9000 +// §8.1.2): a round trip more for a real client, and no state for a flood +// of spoofed Initials. Boxes coming back after a restart arrive spread out +// below it. A variable so a test can lower it. +var retryRate = 200.0 + +// newcomers is a token bucket for retryRate, with a second's worth of +// burst. +type newcomers struct { + mu sync.Mutex + tokens float64 + last time.Time + now func() time.Time +} + +// retry reports whether the next new connection must prove its address. +func (n *newcomers) retry(net.Addr) bool { + n.mu.Lock() + defer n.mu.Unlock() + now := n.now() + if n.last.IsZero() { + n.tokens = retryRate + } else { + n.tokens = min(retryRate, n.tokens+now.Sub(n.last).Seconds()*retryRate) + } + n.last = now + if n.tokens < 1 { + return true + } + n.tokens-- + return false +} + +// Door accepts QUIC connections and serves HTTP/3 on them. +type Door struct { + h3 *http3.Server + log *slog.Logger + // ResetKey makes stateless resets (RFC 9000, 10.3). After a restart, a + // client of the old process gets a reset for its next packet instead + // of silence until its idle timeout. Keep it across restarts; nil sends + // none. Set before Serve. + ResetKey *quic.StatelessResetKey +} + +// New serves handler over HTTP/3. A nil log writes nothing. +func New(handler http.Handler, log *slog.Logger) *Door { + if log == nil { + log = slog.New(slog.DiscardHandler) + } + return &Door{log: log, h3: &http3.Server{ + Handler: onlyRequests(handler), + IdleTimeout: idleTimeout, + // quic-go's own log names the peer's address, and an address beside + // a time is what links a household to its traffic. + Logger: nil, + }} +} + +// ResetKeyFile reads the stateless reset key at path, or writes a new one +// there the first time. +func ResetKeyFile(path string) (*quic.StatelessResetKey, error) { + var key quic.StatelessResetKey + b, err := os.ReadFile(path) + switch { + case err == nil && len(b) == len(key): + copy(key[:], b) + return &key, nil + case err == nil: + return nil, fmt.Errorf("%s holds %d bytes, not %d", path, len(b), len(key)) + case !errors.Is(err, os.ErrNotExist): + return nil, err + } + if _, err := rand.Read(key[:]); err != nil { + return nil, err + } + tmp := path + ".tmp" + if err := os.WriteFile(tmp, key[:], 0o600); err != nil { + return nil, err + } + if err := os.Rename(tmp, path); err != nil { + return nil, err + } + return &key, nil +} + +// AltSvc is the header value that points a client at HTTP/3 on port. +func AltSvc(port int) string { + return fmt.Sprintf(`%s=":%d"; ma=%d`, ALPNH3, port, int(AltSvcMaxAge/time.Second)) +} + +// PortOf is the port of a listen address such as ":443". +func PortOf(addr string) (int, error) { + _, p, err := net.SplitHostPort(addr) + if err != nil { + return 0, err + } + return strconv.Atoi(p) +} + +// TLSConfig is base for QUIC: TLS 1.3 and this door's protocols. It keeps +// base's certificate source, so ACME renews both listeners at once. +func TLSConfig(base *tls.Config) *tls.Config { + c := base.Clone() + c.MinVersion = tls.VersionTLS13 + c.NextProtos = []string{ALPNH3} + return c +} + +// Config is the QUIC configuration the door listens with. 0-RTT is off: a +// request in early data can be replayed, and no route here has said which +// of its requests survive that. +func Config() *quic.Config { + return &quic.Config{ + HandshakeIdleTimeout: handshakeTimeout, + MaxIdleTimeout: idleTimeout, + MaxIncomingStreams: maxStreams, + MaxIncomingUniStreams: 8, + Allow0RTT: false, + } +} + +// Serve accepts connections on pc until ctx ends, then closes them. pc is +// the caller's, and Serve closes it on its way out. +func (d *Door) Serve(ctx context.Context, pc net.PacketConn, tlsConf *tls.Config) error { + tr := &quic.Transport{Conn: pc, StatelessResetKey: d.ResetKey, + VerifySourceAddress: (&newcomers{now: time.Now}).retry} + defer tr.Close() + ln, err := tr.Listen(TLSConfig(tlsConf), Config()) + if err != nil { + return err + } + defer ln.Close() + go func() { + <-ctx.Done() + shutdown, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = d.h3.Shutdown(shutdown) + _ = ln.Close() + }() + for { + conn, err := ln.Accept(ctx) + if err != nil { + if ctx.Err() != nil || errors.Is(err, quic.ErrServerClosed) { + return nil + } + return err + } + go d.serve(conn) + } +} + +func (d *Door) serve(conn *quic.Conn) { + switch conn.ConnectionState().TLS.NegotiatedProtocol { + case ALPNH3: + if err := d.h3.ServeQUICConn(conn); err != nil && !errors.Is(err, http.ErrServerClosed) { + d.log.Debug("HTTP/3 connection ended", "err", err) + } + default: + _ = conn.CloseWithError(quic.ApplicationErrorCode(http3.ErrCodeVersionFallback), "") + } +} + +// onlyRequests refuses extended CONNECT. quic-go's HTTP/3 server always +// offers it, but nothing here speaks WebSocket over HTTP/3: /v1/box and +// /v1/app are WebSockets on TCP. +func onlyRequests(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodConnect { + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(http.StatusNotImplemented) + return + } + h.ServeHTTP(w, r) + }) +} diff --git a/internal/quicdoor/quicdoor_test.go b/internal/quicdoor/quicdoor_test.go new file mode 100644 index 0000000..c94efb0 --- /dev/null +++ b/internal/quicdoor/quicdoor_test.go @@ -0,0 +1,299 @@ +package quicdoor + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "errors" + "io" + "math/big" + "net" + "net/http" + "net/http/httptest" + "os" + "sync" + "testing" + "time" + + "github.com/quic-go/quic-go" + "github.com/quic-go/quic-go/http3" + "github.com/srcfl/ftw/go/protocol/cloudwire" +) + +// testCert is a certificate for 127.0.0.1 and the pool that trusts it. +func testCert(t *testing.T) (tls.Certificate, *x509.CertPool) { + t.Helper() + key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + tmpl := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "quicdoor test"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)}, KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}} + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + leaf, _ := x509.ParseCertificate(der) + pool := x509.NewCertPool() + pool.AddCert(leaf) + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key, Leaf: leaf}, pool +} + +// serve runs a door for handler on a loopback UDP port and returns its +// address. +func serve(t *testing.T, handler http.Handler, cert tls.Certificate) string { + t.Helper() + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- New(handler, nil).Serve(ctx, pc, &tls.Config{Certificates: []tls.Certificate{cert}}) + }() + t.Cleanup(func() { + cancel() + if err := <-done; err != nil { + t.Errorf("Serve: %v", err) + } + }) + return pc.LocalAddr().String() +} + +// client is an HTTP/3 client that trusts pool and keeps each connection it +// dials. +type client struct { + *http.Client + mu sync.Mutex + conns []*quic.Conn +} + +func newClient(t *testing.T, pool *x509.CertPool) *client { + c := &client{} + tr := &http3.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS13}} + tr.Dial = func(ctx context.Context, addr string, tlsCfg *tls.Config, cfg *quic.Config) (*quic.Conn, error) { + conn, err := quic.DialAddr(ctx, addr, tlsCfg, cfg) + if err == nil { + c.mu.Lock() + c.conns = append(c.conns, conn) + c.mu.Unlock() + } + return conn, err + } + t.Cleanup(func() { _ = tr.Close() }) + c.Client = &http.Client{Transport: tr, Timeout: 5 * time.Second} + return c +} + +// Over HTTP/3, r.TLS is this process's own TLS 1.3 session with the client: +// its exporter is the value the client computes on its side of the same +// QUIC connection. A box binds its hello to that value, so a proxy that +// ended TLS in between would show here. +func TestTheExporterOverHTTP3IsTheClientsOwn(t *testing.T) { + cert, pool := testCert(t) + addr := serve(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.TLS == nil || r.TLS.Version != tls.VersionTLS13 || r.ProtoMajor != 3 { + http.Error(w, "not TLS 1.3 over HTTP/3", http.StatusUpgradeRequired) + return + } + ekm, err := r.TLS.ExportKeyingMaterial(cloudwire.EKMLabel, nil, cloudwire.EKMBytes) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + _, _ = w.Write(ekm) + }), cert) + c := newClient(t, pool) + resp, err := c.Get("https://" + addr + "/healthz") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + served, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK { + t.Fatalf("%d %s", resp.StatusCode, served) + } + c.mu.Lock() + cs := c.conns[0].ConnectionState().TLS + c.mu.Unlock() + mine, err := cs.ExportKeyingMaterial(cloudwire.EKMLabel, nil, cloudwire.EKMBytes) + if err != nil { + t.Fatal(err) + } + if len(served) != cloudwire.EKMBytes || !bytes.Equal(served, mine) { + t.Fatalf("the service's exporter %x is not the client's %x", served, mine) + } + if cs.NegotiatedProtocol != ALPNH3 { + t.Fatalf("ALPN %q", cs.NegotiatedProtocol) + } +} + +// A connection that offers no protocol the door speaks gets no handshake. +func TestAnUnknownProtocolIsRefused(t *testing.T) { + cert, pool := testCert(t) + addr := serve(t, http.NotFoundHandler(), cert) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + conn, err := quic.DialAddr(ctx, addr, &tls.Config{RootCAs: pool, NextProtos: []string{"ftw-not-a-protocol"}}, nil) + if err == nil { + _ = conn.CloseWithError(0, "") + t.Fatal("a connection without a known ALPN was let in") + } +} + +// quic-go offers extended CONNECT on every HTTP/3 connection. The sockets +// are not served over HTTP/3, so a CONNECT never reaches the routes. +func TestExtendedConnectNeverReachesARoute(t *testing.T) { + reached := false + h := onlyRequests(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { reached = true })) + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodConnect, "/v1/app?home=x", nil) + r.Proto = "websocket" + h.ServeHTTP(w, r) + if reached || w.Code != http.StatusNotImplemented { + t.Fatalf("CONNECT: %d, reached %v", w.Code, reached) + } +} + +func TestAltSvcNamesThePort(t *testing.T) { + if got := AltSvc(443); got != `h3=":443"; ma=86400` { + t.Fatal(got) + } + if p, err := PortOf(":443"); err != nil || p != 443 { + t.Fatal(p, err) + } + if _, err := PortOf("443"); err == nil { + t.Fatal("a bare port was read as an address") + } +} + +// 0-RTT stays off until a route says which of its requests survive a +// replay. +func TestNoEarlyData(t *testing.T) { + if Config().Allow0RTT { + t.Fatal("0-RTT is on") + } +} + +// A deploy or a crash drops the process without a word to its clients. With +// the same reset key, the next process on the port answers an old +// connection's next packet with a stateless reset, so the client knows at +// once instead of waiting out its idle timeout. +func TestARestartResetsTheOldProcesssConnections(t *testing.T) { + cert, pool := testCert(t) + key, err := ResetKeyFile(t.TempDir() + "/quic-reset.key") + if err != nil { + t.Fatal(err) + } + start := func(addr string) (net.PacketConn, chan error) { + pc, err := net.ListenPacket("udp", addr) + if err != nil { + t.Fatal(err) + } + d := New(http.NotFoundHandler(), nil) + d.ResetKey = key + done := make(chan error, 1) + go func() { done <- d.Serve(context.Background(), pc, &tls.Config{Certificates: []tls.Certificate{cert}}) }() + return pc, done + } + old, oldDone := start("127.0.0.1:0") + addr := old.LocalAddr().String() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + conn, err := quic.DialAddr(ctx, addr, &tls.Config{RootCAs: pool, NextProtos: []string{ALPNH3}}, &quic.Config{}) + if err != nil { + t.Fatal(err) + } + defer conn.CloseWithError(0, "") + + // The old process goes without closing anything. + _ = old.Close() + <-oldDone + next, _ := start(addr) + defer next.Close() + + str, err := conn.OpenUniStream() + if err != nil { + t.Fatal(err) + } + _, _ = str.Write(bytes.Repeat([]byte{0}, 200)) + select { + case <-conn.Context().Done(): + case <-time.After(3 * time.Second): + t.Fatal("the old connection still stands 3 s after the restart") + } + var reset *quic.StatelessResetError + if err := context.Cause(conn.Context()); !errors.As(err, &reset) { + t.Fatalf("the connection ended with %v, not a stateless reset", err) + } +} + +func TestTheResetKeyOutlivesTheProcess(t *testing.T) { + path := t.TempDir() + "/quic-reset.key" + first, err := ResetKeyFile(path) + if err != nil { + t.Fatal(err) + } + again, err := ResetKeyFile(path) + if err != nil { + t.Fatal(err) + } + if *first != *again { + t.Fatal("a second start made a new key") + } + if err := os.WriteFile(path, []byte("short"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := ResetKeyFile(path); err == nil { + t.Fatal("a key of the wrong length was taken") + } +} + +// Past retryRate new connections a second, each must prove its address; +// the bucket refills with time. +func TestAFloodOfNewcomersMustProveItsAddress(t *testing.T) { + now := time.Unix(1_800_000_000, 0) + n := &newcomers{now: func() time.Time { return now }} + for i := range int(retryRate) { + if n.retry(nil) { + t.Fatalf("newcomer %d asked to prove its address", i) + } + } + if !n.retry(nil) { + t.Fatal("past the rate, a newcomer went in on trust") + } + now = now.Add(100 * time.Millisecond) + for i := range int(retryRate / 10) { + if n.retry(nil) { + t.Fatalf("after a tenth of a second, newcomer %d asked to prove its address", i) + } + } + if !n.retry(nil) { + t.Fatal("the bucket refilled past a tenth of a second's worth") + } +} + +// A client asked for a Retry proves its address and gets its answer. +func TestAClientAskedForARetryStillGetsIn(t *testing.T) { + old := retryRate + retryRate = 0 + t.Cleanup(func() { retryRate = old }) + cert, pool := testCert(t) + addr := serve(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "ok") + }), cert) + resp, err := newClient(t, pool).Get("https://" + addr + "/") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if b, _ := io.ReadAll(resp.Body); string(b) != "ok" { + t.Fatalf("answer %q", b) + } +} diff --git a/internal/server/h3_test.go b/internal/server/h3_test.go new file mode 100644 index 0000000..0c68d13 --- /dev/null +++ b/internal/server/h3_test.go @@ -0,0 +1,128 @@ +package server + +import ( + "context" + "crypto/tls" + "crypto/x509" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/quic-go/quic-go/http3" + "github.com/srcfl/ftw-cloud/internal/quicdoor" + "github.com/srcfl/ftw/go/protocol" + "github.com/srcfl/ftw/go/protocol/appwire" + "github.com/srcfl/ftw/go/protocol/cloudwire" +) + +// h3 serves the stack's routes over HTTP/3 on a loopback UDP port with the +// stack's certificate, and returns an HTTP/3 client for them and the base +// URL. +func (s *stack) h3() (*http.Client, string) { + s.t.Helper() + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + s.t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + _ = quicdoor.New(s.srv.Config.Handler, nil).Serve(ctx, pc, &tls.Config{Certificates: s.srv.TLS.Certificates}) + }() + pool := x509.NewCertPool() + pool.AddCert(s.srv.Certificate()) + tr := &http3.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS13}} + s.t.Cleanup(func() { + _ = tr.Close() + cancel() + <-done + }) + return &http.Client{Transport: tr, Timeout: 5 * time.Second}, "https://" + pc.LocalAddr().String() +} + +// The watch's one-shot read and its token read work over HTTP/3 as over +// TCP: the same handler, the same Noise, the box's bytes unchanged. +func TestAWatchReadsItsHomeOverHTTP3(t *testing.T) { + s := newStack(t) + box := s.connectBox() + watchKey, _ := appwire.GenerateKeyPair() + box.roster(1, cloudwire.Device{ID: "watch", Key: watchKey.Public(), Role: "watch", + Scopes: []string{protocol.ScopeLiveRead, protocol.ScopePlanRead, protocol.ScopeEventsRead}, Epoch: 1}) + box.live(true) + box.hb() + waitHeld(t, s.st, box.home, watchKey, 2) + + c, base := s.h3() + watch := &watchClient{t: t, http: c, base: base, key: watchKey, service: s.noise.Public(), home: box.home} + view := cloudwire.NewView(box.home, &box.key.PublicKey) + status, snap, all := watch.read(view, cloudwire.Have{}, cloudwire.HzLooking) + if status != http.StatusOK || !snap.Online || len(all) != 2 { + t.Fatalf("read over HTTP/3: %d, online %v, %d envelopes", status, snap.Online, len(all)) + } + if g := view.Grade(time.Now().UnixMilli()); !g.Anchored || g.Sources["m"] != protocol.SourceLive { + t.Fatalf("the read over HTTP/3 grades %+v", g) + } + + status, reply, _ := watch.call(cloudwire.MsgTokenGet, struct{}{}) + env, _ := cloudwire.DecodeEnvelope(reply) + var tok cloudwire.Token + if status != http.StatusOK || env.T != cloudwire.MsgToken || cloudwire.Unmarshal(env.B, &tok) != nil { + t.Fatalf("token.get over HTTP/3: %d %q", status, env.T) + } + req, _ := http.NewRequest(http.MethodGet, base+"/v1/now", nil) + req.Header.Set("Authorization", "Bearer "+tok.Token) + resp, err := c.Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK || resp.ProtoMajor != 3 { + t.Fatalf("/v1/now over HTTP/3: %d %s", resp.StatusCode, resp.Proto) + } + + // The sockets are not served over HTTP/3. + for _, p := range []string{"/v1/box", "/v1/app?home=" + box.home} { + resp, err := c.Get(base + p) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode < 400 { + t.Fatalf("%s over HTTP/3: %d", p, resp.StatusCode) + } + } +} + +// TCP responses point at HTTP/3, except a WebSocket upgrade's: a client +// that learned HTTP/3 there might try its socket over HTTP/3 next. +func TestAltSvcIsOnPlainResponsesOnly(t *testing.T) { + s, home := newServer(t) + s.AltSvc = quicdoor.AltSvc(443) + h := s.Handler() + for _, p := range []string{"/healthz", "/source", "/v1/now", "/v1/app/once?home=" + home} { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, p, nil)) + if w.Header().Get("Alt-Svc") != s.AltSvc { + t.Fatalf("%s: Alt-Svc %q", p, w.Header().Get("Alt-Svc")) + } + } + for _, p := range []string{"/v1/box", "/v1/app?home=" + home} { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, p, nil) + r.Header.Set("Connection", "Upgrade") + r.Header.Set("Upgrade", "websocket") + h.ServeHTTP(w, r) + if w.Header().Get("Alt-Svc") != "" { + t.Fatalf("an upgrade to %s carried Alt-Svc", p) + } + } + s.AltSvc = "" + w := httptest.NewRecorder() + s.Handler().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/healthz", nil)) + if w.Header().Get("Alt-Svc") != "" { + t.Fatal("Alt-Svc without HTTP/3") + } +} diff --git a/internal/server/once_bench_test.go b/internal/server/once_bench_test.go new file mode 100644 index 0000000..527239d --- /dev/null +++ b/internal/server/once_bench_test.go @@ -0,0 +1,191 @@ +package server + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "net" + "net/http" + "net/http/httptest" + "os" + "slices" + "strconv" + "testing" + "time" + + "github.com/quic-go/quic-go/http3" + "github.com/srcfl/ftw/go/protocol" + "github.com/srcfl/ftw/go/protocol/appwire" + "github.com/srcfl/ftw/go/protocol/cloudwire" +) + +// TestOnceReadTimes measures the watch's one-shot read, from the first byte +// the watch sends to the answer decrypted, over HTTP/1.1 (what TCP serves +// today), HTTP/2 and HTTP/3. Cold is a new connection per read, as after +// the watch slept; warm reuses one. It runs only when FTW_BENCH_ONCE names +// the reads per row, because the numbers mean something only on a link +// with delay and loss. scripts/netem-once.sh runs it under netem. +func TestOnceReadTimes(t *testing.T) { + n, _ := strconv.Atoi(os.Getenv("FTW_BENCH_ONCE")) + if n <= 0 { + t.Skip("set FTW_BENCH_ONCE to the reads per row") + } + s := newStack(t) + box := s.connectBox() + // A device may read 30 times a minute, so the reads go round many + // watches. + var watches []appwire.KeyPair + var devices []cloudwire.Device + for i := range 48 { + k, _ := appwire.GenerateKeyPair() + watches = append(watches, k) + devices = append(devices, cloudwire.Device{ID: fmt.Sprintf("watch%02d", i), Key: k.Public(), Role: "watch", + Scopes: []string{protocol.ScopeLiveRead, protocol.ScopePlanRead, protocol.ScopeEventsRead}, Epoch: 1}) + } + box.roster(1, devices...) + box.live(true) + box.hb() + waitHeld(t, s.st, box.home, watches[0], 2) + + pool := x509.NewCertPool() + pool.AddCert(s.srv.Certificate()) + tlsConf := func() *tls.Config { + return &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS13, ClientSessionCache: tls.NewLRUClientSessionCache(4)} + } + + // HTTP/2 beside the service's own HTTP/1.1, for comparison only: the + // service does not offer it on TCP, because its sockets are HTTP/1.1. + h2 := httptest.NewUnstartedServer(s.srv.Config.Handler) + h2.EnableHTTP2 = true + h2.TLS = &tls.Config{MinVersion: tls.VersionTLS13, Certificates: s.srv.TLS.Certificates} + h2.StartTLS() + defer h2.Close() + _, h3base := s.h3() + + type row struct { + name string + proto int + base string + // client returns a client for one read; cold ones are new each time. + client func() (*http.Client, func()) + } + tcp := func(base string, h2 bool, cold bool) func() (*http.Client, func()) { + var shared *http.Transport + mk := func() *http.Transport { + tr := &http.Transport{TLSClientConfig: tlsConf(), ForceAttemptHTTP2: h2, DialContext: (&net.Dialer{}).DialContext} + if !h2 { + tr.TLSClientConfig.NextProtos = []string{"http/1.1"} + } + return tr + } + return func() (*http.Client, func()) { + if cold { + tr := mk() + return &http.Client{Transport: tr}, tr.CloseIdleConnections + } + if shared == nil { + shared = mk() + } + return &http.Client{Transport: shared}, func() {} + } + } + quic := func(cold bool) func() (*http.Client, func()) { + var shared *http3.Transport + return func() (*http.Client, func()) { + if cold { + tr := &http3.Transport{TLSClientConfig: tlsConf()} + return &http.Client{Transport: tr}, func() { _ = tr.Close() } + } + if shared == nil { + shared = &http3.Transport{TLSClientConfig: tlsConf()} + } + return &http.Client{Transport: shared}, func() {} + } + } + rows := []row{ + {"HTTP/1.1 cold", 1, s.srv.URL, tcp(s.srv.URL, false, true)}, + {"HTTP/2 cold", 2, h2.URL, tcp(h2.URL, true, true)}, + {"HTTP/3 cold", 3, h3base, quic(true)}, + {"HTTP/1.1 warm", 1, s.srv.URL, tcp(s.srv.URL, false, false)}, + {"HTTP/2 warm", 2, h2.URL, tcp(h2.URL, true, false)}, + {"HTTP/3 warm", 3, h3base, quic(false)}, + } + label := os.Getenv("FTW_BENCH_LABEL") + fmt.Printf("| link | read | n | p50 ms | p95 ms | max ms | failed |\n|---|---|---|---|---|---|---|\n") + for _, r := range rows { + var took []time.Duration + failed := 0 + for i := range n { + c, done := r.client() + c.Timeout = 10 * time.Second + w := &watchClient{t: t, http: c, base: r.base, key: watches[i%len(watches)], service: s.noise.Public(), home: box.home} + start := time.Now() + proto, ok := w.tryRead() + if ok && proto != r.proto { + t.Fatalf("%s went over HTTP/%d", r.name, proto) + } + if ok { + took = append(took, time.Since(start)) + } else { + failed++ + } + done() + // The service holds one read per device at a time and a + // number per minute; the watch glances, it does not stream. + time.Sleep(20 * time.Millisecond) + } + slices.Sort(took) + q := func(p float64) float64 { + if len(took) == 0 { + return 0 + } + return float64(took[min(len(took)-1, int(p*float64(len(took))))].Microseconds()) / 1000 + } + fmt.Printf("| %s | %s | %d | %.0f | %.0f | %.0f | %d |\n", label, r.name, len(took), q(0.5), q(0.95), q(1), failed) + } +} + +// tryRead is one read that reports failure instead of ending the test, so +// a read lost on a bad link counts as lost. +func (w *watchClient) tryRead() (proto int, ok bool) { + init, err := appwire.NewInitiator(appwire.InitiatorOptions{StaticKey: w.key, RemoteStatic: w.service, + Prologue: cloudwire.OncePrologue(w.home)}) + if err != nil { + return 0, false + } + payload, _ := cloudwire.Unsigned(cloudwire.MsgRead, cloudwire.Read{Hz: cloudwire.HzAway}) + msg1, err := init.WriteMessage(payload) + if err != nil { + return 0, false + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + req, _ := http.NewRequestWithContext(ctx, http.MethodPost, w.base+cloudwire.OncePath+"?"+cloudwire.AppHomeParam+"="+w.home, + bytes.NewReader(msg1)) + req.Header.Set("Content-Type", "application/octet-stream") + resp, err := w.http.Do(req) + if err != nil { + return 0, false + } + defer resp.Body.Close() + answer := make([]byte, 0, 4096) + buf := make([]byte, 4096) + for { + k, err := resp.Body.Read(buf) + answer = append(answer, buf[:k]...) + if err != nil { + break + } + } + if resp.StatusCode != http.StatusOK { + return resp.ProtoMajor, false + } + parts, err := cloudwire.SplitOnceAnswer(answer) + if err != nil { + return resp.ProtoMajor, false + } + _, err = init.ReadMessage(parts[0]) + return resp.ProtoMajor, err == nil +} diff --git a/internal/server/server.go b/internal/server/server.go index 1c1b530..2b6a66e 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -10,6 +10,9 @@ // /oauth/*, /.well-known/oauth-* the authorization server for assistants (package oauth) // /mcp the MCP server, for assistants with a token (package mcp) // +// The same handler serves HTTP/3 (package quicdoor), where the sockets do not +// open: they are WebSockets over TCP. +// // There is no route for staff. Reading a home needs its box's key, a phone in // its roster, or a token bound to one. package server @@ -56,6 +59,11 @@ type Server struct { OAuth *oauth.Server MCP http.Handler Log *slog.Logger + // AltSvc, when set, is the Alt-Svc header that points TCP clients at + // HTTP/3 (package quicdoor). It goes on plain responses only: a client + // that learns HTTP/3 from a WebSocket upgrade might try the socket over + // HTTP/3 next time, and the sockets are TCP only. + AltSvc string } // Handler is the whole HTTP surface. @@ -90,6 +98,9 @@ func (s *Server) Handler() http.Handler { } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Link", `; rel="copyright"; title="AGPL-3.0 source"`) + if s.AltSvc != "" && r.ProtoMajor < 3 && r.Header.Get("Upgrade") == "" { + w.Header().Set("Alt-Svc", s.AltSvc) + } mux.ServeHTTP(w, r) }) } diff --git a/scripts/netem-once.sh b/scripts/netem-once.sh new file mode 100755 index 0000000..d957dd9 --- /dev/null +++ b/scripts/netem-once.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Times the watch's one-shot read over HTTP/1.1, HTTP/2 and HTTP/3 on a slow, +# lossy link (TestOnceReadTimes). The test binary runs in a Linux container +# whose loopback carries netem: 20 ms each way is a 40 ms round trip, and a +# loss applies to each packet. Needs Docker. Each run ends by itself. +# +# scripts/netem-once.sh [reads per row] + +set -euo pipefail + +n=${1:-200} +case $(docker info --format '{{.Architecture}}') in + aarch64 | arm64) arch=arm64 ;; + x86_64 | amd64) arch=amd64 ;; + *) echo "netem-once: unknown Docker architecture" >&2; exit 1 ;; +esac + +out=$(mktemp -d) +trap 'rm -rf "$out"' EXIT +GOOS=linux GOARCH=$arch CGO_ENABLED=0 go test -c -o "$out/server.test" ./internal/server + +run() { + docker run --rm --cap-add NET_ADMIN -v "$out:/w:ro" alpine:3.20 sh -c " + apk add -q iproute2-tc >/dev/null && + tc qdisc add dev lo root netem $2 && + FTW_BENCH_ONCE=$n FTW_BENCH_LABEL='$1' /w/server.test -test.run '^TestOnceReadTimes\$' -test.timeout 15m" | + grep '^| [0-9]' +} + +echo '| link | read | n | p50 ms | p95 ms | max ms | failed |' +echo '|---|---|---|---|---|---|---|' +run '40 ms' 'delay 20ms' +run '40 ms, 1 % loss' 'delay 20ms loss 1%' +run '40 ms, 5 % loss' 'delay 20ms loss 5%'