From 904155faf4ba175f3c4f9602d2fb8111de292a72 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Fri, 16 Oct 2026 07:04:37 +0200 Subject: [PATCH 1/4] feat: HTTP/3 for the plain HTTPS routes, on UDP 443 beside TCP quicdoor serves h3 with the TCP handler and certificate, Alt-Svc points clients at it, ftwcloudd health checks both, and compose maps 443/udp. A stateless reset key in the state directory lets the next process reset an old connection after a restart. Past 200 new connections a second the door asks for a Retry, a handshake gets 5 s, and a UDP port that will not bind leaves TCP serving alone. Co-Authored-By: Claude Opus 5.5 --- README.md | 9 +- cmd/ftwcloudd/health.go | 75 ++++++++ cmd/ftwcloudd/main.go | 42 +++- deploy/README.md | 14 +- deploy/bootstrap.sh | 8 +- deploy/compose.yml | 12 ++ deploy/http3.md | 114 +++++++++++ go.mod | 2 + go.sum | 12 ++ internal/quicdoor/quicdoor.go | 223 +++++++++++++++++++++ internal/quicdoor/quicdoor_test.go | 299 +++++++++++++++++++++++++++++ internal/server/h3_test.go | 128 ++++++++++++ internal/server/once_bench_test.go | 191 ++++++++++++++++++ internal/server/server.go | 11 ++ scripts/netem-once.sh | 34 ++++ 15 files changed, 1163 insertions(+), 11 deletions(-) create mode 100644 cmd/ftwcloudd/health.go create mode 100644 deploy/http3.md create mode 100644 internal/quicdoor/quicdoor.go create mode 100644 internal/quicdoor/quicdoor_test.go create mode 100644 internal/server/h3_test.go create mode 100644 internal/server/once_bench_test.go create mode 100755 scripts/netem-once.sh diff --git a/README.md b/README.md index 39f186a..8877105 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,12 @@ to it yet. most 4 KiB, the answer within 10 s. - Limits per home: 16 sessions, one per device, 256 envelopes queued per phone, and a small message rate from each phone. +- **HTTP/3.** The plain HTTPS routes, `/v1/app/once`, `/v1/now`, + `/healthz` and `/source`, also answer over HTTP/3 on UDP 443, with the + same certificate (`internal/quicdoor`). TCP responses say so in + `Alt-Svc`, except a WebSocket upgrade's. The sockets stay on TCP. A read + on a new connection takes two round trips instead of three; under loss + the gap is larger (`scripts/netem-once.sh`, [deploy/http3.md](deploy/http3.md)). - **Alerts (C4).** A box's signed `event` goes to its phones through APNs, wrapped unchanged: the phone checks the signature, writes its own words and picks the buttons from the verified kind. Every alert has the same aps, @@ -261,7 +267,8 @@ box ──wss, TLS 1.3, signed frames──▶ /v1/box ──▶ store (memory) judges as it judges the same session over the relay. The box closes the uplink on anything else. - **No TLS terminator in front.** Box and service bind the handshake to the - TLS session, so `ftwcloudd` ends TLS itself. A TCP load balancer is fine. + TLS session, so `ftwcloudd` ends TLS itself. A TCP load balancer is fine, + and so is a UDP one that forwards packets; nothing may end QUIC. - **Root on the host can read memory.** The privacy notice says so. ## Run it locally diff --git a/cmd/ftwcloudd/health.go b/cmd/ftwcloudd/health.go new file mode 100644 index 0000000..4b049b8 --- /dev/null +++ b/cmd/ftwcloudd/health.go @@ -0,0 +1,75 @@ +package main + +import ( + "context" + "crypto/tls" + "errors" + "flag" + "fmt" + "io" + "net/http" + "net/url" + "time" + + "github.com/quic-go/quic-go/http3" +) + +// health checks /healthz over TCP and over HTTP/3, the way the container's +// health check and an operator's laptop run it: +// +// ftwcloudd health -name cloud.ftw.energy https://127.0.0.1/healthz +// ftwcloudd health https://cloud.ftw.energy/healthz +// +// Each line names the protocol that answered and how long it took. Either +// failing fails the check: a host whose UDP is shut while Alt-Svc says +// HTTP/3 is there makes every client race for nothing. +func health(args []string) error { + fs := flag.NewFlagSet("health", flag.ExitOnError) + name := fs.String("name", "", "the certificate's name, when the URL names an address") + insecure := fs.Bool("insecure", false, "accept any certificate (local tests only)") + h3 := fs.Bool("h3", true, "check HTTP/3 too") + _ = fs.Parse(args) + if fs.NArg() != 1 { + return errors.New("usage: ftwcloudd health [-name host] [-h3=false] https://host/healthz") + } + u, err := url.Parse(fs.Arg(0)) + if err != nil || u.Scheme != "https" { + return fmt.Errorf("health: %q is not an https URL", fs.Arg(0)) + } + tlsConf := &tls.Config{MinVersion: tls.VersionTLS13, ServerName: *name, InsecureSkipVerify: *insecure} + tcp := &http.Transport{TLSClientConfig: tlsConf.Clone(), DisableKeepAlives: true} + defer tcp.CloseIdleConnections() + clients := []*http.Client{{Transport: tcp}} + if *h3 { + quic := &http3.Transport{TLSClientConfig: tlsConf.Clone()} + defer quic.Close() + clients = append(clients, &http.Client{Transport: quic}) + } + for _, c := range clients { + c.Timeout = 5 * time.Second + start := time.Now() + proto, err := get(c, u.String()) + if err != nil { + return err + } + fmt.Printf("%s ok in %d ms\n", proto, time.Since(start).Milliseconds()) + } + return nil +} + +func get(c *http.Client, u string) (string, error) { + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, u, nil) + if err != nil { + return "", err + } + resp, err := c.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 64)) + if resp.StatusCode != http.StatusOK || string(body) != "ok\n" { + return resp.Proto, fmt.Errorf("health: %s answered %d", resp.Proto, resp.StatusCode) + } + return resp.Proto, nil +} diff --git a/cmd/ftwcloudd/main.go b/cmd/ftwcloudd/main.go index fff992c..316e5a9 100644 --- a/cmd/ftwcloudd/main.go +++ b/cmd/ftwcloudd/main.go @@ -5,6 +5,7 @@ // ftwcloudd keygen -out service.key make the service key; prints its public half // ftwcloudd keygen -noise -out noise.key make the Noise key phones dial; prints its public half // ftwcloudd metrics print the running service's counts, from inside its container +// ftwcloudd health https://host/healthz check /healthz over TCP and HTTP/3 // ftwcloudd [flags] serve package main @@ -41,6 +42,7 @@ import ( "github.com/srcfl/ftw-cloud/internal/metrics" "github.com/srcfl/ftw-cloud/internal/oauth" "github.com/srcfl/ftw-cloud/internal/push" + "github.com/srcfl/ftw-cloud/internal/quicdoor" "github.com/srcfl/ftw-cloud/internal/registry" "github.com/srcfl/ftw-cloud/internal/server" "github.com/srcfl/ftw-cloud/internal/shard" @@ -53,8 +55,12 @@ import ( ) func main() { - if len(os.Args) > 1 && os.Args[1] == "keygen" { - if err := keygen(os.Args[2:]); err != nil { + if len(os.Args) > 1 && (os.Args[1] == "keygen" || os.Args[1] == "health") { + run := keygen + if os.Args[1] == "health" { + run = health + } + if err := run(os.Args[2:]); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } @@ -93,6 +99,7 @@ func serve(args []string) error { metricsAddr := fs.String("metrics-listen", defaultMetricsAddr, "loopback address for /metrics; empty turns it off") restartRate := fs.Int("restart-rate", defaultRestartRate, "sockets a second the next instance can take after a restart; sets the spread of the redial") restartMin := fs.Duration("restart-spread-min", 15*time.Second, "the shortest spread: longer than the port stays closed while the next container starts") + h3 := fs.Bool("h3", true, "also serve HTTP/3 on the same address over UDP, and say so in Alt-Svc") _ = fs.Parse(args) harden() @@ -226,6 +233,28 @@ func serve(args []string) error { tlsConfig.NextProtos = append(tlsConfig.NextProtos, "acme-tls/1") } + // HTTP/3 on UDP, beside TCP, for the plain HTTPS routes. The sockets + // stay on TCP. The UDP port is bound before Alt-Svc says it is there, + // and a port that will not bind leaves TCP serving alone: QUIC is the + // fast path, never the only one. + var door *quicdoor.Door + var udp net.PacketConn + if *h3 { + port, err := quicdoor.PortOf(*listen) + if err != nil { + return err + } + if udp, err = net.ListenPacket("udp", *listen); err != nil { + log.Error("HTTP/3 is off: UDP did not bind; TCP serves alone", "err", err) + } else { + srv.AltSvc = quicdoor.AltSvc(port) + door = quicdoor.New(srv.Handler(), log) + if door.ResetKey, err = quicdoor.ResetKeyFile(filepath.Join(*stateDir, "quic-reset.key")); err != nil { + log.Warn("no stateless resets: a client waits out its idle timeout after a restart", "err", err) + } + } + } + httpSrv := &http.Server{ Addr: *listen, Handler: srv.Handler(), @@ -303,9 +332,16 @@ func serve(args []string) error { } stopPush() }() + if door != nil { + go func() { + if err := door.Serve(ctx, udp, tlsConfig); err != nil { + log.Error("HTTP/3 stopped; TCP serves on", "err", err) + } + }() + } pub, _ := cloudwire.PublicKeyBytesOf(&key.PublicKey) - log.Info("ftwcloudd listening", "addr", *listen, "shard", shards.Self(), "shards", shards.Shards(), + log.Info("ftwcloudd listening", "addr", *listen, "h3", door != nil, "shard", shards.Self(), "shards", shards.Shards(), "service_key", base64.RawURLEncoding.EncodeToString(pub), "noise_key", base64.RawURLEncoding.EncodeToString(noise.Public())) err = httpSrv.ListenAndServeTLS("", "") if err != nil && !errors.Is(err, http.ErrServerClosed) { diff --git a/deploy/README.md b/deploy/README.md index 468264f..e29f956 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -14,7 +14,7 @@ the uplink (srcfl/ftw#1595), after its owner decides to. | Instance | `i-0ecac4cbc6b5f8adc`, `t4g.small`, Amazon Linux 2023 arm64, IMDSv2 required, no key pair | | Disk | `vol-0760cad0e8551e5e4`, 20 GB gp3, encrypted with `aws/ebs` | | Elastic IP | `13.50.84.149` (`eipalloc-0085f3e68b6f36da4`) | -| Security group | `sg-098992ad382ecf94a` `ftw-cloud`: 443/tcp from anywhere, nothing else in | +| Security group | `sg-098992ad382ecf94a` `ftw-cloud`: 443/tcp from anywhere, and 443/udp for HTTP/3 ([http3.md](http3.md)); nothing else in | | Instance role | `ftw-cloud-host`, with `AmazonSSMManagedInstanceCore` and the inline policy `ftw-cloud-service-key` ([`host-policy.json`](host-policy.json)): read one secret, nothing else | | Secret | `ftw-cloud/service-key` in Secrets Manager, eu-north-1, encrypted with `aws/secretsmanager` | | DNS | `cloud.ftw.energy` A `13.50.84.149` in the Cloudflare zone `ftw.energy`, grey cloud (record `22d28943eea9f6b8584214e1a8256ef1`) | @@ -71,8 +71,8 @@ secret the same way. pipes cores to `/bin/false`, systemd-coredump stores nothing, `compose.yml` sets `ulimits: core: 0`, and `ftwcloudd` marks itself not dumpable. Household readings exist only in this process's memory. -- Nothing comes in but 443. There is no SSH and no key pair; administration - is through SSM. Session logs to a separate account are not set up yet. +- Nothing comes in but 443, TCP and UDP. There is no SSH and no key pair; + administration is through SSM. Session logs to a separate account are not set up yet. - Outbound is open. The host now needs only GitHub and `ghcr.io` (the image and `compose.yml`), AWS endpoints (SSM, Secrets Manager), Let's Encrypt and, from C4, APNs. Narrowing it is still to do. @@ -317,9 +317,12 @@ that holds them adds itself to the drainers in `cmd/ftwcloudd`. ```bash curl -s https://cloud.ftw.energy/healthz curl -sI https://cloud.ftw.energy/source +go run ./cmd/ftwcloudd health https://cloud.ftw.energy/healthz ``` -`ok` means DNS, the certificate and the service all work. `/source` returns +`ok` means DNS, the certificate and the service all work. The last line +checks TCP and HTTP/3 ([http3.md](http3.md)); on the host, +`docker compose ps` shows the container's own check of both. `/source` returns the source archive, and its headers name the commit (`X-Source-Revision`) and the image digest (`X-Source-Image`), which should match what CI built. `/v1/app` answers 501 on a host without the Noise key, and `/v1/now` answers @@ -368,7 +371,8 @@ Cloud data; the images on `ghcr.io` hold only code. ## Not here yet The APNs key ([apns.md](apns.md)) and the Noise key phones dial -([noise.md](noise.md)) come from Parameter Store. Widget tokens need no +([noise.md](noise.md)) come from Parameter Store. HTTP/3 needs UDP 443 in the +security group ([http3.md](http3.md)). Widget tokens need no secret: the registry holds their hashes. Statistics (C7–C8) add secrets, egress and storage when they land. The erase test, canary homes and the external heartbeat that pages Sourceful belong with the first homes. diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index 0842a6c..d27174d 100644 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -13,6 +13,9 @@ sed -i '/\sswap\s/d' /etc/fstab cat > /etc/sysctl.d/99-ftw-cloud.conf <<'SYSCTL' kernel.core_pattern = |/bin/false fs.suid_dumpable = 0 +# quic-go asks for 7.5 MB socket buffers for HTTP/3 (deploy/http3.md). +net.core.rmem_max = 7500000 +net.core.wmem_max = 7500000 SYSCTL sysctl --system >/dev/null install -d /etc/systemd/coredump.conf.d @@ -171,7 +174,8 @@ if ! [[ $rev =~ ^[0-9a-f]{40}$ ]] || ! git merge-base --is-ancestor "$rev" origi fi git checkout --quiet --detach "$rev" -# Only the image line changes; what keys.sh set stays. +# Only the image line changes; what keys.sh set, and FTW_CLOUD_H3=false set +# by hand to turn HTTP/3 off (deploy/http3.md), stay. touch "$env" sed -i '/^FTW_CLOUD_IMAGE=/d' "$env" printf 'FTW_CLOUD_IMAGE=%s\n' "$image" >> "$env" @@ -192,7 +196,7 @@ systemctl enable ftw-cloud-key.service ftw-cloud.service # Show that the host rules hold. swapon --show -sysctl kernel.core_pattern +sysctl kernel.core_pattern net.core.rmem_max # A host that already runs an image from CI runs it again with the new # tooling. A new host waits for the owner to name a digest. diff --git a/deploy/compose.yml b/deploy/compose.yml index 013b3b6..da0af5b 100644 --- a/deploy/compose.yml +++ b/deploy/compose.yml @@ -14,8 +14,12 @@ services: restart: unless-stopped ports: - '443:443' + # HTTP/3 for the plain HTTPS routes (deploy/http3.md). The security + # group must let UDP 443 in before this runs. + - '443:443/udp' command: - -listen=:443 + - -h3=${FTW_CLOUD_H3:-true} - -domain=cloud.ftw.energy - -state=/state - -service-key=/run/secrets/ftw-cloud-service.key @@ -47,6 +51,14 @@ services: - /run/ftw-cloud-keys:/run/ftw-cloud-keys secrets: - ftw-cloud-service.key + # /healthz over TCP and HTTP/3 on the container's own port, with the + # certificate's name. Docker shows the result; nothing restarts on it. + healthcheck: + test: ['CMD', '/usr/local/bin/ftwcloudd', 'health', '-name', 'cloud.ftw.energy', '-h3=${FTW_CLOUD_H3:-true}', 'https://127.0.0.1/healthz'] + interval: 60s + timeout: 15s + start_period: 60s + retries: 3 read_only: true tmpfs: - /tmp diff --git a/deploy/http3.md b/deploy/http3.md new file mode 100644 index 0000000..a22645a --- /dev/null +++ b/deploy/http3.md @@ -0,0 +1,114 @@ +# HTTP/3 on UDP 443 + +`ftwcloudd` serves HTTP/3 on UDP 443 beside TCP 443, with the same +certificate, for the routes that are plain HTTPS: `/v1/app/once`, `/v1/now`, +`/healthz` and `/source`. The sockets, `/v1/box` and `/v1/app`, stay on TCP. +TCP responses carry `Alt-Svc: h3=":443"; ma=86400`, except a WebSocket +upgrade's. + +What it buys, measured by `scripts/netem-once.sh` (a watch's one-shot read, +40 ms round trip): a read on a new connection takes two round trips instead +of three. The pull request that added it has the table. + +## Order + +Open the port before the image that listens on it runs, and shut it after +the image stops saying so. A client that heard `Alt-Svc` races QUIC against +TCP for up to a day; with UDP shut it loses the race every time and gains +nothing. + +### 1. Open UDP 443 in the security group (once) + +In a terminal signed in to the FTW Cloud account (976075258475): + +```bash +aws ec2 authorize-security-group-ingress --region eu-north-1 --group-id sg-098992ad382ecf94a \ + --ip-permissions 'IpProtocol=udp,FromPort=443,ToPort=443,IpRanges=[{CidrIp=0.0.0.0/0,Description=HTTP/3}]' \ + --tag-specifications 'ResourceType=security-group-rule,Tags=[{Key=Project,Value=ftw-cloud}]' +``` + +If the TCP rule also has `::/0`, add `Ipv6Ranges=[{CidrIpv6=::/0,Description=HTTP/3}]` +to match it. Check that the group now lets in 443/tcp and 443/udp and +nothing else: + +```bash +aws ec2 describe-security-group-rules --region eu-north-1 \ + --filters Name=group-id,Values=sg-098992ad382ecf94a \ + --query 'SecurityGroupRules[?!IsEgress].[IpProtocol,FromPort,ToPort,CidrIpv4,CidrIpv6]' --output table +``` + +### 2. Run bootstrap.sh again (once per host) + +It now raises the socket buffer ceiling to 7.5 MB, which quic-go asks for +(`net.core.rmem_max` and `net.core.wmem_max`). Without it HTTP/3 still works +and quic-go logs once that its buffers are small. Run it as under "Bringing +it up" in [README.md](README.md). + +### 3. Deploy + +Deploy the image built from the commit that added HTTP/3, or any later one, +as under "Updating" in [README.md](README.md). Its `compose.yml` maps +`443:443/udp`. No flag is needed: HTTP/3 is on by default. + +### 4. Check it + +From any machine with Go and a checkout of this repository: + +```bash +go run ./cmd/ftwcloudd health https://cloud.ftw.energy/healthz +``` + +It prints `HTTP/1.1 ok` and `HTTP/3.0 ok`, each with its time, and fails if +either fails. A failing HTTP/3 line from outside, with a healthy container, +means the security group or a network in between drops UDP. `curl --http3` +does the same where curl was built with HTTP/3 (macOS's own curl is not). + +The container checks itself the same way every minute, against its own +port, and `docker compose ps` on the host shows `healthy` or `unhealthy`. +Nothing restarts on it; it is there to look at after a deploy. + +## Turning it off + +Without a release: on the host, add `FTW_CLOUD_H3=false` to +`/etc/ftw-cloud/compose.env` and run `systemctl restart ftw-cloud.service`. +`ftw-cloud-deploy` keeps that line. The service then stops listening on UDP +and stops sending `Alt-Svc`. Wait a day, the `ma` in `Alt-Svc`, before you +shut UDP 443 in the security group: + +```bash +aws ec2 revoke-security-group-ingress --region eu-north-1 --group-id sg-098992ad382ecf94a \ + --ip-permissions 'IpProtocol=udp,FromPort=443,ToPort=443,IpRanges=[{CidrIp=0.0.0.0/0}]' +``` + +## Notes + +- **TLS still ends in `ftwcloudd`.** Over HTTP/3, `r.TLS` is the service's + own TLS 1.3 session with the client, and its exporter is the client's + (`TestTheExporterOverHTTP3IsTheClientsOwn`). A UDP load balancer that + forwards packets is fine; anything that ends QUIC is not. +- **Restarts.** When the process goes away, a TCP client hears from the + kernel, but a QUIC client hears nothing. So `ftwcloudd` keeps a stateless + reset key in `quic-reset.key` in the state directory, and the next + process answers an old connection's next packet with a reset: the client + knows at once, not after its idle timeout + (`TestARestartResetsTheOldProcesssConnections`). Keep the file with the + rest of `/state`; a new key only costs one idle timeout per client. +- **Floods.** Past 200 new connections a second the door asks each to prove + its address with a Retry, which costs a real client one round trip and a + spoofed one everything. A handshake gets 5 s. A UDP port that will not + bind turns HTTP/3 off and leaves TCP serving. +- **No 0-RTT.** A request in early data can be replayed, so the door refuses + early data until a route says which of its requests survive a replay. +- **No DNS record for HTTP/3.** An `HTTPS` record with `alpn=h3` would tell + every client, the sockets' too, to try QUIC on the first connection. + `Alt-Svc` reaches only clients of the plain routes. +- **Apple's URLSession** speaks HTTP/3 to this service. With + `assumesHTTP3Capable` it tries HTTP/3 on the first request, and with UDP + refused it fell back to TCP in the same time as without the flag (macOS, + loopback; a network that drops UDP silently is not measured). A + WebSocket in a session that already holds an HTTP/3 connection still + opened over HTTP/1.1. On a port other than 443 URLSession did not move to + HTTP/3 from `Alt-Svc` alone. So the watch's direct path should set + `assumesHTTP3Capable` and keep one URLSession for its requests. +- **Logs.** Neither quic-go nor its HTTP/3 server logs anything here; both + would name the peer's address. diff --git a/go.mod b/go.mod index b151fe4..db79595 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/kms v1.61.4 github.com/fxamacker/cbor/v2 v2.9.4 github.com/gorilla/websocket v1.5.3 + github.com/quic-go/quic-go v0.63.0 github.com/srcfl/ftw/go/protocol v0.0.0-20261015212100-f024251f3248 golang.org/x/crypto v0.57.0 golang.org/x/sys v0.48.0 @@ -32,6 +33,7 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/mattn/go-isatty v0.0.24 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/quic-go/qpack v0.6.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/x448/float16 v0.8.4 // indirect golang.org/x/net v0.58.0 // indirect diff --git a/go.sum b/go.sum index e187d08..68682d1 100644 --- a/go.sum +++ b/go.sum @@ -44,12 +44,24 @@ github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsRe github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4TzM7MFjy0= +github.com/quic-go/go-ossfuzz-seeds v0.1.0/go.mod h1:3IOHRbJIc+L6YKMwfDtJAM9Vj9k0YY4muhuyUYk5tbk= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.63.0 h1:LIFGHI4PFUhhw2dDD1ARHdCff143ffMHwZtbnbuJ78A= +github.com/quic-go/quic-go v0.63.0/go.mod h1:RAro2j2yN9a9EiPACLHT9IB2NXCvGQmmo/alT0yYI0w= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/srcfl/ftw/go/protocol v0.0.0-20261015212100-f024251f3248 h1:56VRLOp7pcN9FyU6ncNri5y1nYCRlG6zD+ZKko46HTA= github.com/srcfl/ftw/go/protocol v0.0.0-20261015212100-f024251f3248/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= +go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= diff --git a/internal/quicdoor/quicdoor.go b/internal/quicdoor/quicdoor.go new file mode 100644 index 0000000..8b35817 --- /dev/null +++ b/internal/quicdoor/quicdoor.go @@ -0,0 +1,223 @@ +// Package quicdoor is FTW Cloud's door over QUIC: UDP on the same port as +// the TCP listener, with the same certificate. Each connection goes to the +// protocol its TLS handshake negotiated by ALPN. Today that is HTTP/3, for +// the routes that are plain HTTPS: /v1/app/once, /v1/now, /healthz and +// /source. The sockets, /v1/box and /v1/app, stay on TCP. +// +// TLS still ends in ftwcloudd, so r.TLS over HTTP/3 is this process's own +// TLS 1.3 session, and its exporter is the one a client on this connection +// computes. +package quicdoor + +import ( + "context" + "crypto/rand" + "crypto/tls" + "errors" + "fmt" + "log/slog" + "net" + "net/http" + "os" + "strconv" + "sync" + "time" + + "github.com/quic-go/quic-go" + "github.com/quic-go/quic-go/http3" +) + +// ALPNH3 is HTTP/3. +const ALPNH3 = http3.NextProtoH3 + +// AltSvcMaxAge is how long a client may remember that HTTP/3 is here. A day, +// so a host that loses UDP costs a client at most a day of racing for it. +const AltSvcMaxAge = 24 * time.Hour + +const ( + // idleTimeout closes a quiet connection. A watch posts once per glance + // and a widget reads once per refresh, so nothing here stays open long. + idleTimeout = 30 * time.Second + // handshakeTimeout bounds the TLS handshake. A client that takes longer + // is better served by the WebSocket it races against. + handshakeTimeout = 5 * time.Second + // maxStreams bounds concurrent requests on one connection. + maxStreams = 32 +) + +// retryRate is how many new connections a second the door takes on trust. +// Past it, each must prove its address with a Retry first (RFC 9000 +// §8.1.2): a round trip more for a real client, and no state for a flood +// of spoofed Initials. Boxes coming back after a restart arrive spread out +// below it. A variable so a test can lower it. +var retryRate = 200.0 + +// newcomers is a token bucket for retryRate, with a second's worth of +// burst. +type newcomers struct { + mu sync.Mutex + tokens float64 + last time.Time + now func() time.Time +} + +// retry reports whether the next new connection must prove its address. +func (n *newcomers) retry(net.Addr) bool { + n.mu.Lock() + defer n.mu.Unlock() + now := n.now() + if n.last.IsZero() { + n.tokens = retryRate + } else { + n.tokens = min(retryRate, n.tokens+now.Sub(n.last).Seconds()*retryRate) + } + n.last = now + if n.tokens < 1 { + return true + } + n.tokens-- + return false +} + +// Door accepts QUIC connections and serves HTTP/3 on them. +type Door struct { + h3 *http3.Server + log *slog.Logger + // ResetKey makes stateless resets (RFC 9000, 10.3). After a restart, a + // client of the old process gets a reset for its next packet instead + // of silence until its idle timeout. Keep it across restarts; nil sends + // none. Set before Serve. + ResetKey *quic.StatelessResetKey +} + +// New serves handler over HTTP/3. A nil log writes nothing. +func New(handler http.Handler, log *slog.Logger) *Door { + if log == nil { + log = slog.New(slog.DiscardHandler) + } + return &Door{log: log, h3: &http3.Server{ + Handler: onlyRequests(handler), + IdleTimeout: idleTimeout, + // quic-go's own log names the peer's address, and an address beside + // a time is what links a household to its traffic. + Logger: nil, + }} +} + +// ResetKeyFile reads the stateless reset key at path, or writes a new one +// there the first time. +func ResetKeyFile(path string) (*quic.StatelessResetKey, error) { + var key quic.StatelessResetKey + b, err := os.ReadFile(path) + switch { + case err == nil && len(b) == len(key): + copy(key[:], b) + return &key, nil + case err == nil: + return nil, fmt.Errorf("%s holds %d bytes, not %d", path, len(b), len(key)) + case !errors.Is(err, os.ErrNotExist): + return nil, err + } + if _, err := rand.Read(key[:]); err != nil { + return nil, err + } + tmp := path + ".tmp" + if err := os.WriteFile(tmp, key[:], 0o600); err != nil { + return nil, err + } + if err := os.Rename(tmp, path); err != nil { + return nil, err + } + return &key, nil +} + +// AltSvc is the header value that points a client at HTTP/3 on port. +func AltSvc(port int) string { + return fmt.Sprintf(`%s=":%d"; ma=%d`, ALPNH3, port, int(AltSvcMaxAge/time.Second)) +} + +// PortOf is the port of a listen address such as ":443". +func PortOf(addr string) (int, error) { + _, p, err := net.SplitHostPort(addr) + if err != nil { + return 0, err + } + return strconv.Atoi(p) +} + +// TLSConfig is base for QUIC: TLS 1.3 and this door's protocols. It keeps +// base's certificate source, so ACME renews both listeners at once. +func TLSConfig(base *tls.Config) *tls.Config { + c := base.Clone() + c.MinVersion = tls.VersionTLS13 + c.NextProtos = []string{ALPNH3} + return c +} + +// Config is the QUIC configuration the door listens with. 0-RTT is off: a +// request in early data can be replayed, and no route here has said which +// of its requests survive that. +func Config() *quic.Config { + return &quic.Config{ + HandshakeIdleTimeout: handshakeTimeout, + MaxIdleTimeout: idleTimeout, + MaxIncomingStreams: maxStreams, + MaxIncomingUniStreams: 8, + Allow0RTT: false, + } +} + +// Serve accepts connections on pc until ctx ends, then closes them. pc is +// the caller's, and Serve closes it on its way out. +func (d *Door) Serve(ctx context.Context, pc net.PacketConn, tlsConf *tls.Config) error { + tr := &quic.Transport{Conn: pc, StatelessResetKey: d.ResetKey, + VerifySourceAddress: (&newcomers{now: time.Now}).retry} + defer tr.Close() + ln, err := tr.Listen(TLSConfig(tlsConf), Config()) + if err != nil { + return err + } + defer ln.Close() + go func() { + <-ctx.Done() + shutdown, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = d.h3.Shutdown(shutdown) + _ = ln.Close() + }() + for { + conn, err := ln.Accept(ctx) + if err != nil { + if ctx.Err() != nil || errors.Is(err, quic.ErrServerClosed) { + return nil + } + return err + } + go d.serve(conn) + } +} + +func (d *Door) serve(conn *quic.Conn) { + switch conn.ConnectionState().TLS.NegotiatedProtocol { + case ALPNH3: + if err := d.h3.ServeQUICConn(conn); err != nil && !errors.Is(err, http.ErrServerClosed) { + d.log.Debug("HTTP/3 connection ended", "err", err) + } + default: + _ = conn.CloseWithError(quic.ApplicationErrorCode(http3.ErrCodeVersionFallback), "") + } +} + +// onlyRequests refuses extended CONNECT. quic-go's HTTP/3 server always +// offers it, but nothing here speaks WebSocket over HTTP/3: /v1/box and +// /v1/app are WebSockets on TCP. +func onlyRequests(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodConnect { + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(http.StatusNotImplemented) + return + } + h.ServeHTTP(w, r) + }) +} diff --git a/internal/quicdoor/quicdoor_test.go b/internal/quicdoor/quicdoor_test.go new file mode 100644 index 0000000..c94efb0 --- /dev/null +++ b/internal/quicdoor/quicdoor_test.go @@ -0,0 +1,299 @@ +package quicdoor + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "errors" + "io" + "math/big" + "net" + "net/http" + "net/http/httptest" + "os" + "sync" + "testing" + "time" + + "github.com/quic-go/quic-go" + "github.com/quic-go/quic-go/http3" + "github.com/srcfl/ftw/go/protocol/cloudwire" +) + +// testCert is a certificate for 127.0.0.1 and the pool that trusts it. +func testCert(t *testing.T) (tls.Certificate, *x509.CertPool) { + t.Helper() + key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + tmpl := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "quicdoor test"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)}, KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}} + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + leaf, _ := x509.ParseCertificate(der) + pool := x509.NewCertPool() + pool.AddCert(leaf) + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key, Leaf: leaf}, pool +} + +// serve runs a door for handler on a loopback UDP port and returns its +// address. +func serve(t *testing.T, handler http.Handler, cert tls.Certificate) string { + t.Helper() + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- New(handler, nil).Serve(ctx, pc, &tls.Config{Certificates: []tls.Certificate{cert}}) + }() + t.Cleanup(func() { + cancel() + if err := <-done; err != nil { + t.Errorf("Serve: %v", err) + } + }) + return pc.LocalAddr().String() +} + +// client is an HTTP/3 client that trusts pool and keeps each connection it +// dials. +type client struct { + *http.Client + mu sync.Mutex + conns []*quic.Conn +} + +func newClient(t *testing.T, pool *x509.CertPool) *client { + c := &client{} + tr := &http3.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS13}} + tr.Dial = func(ctx context.Context, addr string, tlsCfg *tls.Config, cfg *quic.Config) (*quic.Conn, error) { + conn, err := quic.DialAddr(ctx, addr, tlsCfg, cfg) + if err == nil { + c.mu.Lock() + c.conns = append(c.conns, conn) + c.mu.Unlock() + } + return conn, err + } + t.Cleanup(func() { _ = tr.Close() }) + c.Client = &http.Client{Transport: tr, Timeout: 5 * time.Second} + return c +} + +// Over HTTP/3, r.TLS is this process's own TLS 1.3 session with the client: +// its exporter is the value the client computes on its side of the same +// QUIC connection. A box binds its hello to that value, so a proxy that +// ended TLS in between would show here. +func TestTheExporterOverHTTP3IsTheClientsOwn(t *testing.T) { + cert, pool := testCert(t) + addr := serve(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.TLS == nil || r.TLS.Version != tls.VersionTLS13 || r.ProtoMajor != 3 { + http.Error(w, "not TLS 1.3 over HTTP/3", http.StatusUpgradeRequired) + return + } + ekm, err := r.TLS.ExportKeyingMaterial(cloudwire.EKMLabel, nil, cloudwire.EKMBytes) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + _, _ = w.Write(ekm) + }), cert) + c := newClient(t, pool) + resp, err := c.Get("https://" + addr + "/healthz") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + served, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK { + t.Fatalf("%d %s", resp.StatusCode, served) + } + c.mu.Lock() + cs := c.conns[0].ConnectionState().TLS + c.mu.Unlock() + mine, err := cs.ExportKeyingMaterial(cloudwire.EKMLabel, nil, cloudwire.EKMBytes) + if err != nil { + t.Fatal(err) + } + if len(served) != cloudwire.EKMBytes || !bytes.Equal(served, mine) { + t.Fatalf("the service's exporter %x is not the client's %x", served, mine) + } + if cs.NegotiatedProtocol != ALPNH3 { + t.Fatalf("ALPN %q", cs.NegotiatedProtocol) + } +} + +// A connection that offers no protocol the door speaks gets no handshake. +func TestAnUnknownProtocolIsRefused(t *testing.T) { + cert, pool := testCert(t) + addr := serve(t, http.NotFoundHandler(), cert) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + conn, err := quic.DialAddr(ctx, addr, &tls.Config{RootCAs: pool, NextProtos: []string{"ftw-not-a-protocol"}}, nil) + if err == nil { + _ = conn.CloseWithError(0, "") + t.Fatal("a connection without a known ALPN was let in") + } +} + +// quic-go offers extended CONNECT on every HTTP/3 connection. The sockets +// are not served over HTTP/3, so a CONNECT never reaches the routes. +func TestExtendedConnectNeverReachesARoute(t *testing.T) { + reached := false + h := onlyRequests(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { reached = true })) + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodConnect, "/v1/app?home=x", nil) + r.Proto = "websocket" + h.ServeHTTP(w, r) + if reached || w.Code != http.StatusNotImplemented { + t.Fatalf("CONNECT: %d, reached %v", w.Code, reached) + } +} + +func TestAltSvcNamesThePort(t *testing.T) { + if got := AltSvc(443); got != `h3=":443"; ma=86400` { + t.Fatal(got) + } + if p, err := PortOf(":443"); err != nil || p != 443 { + t.Fatal(p, err) + } + if _, err := PortOf("443"); err == nil { + t.Fatal("a bare port was read as an address") + } +} + +// 0-RTT stays off until a route says which of its requests survive a +// replay. +func TestNoEarlyData(t *testing.T) { + if Config().Allow0RTT { + t.Fatal("0-RTT is on") + } +} + +// A deploy or a crash drops the process without a word to its clients. With +// the same reset key, the next process on the port answers an old +// connection's next packet with a stateless reset, so the client knows at +// once instead of waiting out its idle timeout. +func TestARestartResetsTheOldProcesssConnections(t *testing.T) { + cert, pool := testCert(t) + key, err := ResetKeyFile(t.TempDir() + "/quic-reset.key") + if err != nil { + t.Fatal(err) + } + start := func(addr string) (net.PacketConn, chan error) { + pc, err := net.ListenPacket("udp", addr) + if err != nil { + t.Fatal(err) + } + d := New(http.NotFoundHandler(), nil) + d.ResetKey = key + done := make(chan error, 1) + go func() { done <- d.Serve(context.Background(), pc, &tls.Config{Certificates: []tls.Certificate{cert}}) }() + return pc, done + } + old, oldDone := start("127.0.0.1:0") + addr := old.LocalAddr().String() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + conn, err := quic.DialAddr(ctx, addr, &tls.Config{RootCAs: pool, NextProtos: []string{ALPNH3}}, &quic.Config{}) + if err != nil { + t.Fatal(err) + } + defer conn.CloseWithError(0, "") + + // The old process goes without closing anything. + _ = old.Close() + <-oldDone + next, _ := start(addr) + defer next.Close() + + str, err := conn.OpenUniStream() + if err != nil { + t.Fatal(err) + } + _, _ = str.Write(bytes.Repeat([]byte{0}, 200)) + select { + case <-conn.Context().Done(): + case <-time.After(3 * time.Second): + t.Fatal("the old connection still stands 3 s after the restart") + } + var reset *quic.StatelessResetError + if err := context.Cause(conn.Context()); !errors.As(err, &reset) { + t.Fatalf("the connection ended with %v, not a stateless reset", err) + } +} + +func TestTheResetKeyOutlivesTheProcess(t *testing.T) { + path := t.TempDir() + "/quic-reset.key" + first, err := ResetKeyFile(path) + if err != nil { + t.Fatal(err) + } + again, err := ResetKeyFile(path) + if err != nil { + t.Fatal(err) + } + if *first != *again { + t.Fatal("a second start made a new key") + } + if err := os.WriteFile(path, []byte("short"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := ResetKeyFile(path); err == nil { + t.Fatal("a key of the wrong length was taken") + } +} + +// Past retryRate new connections a second, each must prove its address; +// the bucket refills with time. +func TestAFloodOfNewcomersMustProveItsAddress(t *testing.T) { + now := time.Unix(1_800_000_000, 0) + n := &newcomers{now: func() time.Time { return now }} + for i := range int(retryRate) { + if n.retry(nil) { + t.Fatalf("newcomer %d asked to prove its address", i) + } + } + if !n.retry(nil) { + t.Fatal("past the rate, a newcomer went in on trust") + } + now = now.Add(100 * time.Millisecond) + for i := range int(retryRate / 10) { + if n.retry(nil) { + t.Fatalf("after a tenth of a second, newcomer %d asked to prove its address", i) + } + } + if !n.retry(nil) { + t.Fatal("the bucket refilled past a tenth of a second's worth") + } +} + +// A client asked for a Retry proves its address and gets its answer. +func TestAClientAskedForARetryStillGetsIn(t *testing.T) { + old := retryRate + retryRate = 0 + t.Cleanup(func() { retryRate = old }) + cert, pool := testCert(t) + addr := serve(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "ok") + }), cert) + resp, err := newClient(t, pool).Get("https://" + addr + "/") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if b, _ := io.ReadAll(resp.Body); string(b) != "ok" { + t.Fatalf("answer %q", b) + } +} diff --git a/internal/server/h3_test.go b/internal/server/h3_test.go new file mode 100644 index 0000000..0c68d13 --- /dev/null +++ b/internal/server/h3_test.go @@ -0,0 +1,128 @@ +package server + +import ( + "context" + "crypto/tls" + "crypto/x509" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/quic-go/quic-go/http3" + "github.com/srcfl/ftw-cloud/internal/quicdoor" + "github.com/srcfl/ftw/go/protocol" + "github.com/srcfl/ftw/go/protocol/appwire" + "github.com/srcfl/ftw/go/protocol/cloudwire" +) + +// h3 serves the stack's routes over HTTP/3 on a loopback UDP port with the +// stack's certificate, and returns an HTTP/3 client for them and the base +// URL. +func (s *stack) h3() (*http.Client, string) { + s.t.Helper() + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + s.t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + _ = quicdoor.New(s.srv.Config.Handler, nil).Serve(ctx, pc, &tls.Config{Certificates: s.srv.TLS.Certificates}) + }() + pool := x509.NewCertPool() + pool.AddCert(s.srv.Certificate()) + tr := &http3.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS13}} + s.t.Cleanup(func() { + _ = tr.Close() + cancel() + <-done + }) + return &http.Client{Transport: tr, Timeout: 5 * time.Second}, "https://" + pc.LocalAddr().String() +} + +// The watch's one-shot read and its token read work over HTTP/3 as over +// TCP: the same handler, the same Noise, the box's bytes unchanged. +func TestAWatchReadsItsHomeOverHTTP3(t *testing.T) { + s := newStack(t) + box := s.connectBox() + watchKey, _ := appwire.GenerateKeyPair() + box.roster(1, cloudwire.Device{ID: "watch", Key: watchKey.Public(), Role: "watch", + Scopes: []string{protocol.ScopeLiveRead, protocol.ScopePlanRead, protocol.ScopeEventsRead}, Epoch: 1}) + box.live(true) + box.hb() + waitHeld(t, s.st, box.home, watchKey, 2) + + c, base := s.h3() + watch := &watchClient{t: t, http: c, base: base, key: watchKey, service: s.noise.Public(), home: box.home} + view := cloudwire.NewView(box.home, &box.key.PublicKey) + status, snap, all := watch.read(view, cloudwire.Have{}, cloudwire.HzLooking) + if status != http.StatusOK || !snap.Online || len(all) != 2 { + t.Fatalf("read over HTTP/3: %d, online %v, %d envelopes", status, snap.Online, len(all)) + } + if g := view.Grade(time.Now().UnixMilli()); !g.Anchored || g.Sources["m"] != protocol.SourceLive { + t.Fatalf("the read over HTTP/3 grades %+v", g) + } + + status, reply, _ := watch.call(cloudwire.MsgTokenGet, struct{}{}) + env, _ := cloudwire.DecodeEnvelope(reply) + var tok cloudwire.Token + if status != http.StatusOK || env.T != cloudwire.MsgToken || cloudwire.Unmarshal(env.B, &tok) != nil { + t.Fatalf("token.get over HTTP/3: %d %q", status, env.T) + } + req, _ := http.NewRequest(http.MethodGet, base+"/v1/now", nil) + req.Header.Set("Authorization", "Bearer "+tok.Token) + resp, err := c.Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK || resp.ProtoMajor != 3 { + t.Fatalf("/v1/now over HTTP/3: %d %s", resp.StatusCode, resp.Proto) + } + + // The sockets are not served over HTTP/3. + for _, p := range []string{"/v1/box", "/v1/app?home=" + box.home} { + resp, err := c.Get(base + p) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode < 400 { + t.Fatalf("%s over HTTP/3: %d", p, resp.StatusCode) + } + } +} + +// TCP responses point at HTTP/3, except a WebSocket upgrade's: a client +// that learned HTTP/3 there might try its socket over HTTP/3 next. +func TestAltSvcIsOnPlainResponsesOnly(t *testing.T) { + s, home := newServer(t) + s.AltSvc = quicdoor.AltSvc(443) + h := s.Handler() + for _, p := range []string{"/healthz", "/source", "/v1/now", "/v1/app/once?home=" + home} { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, p, nil)) + if w.Header().Get("Alt-Svc") != s.AltSvc { + t.Fatalf("%s: Alt-Svc %q", p, w.Header().Get("Alt-Svc")) + } + } + for _, p := range []string{"/v1/box", "/v1/app?home=" + home} { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, p, nil) + r.Header.Set("Connection", "Upgrade") + r.Header.Set("Upgrade", "websocket") + h.ServeHTTP(w, r) + if w.Header().Get("Alt-Svc") != "" { + t.Fatalf("an upgrade to %s carried Alt-Svc", p) + } + } + s.AltSvc = "" + w := httptest.NewRecorder() + s.Handler().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/healthz", nil)) + if w.Header().Get("Alt-Svc") != "" { + t.Fatal("Alt-Svc without HTTP/3") + } +} diff --git a/internal/server/once_bench_test.go b/internal/server/once_bench_test.go new file mode 100644 index 0000000..527239d --- /dev/null +++ b/internal/server/once_bench_test.go @@ -0,0 +1,191 @@ +package server + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "net" + "net/http" + "net/http/httptest" + "os" + "slices" + "strconv" + "testing" + "time" + + "github.com/quic-go/quic-go/http3" + "github.com/srcfl/ftw/go/protocol" + "github.com/srcfl/ftw/go/protocol/appwire" + "github.com/srcfl/ftw/go/protocol/cloudwire" +) + +// TestOnceReadTimes measures the watch's one-shot read, from the first byte +// the watch sends to the answer decrypted, over HTTP/1.1 (what TCP serves +// today), HTTP/2 and HTTP/3. Cold is a new connection per read, as after +// the watch slept; warm reuses one. It runs only when FTW_BENCH_ONCE names +// the reads per row, because the numbers mean something only on a link +// with delay and loss. scripts/netem-once.sh runs it under netem. +func TestOnceReadTimes(t *testing.T) { + n, _ := strconv.Atoi(os.Getenv("FTW_BENCH_ONCE")) + if n <= 0 { + t.Skip("set FTW_BENCH_ONCE to the reads per row") + } + s := newStack(t) + box := s.connectBox() + // A device may read 30 times a minute, so the reads go round many + // watches. + var watches []appwire.KeyPair + var devices []cloudwire.Device + for i := range 48 { + k, _ := appwire.GenerateKeyPair() + watches = append(watches, k) + devices = append(devices, cloudwire.Device{ID: fmt.Sprintf("watch%02d", i), Key: k.Public(), Role: "watch", + Scopes: []string{protocol.ScopeLiveRead, protocol.ScopePlanRead, protocol.ScopeEventsRead}, Epoch: 1}) + } + box.roster(1, devices...) + box.live(true) + box.hb() + waitHeld(t, s.st, box.home, watches[0], 2) + + pool := x509.NewCertPool() + pool.AddCert(s.srv.Certificate()) + tlsConf := func() *tls.Config { + return &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS13, ClientSessionCache: tls.NewLRUClientSessionCache(4)} + } + + // HTTP/2 beside the service's own HTTP/1.1, for comparison only: the + // service does not offer it on TCP, because its sockets are HTTP/1.1. + h2 := httptest.NewUnstartedServer(s.srv.Config.Handler) + h2.EnableHTTP2 = true + h2.TLS = &tls.Config{MinVersion: tls.VersionTLS13, Certificates: s.srv.TLS.Certificates} + h2.StartTLS() + defer h2.Close() + _, h3base := s.h3() + + type row struct { + name string + proto int + base string + // client returns a client for one read; cold ones are new each time. + client func() (*http.Client, func()) + } + tcp := func(base string, h2 bool, cold bool) func() (*http.Client, func()) { + var shared *http.Transport + mk := func() *http.Transport { + tr := &http.Transport{TLSClientConfig: tlsConf(), ForceAttemptHTTP2: h2, DialContext: (&net.Dialer{}).DialContext} + if !h2 { + tr.TLSClientConfig.NextProtos = []string{"http/1.1"} + } + return tr + } + return func() (*http.Client, func()) { + if cold { + tr := mk() + return &http.Client{Transport: tr}, tr.CloseIdleConnections + } + if shared == nil { + shared = mk() + } + return &http.Client{Transport: shared}, func() {} + } + } + quic := func(cold bool) func() (*http.Client, func()) { + var shared *http3.Transport + return func() (*http.Client, func()) { + if cold { + tr := &http3.Transport{TLSClientConfig: tlsConf()} + return &http.Client{Transport: tr}, func() { _ = tr.Close() } + } + if shared == nil { + shared = &http3.Transport{TLSClientConfig: tlsConf()} + } + return &http.Client{Transport: shared}, func() {} + } + } + rows := []row{ + {"HTTP/1.1 cold", 1, s.srv.URL, tcp(s.srv.URL, false, true)}, + {"HTTP/2 cold", 2, h2.URL, tcp(h2.URL, true, true)}, + {"HTTP/3 cold", 3, h3base, quic(true)}, + {"HTTP/1.1 warm", 1, s.srv.URL, tcp(s.srv.URL, false, false)}, + {"HTTP/2 warm", 2, h2.URL, tcp(h2.URL, true, false)}, + {"HTTP/3 warm", 3, h3base, quic(false)}, + } + label := os.Getenv("FTW_BENCH_LABEL") + fmt.Printf("| link | read | n | p50 ms | p95 ms | max ms | failed |\n|---|---|---|---|---|---|---|\n") + for _, r := range rows { + var took []time.Duration + failed := 0 + for i := range n { + c, done := r.client() + c.Timeout = 10 * time.Second + w := &watchClient{t: t, http: c, base: r.base, key: watches[i%len(watches)], service: s.noise.Public(), home: box.home} + start := time.Now() + proto, ok := w.tryRead() + if ok && proto != r.proto { + t.Fatalf("%s went over HTTP/%d", r.name, proto) + } + if ok { + took = append(took, time.Since(start)) + } else { + failed++ + } + done() + // The service holds one read per device at a time and a + // number per minute; the watch glances, it does not stream. + time.Sleep(20 * time.Millisecond) + } + slices.Sort(took) + q := func(p float64) float64 { + if len(took) == 0 { + return 0 + } + return float64(took[min(len(took)-1, int(p*float64(len(took))))].Microseconds()) / 1000 + } + fmt.Printf("| %s | %s | %d | %.0f | %.0f | %.0f | %d |\n", label, r.name, len(took), q(0.5), q(0.95), q(1), failed) + } +} + +// tryRead is one read that reports failure instead of ending the test, so +// a read lost on a bad link counts as lost. +func (w *watchClient) tryRead() (proto int, ok bool) { + init, err := appwire.NewInitiator(appwire.InitiatorOptions{StaticKey: w.key, RemoteStatic: w.service, + Prologue: cloudwire.OncePrologue(w.home)}) + if err != nil { + return 0, false + } + payload, _ := cloudwire.Unsigned(cloudwire.MsgRead, cloudwire.Read{Hz: cloudwire.HzAway}) + msg1, err := init.WriteMessage(payload) + if err != nil { + return 0, false + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + req, _ := http.NewRequestWithContext(ctx, http.MethodPost, w.base+cloudwire.OncePath+"?"+cloudwire.AppHomeParam+"="+w.home, + bytes.NewReader(msg1)) + req.Header.Set("Content-Type", "application/octet-stream") + resp, err := w.http.Do(req) + if err != nil { + return 0, false + } + defer resp.Body.Close() + answer := make([]byte, 0, 4096) + buf := make([]byte, 4096) + for { + k, err := resp.Body.Read(buf) + answer = append(answer, buf[:k]...) + if err != nil { + break + } + } + if resp.StatusCode != http.StatusOK { + return resp.ProtoMajor, false + } + parts, err := cloudwire.SplitOnceAnswer(answer) + if err != nil { + return resp.ProtoMajor, false + } + _, err = init.ReadMessage(parts[0]) + return resp.ProtoMajor, err == nil +} diff --git a/internal/server/server.go b/internal/server/server.go index 1c1b530..2b6a66e 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -10,6 +10,9 @@ // /oauth/*, /.well-known/oauth-* the authorization server for assistants (package oauth) // /mcp the MCP server, for assistants with a token (package mcp) // +// The same handler serves HTTP/3 (package quicdoor), where the sockets do not +// open: they are WebSockets over TCP. +// // There is no route for staff. Reading a home needs its box's key, a phone in // its roster, or a token bound to one. package server @@ -56,6 +59,11 @@ type Server struct { OAuth *oauth.Server MCP http.Handler Log *slog.Logger + // AltSvc, when set, is the Alt-Svc header that points TCP clients at + // HTTP/3 (package quicdoor). It goes on plain responses only: a client + // that learns HTTP/3 from a WebSocket upgrade might try the socket over + // HTTP/3 next time, and the sockets are TCP only. + AltSvc string } // Handler is the whole HTTP surface. @@ -90,6 +98,9 @@ func (s *Server) Handler() http.Handler { } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Link", `; rel="copyright"; title="AGPL-3.0 source"`) + if s.AltSvc != "" && r.ProtoMajor < 3 && r.Header.Get("Upgrade") == "" { + w.Header().Set("Alt-Svc", s.AltSvc) + } mux.ServeHTTP(w, r) }) } diff --git a/scripts/netem-once.sh b/scripts/netem-once.sh new file mode 100755 index 0000000..d957dd9 --- /dev/null +++ b/scripts/netem-once.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Times the watch's one-shot read over HTTP/1.1, HTTP/2 and HTTP/3 on a slow, +# lossy link (TestOnceReadTimes). The test binary runs in a Linux container +# whose loopback carries netem: 20 ms each way is a 40 ms round trip, and a +# loss applies to each packet. Needs Docker. Each run ends by itself. +# +# scripts/netem-once.sh [reads per row] + +set -euo pipefail + +n=${1:-200} +case $(docker info --format '{{.Architecture}}') in + aarch64 | arm64) arch=arm64 ;; + x86_64 | amd64) arch=amd64 ;; + *) echo "netem-once: unknown Docker architecture" >&2; exit 1 ;; +esac + +out=$(mktemp -d) +trap 'rm -rf "$out"' EXIT +GOOS=linux GOARCH=$arch CGO_ENABLED=0 go test -c -o "$out/server.test" ./internal/server + +run() { + docker run --rm --cap-add NET_ADMIN -v "$out:/w:ro" alpine:3.20 sh -c " + apk add -q iproute2-tc >/dev/null && + tc qdisc add dev lo root netem $2 && + FTW_BENCH_ONCE=$n FTW_BENCH_LABEL='$1' /w/server.test -test.run '^TestOnceReadTimes\$' -test.timeout 15m" | + grep '^| [0-9]' +} + +echo '| link | read | n | p50 ms | p95 ms | max ms | failed |' +echo '|---|---|---|---|---|---|---|' +run '40 ms' 'delay 20ms' +run '40 ms, 1 % loss' 'delay 20ms loss 1%' +run '40 ms, 5 % loss' 'delay 20ms loss 5%' From 12ff6f45f7dfdf14b25f3a0e48382651e19b6c1a Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Fri, 16 Oct 2026 07:14:20 +0200 Subject: [PATCH 2/4] feat: /v1/box and /v1/app over QUIC, with 0-RTT a replay cannot use quicdoor takes ftw-cloud-box/1 and ftw-cloud-app/1 beside h3 and hands each connection to the WebSocket's handler through wire.Conn: a control stream that starts with the target, a bulk stream each way for lane 1 tun, and DATAGRAMs for hb and a table-less live. wsclose tracks QUIC connections too, so the drain closes them with 4503. store.TunOut carries Bulk. Door.Close ends live sessions with 1001 on the way out. The go/protocol pin moves to srcfl/ftw#1653. Co-Authored-By: Claude Opus 5.5 --- README.md | 15 +- cmd/ftwcloudd/main.go | 36 +- deploy/bootstrap.sh | 5 +- deploy/compose.yml | 1 + deploy/http3.md | 47 ++- go.mod | 2 +- go.sum | 2 + internal/appsession/appsession.go | 52 ++- internal/quicdoor/conn.go | 545 +++++++++++++++++++++++++++++ internal/quicdoor/early.go | 123 +++++++ internal/quicdoor/early_test.go | 68 ++++ internal/quicdoor/quicdoor.go | 279 +++++++++++++-- internal/quicdoor/quicdoor_test.go | 72 +++- internal/server/e2e_test.go | 7 +- internal/server/early_e2e_test.go | 213 +++++++++++ internal/server/quic_e2e_test.go | 486 +++++++++++++++++++++++++ internal/server/tunnel_e2e_test.go | 75 +++- internal/store/session.go | 14 +- internal/uplink/uplink.go | 57 ++- internal/uplink/uplink_test.go | 2 +- internal/wire/wire.go | 65 ++++ internal/wsclose/wsclose.go | 9 +- 22 files changed, 2059 insertions(+), 116 deletions(-) create mode 100644 internal/quicdoor/conn.go create mode 100644 internal/quicdoor/early.go create mode 100644 internal/quicdoor/early_test.go create mode 100644 internal/server/early_e2e_test.go create mode 100644 internal/server/quic_e2e_test.go create mode 100644 internal/wire/wire.go diff --git a/README.md b/README.md index 8877105..ed1a09b 100644 --- a/README.md +++ b/README.md @@ -93,12 +93,15 @@ to it yet. most 4 KiB, the answer within 10 s. - Limits per home: 16 sessions, one per device, 256 envelopes queued per phone, and a small message rate from each phone. -- **HTTP/3.** The plain HTTPS routes, `/v1/app/once`, `/v1/now`, - `/healthz` and `/source`, also answer over HTTP/3 on UDP 443, with the - same certificate (`internal/quicdoor`). TCP responses say so in - `Alt-Svc`, except a WebSocket upgrade's. The sockets stay on TCP. A read - on a new connection takes two round trips instead of three; under loss - the gap is larger (`scripts/netem-once.sh`, [deploy/http3.md](deploy/http3.md)). +- **QUIC.** UDP 443 carries what TCP 443 does, with the same certificate + (`internal/quicdoor`, [deploy/http3.md](deploy/http3.md)). The plain + HTTPS routes answer over HTTP/3, and TCP responses say so in `Alt-Svc`, + except a WebSocket upgrade's. The uplink and the app session run over + QUIC with the WebSockets' handlers (`internal/wire`): a control stream, a + bulk stream for the tunnel's lane 1, and datagrams for `hb` and a `live` + without its table. Clients race QUIC against the WebSocket. Early data + is answered once per Noise message 1. A phone that moves from Wi-Fi to + cellular keeps its connection and its box session. - **Alerts (C4).** A box's signed `event` goes to its phones through APNs, wrapped unchanged: the phone checks the signature, writes its own words and picks the buttons from the verified kind. Every alert has the same aps, diff --git a/cmd/ftwcloudd/main.go b/cmd/ftwcloudd/main.go index 316e5a9..7c5c299 100644 --- a/cmd/ftwcloudd/main.go +++ b/cmd/ftwcloudd/main.go @@ -100,6 +100,7 @@ func serve(args []string) error { restartRate := fs.Int("restart-rate", defaultRestartRate, "sockets a second the next instance can take after a restart; sets the spread of the redial") restartMin := fs.Duration("restart-spread-min", 15*time.Second, "the shortest spread: longer than the port stays closed while the next container starts") h3 := fs.Bool("h3", true, "also serve HTTP/3 on the same address over UDP, and say so in Alt-Svc") + quicSessions := fs.Bool("quic-sessions", true, "also serve /v1/box and /v1/app over QUIC on the same address over UDP") _ = fs.Parse(args) harden() @@ -233,22 +234,33 @@ func serve(args []string) error { tlsConfig.NextProtos = append(tlsConfig.NextProtos, "acme-tls/1") } - // HTTP/3 on UDP, beside TCP, for the plain HTTPS routes. The sockets - // stay on TCP. The UDP port is bound before Alt-Svc says it is there, - // and a port that will not bind leaves TCP serving alone: QUIC is the - // fast path, never the only one. + // QUIC on UDP, beside TCP: HTTP/3 for the plain HTTPS routes, and the + // uplink and the app session with the handlers their WebSockets have. + // The UDP port is bound before Alt-Svc says it is there, and a port that + // will not bind leaves TCP serving alone: QUIC is the fast path, never + // the only one. var door *quicdoor.Door var udp net.PacketConn - if *h3 { + if *h3 || *quicSessions { port, err := quicdoor.PortOf(*listen) if err != nil { return err } if udp, err = net.ListenPacket("udp", *listen); err != nil { - log.Error("HTTP/3 is off: UDP did not bind; TCP serves alone", "err", err) + log.Error("QUIC is off: UDP did not bind; TCP serves alone", "err", err) } else { - srv.AltSvc = quicdoor.AltSvc(port) - door = quicdoor.New(srv.Handler(), log) + var h3Handler http.Handler + if *h3 { + srv.AltSvc = quicdoor.AltSvc(port) + h3Handler = srv.Handler() + } + door = quicdoor.New(h3Handler, log) + if *quicSessions { + door.Box = func(c *quicdoor.Conn, ekm []byte) { up.ServeConn(c, ekm) } + if srv.App != nil { + door.App = func(c *quicdoor.Conn, home string) { srv.App.ServeConn(c, home) } + } + } if door.ResetKey, err = quicdoor.ResetKeyFile(filepath.Join(*stateDir, "quic-reset.key")); err != nil { log.Warn("no stateless resets: a client waits out its idle timeout after a restart", "err", err) } @@ -333,15 +345,19 @@ func serve(args []string) error { stopPush() }() if door != nil { + // On the way out the live QUIC sessions end with 1001. Silence would + // leave each client to its idle timeout. + defer door.Close() go func() { if err := door.Serve(ctx, udp, tlsConfig); err != nil { - log.Error("HTTP/3 stopped; TCP serves on", "err", err) + log.Error("QUIC stopped; TCP serves on", "err", err) } }() } pub, _ := cloudwire.PublicKeyBytesOf(&key.PublicKey) - log.Info("ftwcloudd listening", "addr", *listen, "h3", door != nil, "shard", shards.Self(), "shards", shards.Shards(), + log.Info("ftwcloudd listening", "addr", *listen, "h3", srv.AltSvc != "", "quic_sessions", door != nil && door.Box != nil, + "shard", shards.Self(), "shards", shards.Shards(), "service_key", base64.RawURLEncoding.EncodeToString(pub), "noise_key", base64.RawURLEncoding.EncodeToString(noise.Public())) err = httpSrv.ListenAndServeTLS("", "") if err != nil && !errors.Is(err, http.ErrServerClosed) { diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index d27174d..b2d1c8e 100644 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -174,8 +174,9 @@ if ! [[ $rev =~ ^[0-9a-f]{40}$ ]] || ! git merge-base --is-ancestor "$rev" origi fi git checkout --quiet --detach "$rev" -# Only the image line changes; what keys.sh set, and FTW_CLOUD_H3=false set -# by hand to turn HTTP/3 off (deploy/http3.md), stay. +# Only the image line changes; what keys.sh set, and FTW_CLOUD_H3=false or +# FTW_CLOUD_QUIC_SESSIONS=false set by hand to turn QUIC off +# (deploy/http3.md), stay. touch "$env" sed -i '/^FTW_CLOUD_IMAGE=/d' "$env" printf 'FTW_CLOUD_IMAGE=%s\n' "$image" >> "$env" diff --git a/deploy/compose.yml b/deploy/compose.yml index da0af5b..62e071f 100644 --- a/deploy/compose.yml +++ b/deploy/compose.yml @@ -20,6 +20,7 @@ services: command: - -listen=:443 - -h3=${FTW_CLOUD_H3:-true} + - -quic-sessions=${FTW_CLOUD_QUIC_SESSIONS:-true} - -domain=cloud.ftw.energy - -state=/state - -service-key=/run/secrets/ftw-cloud-service.key diff --git a/deploy/http3.md b/deploy/http3.md index a22645a..8af95f4 100644 --- a/deploy/http3.md +++ b/deploy/http3.md @@ -1,10 +1,16 @@ -# HTTP/3 on UDP 443 +# QUIC on UDP 443 -`ftwcloudd` serves HTTP/3 on UDP 443 beside TCP 443, with the same -certificate, for the routes that are plain HTTPS: `/v1/app/once`, `/v1/now`, -`/healthz` and `/source`. The sockets, `/v1/box` and `/v1/app`, stay on TCP. -TCP responses carry `Alt-Svc: h3=":443"; ma=86400`, except a WebSocket -upgrade's. +`ftwcloudd` serves QUIC on UDP 443 beside TCP 443, with the same +certificate: + +- HTTP/3 (`h3`) for the routes that are plain HTTPS: `/v1/app/once`, + `/v1/now`, `/healthz` and `/source`. TCP responses carry + `Alt-Svc: h3=":443"; ma=86400`, except a WebSocket upgrade's. +- The uplink (`ftw-cloud-box/1`) and the app session (`ftw-cloud-app/1`), + with the WebSockets' handlers and messages (srcfl/ftw + `docs/protocol-cloud.md`, "QUIC"). Boxes and phones race QUIC against the + WebSocket and fall back to it on their own, so shutting UDP never cuts + anyone off; it only makes them slower. What it buys, measured by `scripts/netem-once.sh` (a watch's one-shot read, 40 ms round trip): a read on a new connection takes two round trips instead @@ -69,11 +75,12 @@ Nothing restarts on it; it is there to look at after a deploy. ## Turning it off -Without a release: on the host, add `FTW_CLOUD_H3=false` to -`/etc/ftw-cloud/compose.env` and run `systemctl restart ftw-cloud.service`. -`ftw-cloud-deploy` keeps that line. The service then stops listening on UDP -and stops sending `Alt-Svc`. Wait a day, the `ma` in `Alt-Svc`, before you -shut UDP 443 in the security group: +Without a release: on the host, add `FTW_CLOUD_H3=false`, +`FTW_CLOUD_QUIC_SESSIONS=false` or both to `/etc/ftw-cloud/compose.env` and +run `systemctl restart ftw-cloud.service`. `ftw-cloud-deploy` keeps those +lines. With both off the service stops listening on UDP and stops sending +`Alt-Svc`. Wait a day, the `ma` in `Alt-Svc`, before you shut UDP 443 in +the security group: ```bash aws ec2 revoke-security-group-ingress --region eu-north-1 --group-id sg-098992ad382ecf94a \ @@ -93,12 +100,22 @@ aws ec2 revoke-security-group-ingress --region eu-north-1 --group-id sg-098992ad knows at once, not after its idle timeout (`TestARestartResetsTheOldProcesssConnections`). Keep the file with the rest of `/state`; a new key only costs one idle timeout per client. + On SIGTERM the door stops taking connections, and on the way out it ends + each live session with 1001, going away + (`TestCloseEndsEachSessionWithGoingAway`). The drain, which closes them + first with 4503 and a spread, goes between the two. - **Floods.** Past 200 new connections a second the door asks each to prove its address with a Retry, which costs a real client one round trip and a - spoofed one everything. A handshake gets 5 s. A UDP port that will not - bind turns HTTP/3 off and leaves TCP serving. -- **No 0-RTT.** A request in early data can be replayed, so the door refuses - early data until a route says which of its requests survive a replay. + spoofed one everything. A handshake gets 5 s, and the door holds at most + a set number of handshakes and connections at once. A UDP port that will + not bind turns QUIC off and leaves TCP serving. +- **0-RTT.** The sessions take early data under the protocol's rules: a + box's target may get its `challenge` before the handshake ends, and a + phone's session waits for the handshake, so a replayed flight changes + nothing. A ticket allows early data for a day. HTTP/3 requests in early + data wait for the handshake. A restart draws new ticket keys. +- **A UDP load balancer** in front would have to keep each QUIC connection + on one host by its connection id; with one host there is none. - **No DNS record for HTTP/3.** An `HTTPS` record with `alpn=h3` would tell every client, the sockets' too, to try QUIC on the first connection. `Alt-Svc` reaches only clients of the plain routes. diff --git a/go.mod b/go.mod index db79595..f76725a 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/fxamacker/cbor/v2 v2.9.4 github.com/gorilla/websocket v1.5.3 github.com/quic-go/quic-go v0.63.0 - github.com/srcfl/ftw/go/protocol v0.0.0-20261015212100-f024251f3248 + github.com/srcfl/ftw/go/protocol v0.0.0-20261016050141-c49da79dfc79 golang.org/x/crypto v0.57.0 golang.org/x/sys v0.48.0 modernc.org/sqlite v1.59.0 diff --git a/go.sum b/go.sum index 68682d1..aec6aad 100644 --- a/go.sum +++ b/go.sum @@ -54,6 +54,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/srcfl/ftw/go/protocol v0.0.0-20261015212100-f024251f3248 h1:56VRLOp7pcN9FyU6ncNri5y1nYCRlG6zD+ZKko46HTA= github.com/srcfl/ftw/go/protocol v0.0.0-20261015212100-f024251f3248/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= +github.com/srcfl/ftw/go/protocol v0.0.0-20261016050141-c49da79dfc79 h1:BnMVCZe7r6zmaCQeiImMeZ5AlJOqL52ok8ef8s39UKo= +github.com/srcfl/ftw/go/protocol v0.0.0-20261016050141-c49da79dfc79/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= diff --git a/internal/appsession/appsession.go b/internal/appsession/appsession.go index c34c22b..4b94a4a 100644 --- a/internal/appsession/appsession.go +++ b/internal/appsession/appsession.go @@ -31,6 +31,7 @@ import ( "github.com/srcfl/ftw-cloud/internal/registry" "github.com/srcfl/ftw-cloud/internal/shard" "github.com/srcfl/ftw-cloud/internal/store" + "github.com/srcfl/ftw-cloud/internal/wire" "github.com/srcfl/ftw-cloud/internal/wsclose" "github.com/srcfl/ftw/go/protocol/appwire" "github.com/srcfl/ftw/go/protocol/cloudwire" @@ -200,9 +201,19 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { if err != nil { return } + s.ServeConn(conn, home) +} + +// ServeConn runs one phone's session on conn, a WebSocket or QUIC (package +// quicdoor), for home. It closes conn. +func (s *Server) ServeConn(conn wire.Conn, home string) { defer conn.Close() appSockets.Add(1) defer appSockets.Add(-1) + if !cloudwire.ValidHome(home) { + _ = conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(cloudwire.CloseProtocol, ""), time.Now().Add(time.Second)) + return + } conn.SetReadLimit(maxPhoneMessage) sock := s.sockets.Add(conn) defer s.sockets.Remove(sock) @@ -269,7 +280,7 @@ var errProtocol = errors.New("appsession: the phone broke the protocol") // serve runs one session and returns the close code to end it with. Once the // session's reader runs, reading is set to a channel that closes when it // stops, and only that reader may read the socket. -func (s *Server) serve(conn *websocket.Conn, home string, reading *<-chan struct{}) (int, error) { +func (s *Server) serve(conn wire.Conn, home string, reading *<-chan struct{}) (int, error) { start := time.Now() _ = conn.SetReadDeadline(time.Now().Add(handshakeTimeout)) kind, msg1, err := conn.ReadMessage() @@ -445,7 +456,7 @@ func (s *Server) watchOnce(home string) { // phoneConn is one phone's session after the handshake. type phoneConn struct { s *Server - conn *websocket.Conn + conn wire.Conn tr *appwire.Transport dev *store.Handle home string @@ -476,6 +487,11 @@ type phoneConn struct { tunMu sync.Mutex tunSid uint32 hasTun bool + + // heard is set by the phone's first transport message, which proves it + // holds message 2. Until then everything goes on the control stream, + // behind message 2: over QUIC a datagram could overtake it. + heard atomic.Bool } func (c *phoneConn) setLooking(on bool) { @@ -503,26 +519,43 @@ func (c *phoneConn) stopLooking() { // write sends one WebSocket message. Only the session's own goroutine // writes; pings go through WriteControl, which may run beside it. -func (c *phoneConn) write(msg []byte) error { +func (c *phoneConn) write(msg []byte) error { return c.writeAs(websocket.BinaryMessage, msg) } + +// writeAs sends one message the way kind says, over QUIC; a WebSocket +// carries every kind as a binary message. +func (c *phoneConn) writeAs(kind int, msg []byte) error { if err := c.conn.SetWriteDeadline(time.Now().Add(writeTimeout)); err != nil { return err } - return c.conn.WriteMessage(websocket.BinaryMessage, msg) + return c.conn.WriteMessage(wire.Kind(c.conn, kind), msg) } // forward sends one box envelope as the box sent it. An envelope too large // for a Noise message cannot reach a phone; it is dropped, and the phone sees // the gap through the next hb. -func (c *phoneConn) forward(env []byte) error { +func (c *phoneConn) forward(env []byte) error { return c.forwardAs(websocket.BinaryMessage, env) } + +// forwardAs is forward over the path kind names. Over QUIC an hb, or a live +// without its source table, goes as a datagram: the newest wins, and a +// lost one shows as a gap in the phone's freshness rules. +func (c *phoneConn) forwardAs(kind int, env []byte) error { frame, err := cloudwire.EncodeFrame(env) if err != nil || len(frame)+appwire.TransportOverhead > maxNoiseMessage { return nil } + switch { + case !c.heard.Load(): + kind = websocket.BinaryMessage + case kind == websocket.BinaryMessage && wire.Kind(c.conn, wire.Datagram) == wire.Datagram: + if e, err := cloudwire.DecodeEnvelope(env); err == nil && cloudwire.Datagram(e) { + kind = wire.Datagram + } + } msg, err := c.tr.Encrypt(frame) if err != nil { return err } - return c.write(msg) + return c.writeAs(kind, msg) } // run forwards until the store ends the session, the phone leaves or breaks @@ -557,7 +590,11 @@ func (c *phoneConn) run(sess *store.Session) (int, error) { if err != nil { return 0, err } - if err := c.forward(env); err != nil { + kind := websocket.BinaryMessage + if t.Bulk { + kind = wire.Bulk + } + if err := c.forwardAs(kind, env); err != nil { return 0, err } tunUp.Observe(t.Waited()) @@ -598,6 +635,7 @@ func (c *phoneConn) readLoop() (int, error) { if err != nil { return cloudwire.CloseProtocol, err } + c.heard.Store(true) if code, err := c.apply(frame); err != nil { return code, err } diff --git a/internal/quicdoor/conn.go b/internal/quicdoor/conn.go new file mode 100644 index 0000000..f5512a4 --- /dev/null +++ b/internal/quicdoor/conn.go @@ -0,0 +1,545 @@ +package quicdoor + +import ( + "bufio" + "context" + "crypto/tls" + "encoding/binary" + "errors" + "io" + "net" + "sync" + "sync/atomic" + "time" + + "github.com/gorilla/websocket" + "github.com/quic-go/quic-go" + "github.com/srcfl/ftw-cloud/internal/wire" + "github.com/srcfl/ftw/go/protocol/cloudwire" +) + +// Conn is one QUIC connection to a box or a phone as a wire.Conn, so the +// uplink and the app session serve it as they serve a WebSocket. Binary +// messages go on the control stream, wire.Bulk on a unidirectional stream +// each way, and wire.Datagram as QUIC DATAGRAMs. docs/protocol-cloud.md in +// srcfl/ftw, "QUIC", is the framing. +// +// QUIC's own keepalive and idle timeout stand in for the WebSocket ping: a +// ping succeeds while the connection lives, and its pong is immediate. +type Conn struct { + qc *quic.Conn + ctrl *quic.Stream + r *bufio.Reader + // bulkIn says whether the peer may open a bulk stream: a box may, a + // phone sends everything on the control stream. + bulkIn bool + + limit atomic.Int64 + start sync.Once + in chan inbound + // pending holds messages the door read before handing the connection + // over; ReadMessage returns them first. + pending [][]byte + + mu sync.Mutex + deadline time.Time + kick chan struct{} + pong func(string) error + writeDL time.Time + readErr error + + writeMu sync.Mutex + bulkMu sync.Mutex + bulk *quic.SendStream + noBulk bool + closed sync.Once + + // A client's early data: what it wrote before the handshake completed, + // sent again on a new control stream if the service rejects it. resumed + // closes once that is done. Under writeMu. + early bool + earlySent [][]byte + resumed chan struct{} +} + +type inbound struct { + kind int + msg []byte + err error +} + +func newConn(qc *quic.Conn, ctrl *quic.Stream, r *bufio.Reader, bulkIn bool) *Conn { + c := &Conn{qc: qc, ctrl: ctrl, r: r, bulkIn: bulkIn, in: make(chan inbound), kick: make(chan struct{}, 1)} + c.limit.Store(cloudwire.MaxTunFrame) + return c +} + +var _ wire.Conn = (*Conn)(nil) +var _ wire.Carrier = (*Conn)(nil) + +// Exporter is the TLS exporter of the session inside this connection, which +// a box binds its hello to. +func (c *Conn) Exporter() ([]byte, error) { + cs := c.qc.ConnectionState().TLS + return cs.ExportKeyingMaterial(cloudwire.EKMLabel, nil, cloudwire.EKMBytes) +} + +// Transport is the registry's name for QUIC. +func (c *Conn) Transport() string { return cloudwire.TransportQUIC } + +// Carries reports whether a message type can go now: bulk while the peer +// grants a stream for it, a datagram when both ends offered datagrams. +func (c *Conn) Carries(messageType int) bool { + switch messageType { + case websocket.BinaryMessage: + return true + case wire.Bulk: + c.bulkMu.Lock() + defer c.bulkMu.Unlock() + return !c.noBulk + case wire.Datagram: + d := c.qc.ConnectionState().SupportsDatagrams + return d.Local && d.Remote + } + return false +} + +// ReadMessage returns the next message from any of the connection's paths, +// with the type it came by. +func (c *Conn) ReadMessage() (int, []byte, error) { + if len(c.pending) > 0 { + m := c.pending[0] + c.pending = c.pending[1:] + return websocket.BinaryMessage, m, nil + } + c.start.Do(c.readers) + for { + c.mu.Lock() + if c.readErr != nil { + err := c.readErr + c.mu.Unlock() + return 0, nil, err + } + deadline := c.deadline + c.mu.Unlock() + var expired <-chan time.Time + var timer *time.Timer + if !deadline.IsZero() { + wait := time.Until(deadline) + if wait <= 0 { + return 0, nil, c.fail(errTimeout) + } + timer = time.NewTimer(wait) + expired = timer.C + } + select { + case m := <-c.in: + stop(timer) + if m.err != nil { + return 0, nil, c.fail(m.err) + } + return m.kind, m.msg, nil + case <-expired: + return 0, nil, c.fail(errTimeout) + case <-c.kick: + stop(timer) + } + } +} + +func stop(t *time.Timer) { + if t != nil { + t.Stop() + } +} + +// fail makes err the answer to every later read, as gorilla does. +func (c *Conn) fail(err error) error { + c.mu.Lock() + defer c.mu.Unlock() + if c.readErr == nil { + c.readErr = err + } + return c.readErr +} + +// readers start on the first read, once the handler has set its limit. +func (c *Conn) readers() { + ctx := c.qc.Context() + deliver := func(m inbound) bool { + if m.err != nil { + // An error is kept, not queued: the connection may already be + // gone, and a reader waiting on it must still hear why. + c.fail(m.err) + select { + case c.kick <- struct{}{}: + default: + } + return false + } + select { + case c.in <- m: + return true + case <-ctx.Done(): + return false + } + } + go func() { + for { + msg, err := cloudwire.ReadMessage(c.reader(), int(c.limit.Load())) + if errors.Is(err, quic.Err0RTTRejected) && c.resumed != nil { + // The service rejected the early data; the control stream + // starts again once the handshake is done. + select { + case <-c.resumed: + continue + case <-ctx.Done(): + return + } + } + if err != nil { + deliver(inbound{err: c.readError(err)}) + return + } + if !deliver(inbound{kind: websocket.BinaryMessage, msg: msg}) { + return + } + } + }() + go func() { + for { + d, err := c.qc.ReceiveDatagram(ctx) + if err != nil { + return + } + if !deliver(inbound{kind: wire.Datagram, msg: d}) { + return + } + } + }() + go func() { + // The peer gets one bulk stream, if any; a second stream of either + // kind breaks the protocol. + gotBulk := false + for { + str, err := c.qc.AcceptUniStream(ctx) + if err != nil { + return + } + if !c.bulkIn || gotBulk { + c.closeWith(cloudwire.CloseProtocol, "") + return + } + gotBulk = true + go func() { + r := bufio.NewReader(str) + for { + msg, err := cloudwire.ReadMessage(r, int(c.limit.Load())) + if err != nil { + if !errors.Is(err, io.EOF) { + deliver(inbound{err: c.readError(err)}) + } + return + } + if !deliver(inbound{kind: wire.Bulk, msg: msg}) { + return + } + } + }() + } + }() + go func() { + if str, err := c.qc.AcceptStream(ctx); err == nil { + str.CancelRead(0) + c.closeWith(cloudwire.CloseProtocol, "") + } + }() +} + +// readError is what a read reports: the peer's close as a WebSocket close, +// so the handlers' checks read the same over both. +func (c *Conn) readError(err error) error { + if ae, ok := errors.AsType[*quic.ApplicationError](err); ok { + return &websocket.CloseError{Code: int(ae.ErrorCode), Text: ae.ErrorMessage} + } + if errors.Is(err, io.EOF) { + return &websocket.CloseError{Code: websocket.CloseNormalClosure} + } + if errors.Is(err, cloudwire.ErrQUICMessage) { + c.closeWith(cloudwire.CloseProtocol, "") + return websocket.ErrReadLimit + } + return err +} + +// WriteMessage sends one message by the path its type names. A datagram +// that does not fit, or a bulk message the peer has no stream for, goes on +// the control stream. +func (c *Conn) WriteMessage(messageType int, data []byte) error { + switch messageType { + case websocket.BinaryMessage: + return c.writeStream(data) + case wire.Datagram: + if c.Carries(wire.Datagram) { + err := c.qc.SendDatagram(data) + if err == nil { + return nil + } + if _, tooLarge := errors.AsType[*quic.DatagramTooLargeError](err); !tooLarge { + return err + } + } + return c.writeStream(data) + case wire.Bulk: + if ok, err := c.writeBulk(data); ok || err != nil { + return err + } + return c.writeStream(data) + } + return errors.New("quicdoor: no such message type") +} + +// reader is the control stream's reader, which a rejected early flight +// replaces. +func (c *Conn) reader() *bufio.Reader { + c.writeMu.Lock() + defer c.writeMu.Unlock() + return c.r +} + +func (c *Conn) writeStream(data []byte) error { + c.writeMu.Lock() + defer c.writeMu.Unlock() + if c.early { + c.earlySent = append(c.earlySent, data) + } + c.mu.Lock() + dl := c.writeDL + c.mu.Unlock() + if err := c.ctrl.SetWriteDeadline(dl); err != nil { + return err + } + _, err := c.ctrl.Write(cloudwire.AppendMessage(nil, data)) + return err +} + +// writeBulk sends on the bulk stream, opening it the first time. False +// means the peer grants no stream for it. +func (c *Conn) writeBulk(data []byte) (bool, error) { + c.bulkMu.Lock() + defer c.bulkMu.Unlock() + if c.noBulk { + return false, nil + } + if c.bulk == nil { + s, err := c.qc.OpenUniStream() + if err != nil { + c.noBulk = true + return false, nil + } + c.bulk = s + } + c.mu.Lock() + dl := c.writeDL + c.mu.Unlock() + if err := c.bulk.SetWriteDeadline(dl); err != nil { + return true, err + } + _, err := c.bulk.Write(cloudwire.AppendMessage(nil, data)) + return true, err +} + +// WriteControl stands in for the WebSocket's control frames. A ping +// succeeds while the connection lives, since QUIC closes one that has been +// silent for its idle timeout; a close ends the connection with the code as +// its application error. +func (c *Conn) WriteControl(messageType int, data []byte, _ time.Time) error { + switch messageType { + case websocket.PingMessage: + if err := c.qc.Context().Err(); err != nil { + return net.ErrClosed + } + c.mu.Lock() + pong := c.pong + c.mu.Unlock() + if pong != nil { + return pong(string(data)) + } + return nil + case websocket.CloseMessage: + code, text := websocket.CloseNoStatusReceived, "" + if len(data) >= 2 { + code, text = int(binary.BigEndian.Uint16(data)), string(data[2:]) + } + c.closeWith(code, text) + return nil + case websocket.PongMessage: + return nil + } + return errors.New("quicdoor: no such control message") +} + +func (c *Conn) closeWith(code int, text string) { + c.closed.Do(func() { + _ = c.qc.CloseWithError(quic.ApplicationErrorCode(code), text) + }) +} + +// SetReadDeadline bounds the next read, as on a WebSocket. +func (c *Conn) SetReadDeadline(t time.Time) error { + c.mu.Lock() + c.deadline = t + c.mu.Unlock() + select { + case c.kick <- struct{}{}: + default: + } + return nil +} + +// SetWriteDeadline bounds each later write. +func (c *Conn) SetWriteDeadline(t time.Time) error { + c.mu.Lock() + c.writeDL = t + c.mu.Unlock() + return nil +} + +// SetReadLimit bounds a message on either stream. +func (c *Conn) SetReadLimit(limit int64) { c.limit.Store(limit) } + +// SetPongHandler runs on each ping that finds the connection alive. +func (c *Conn) SetPongHandler(h func(appData string) error) { + c.mu.Lock() + c.pong = h + c.mu.Unlock() +} + +// Close ends the connection. A close code sent before wins. +func (c *Conn) Close() error { + c.closeWith(websocket.CloseGoingAway, "") + return nil +} + +// Context ends when the connection does. +func (c *Conn) Context() context.Context { return c.qc.Context() } + +// errTimeout is a read past its deadline, a net.Error like the WebSocket's. +var errTimeout error = timeoutError{} + +type timeoutError struct{} + +func (timeoutError) Error() string { return "quicdoor: read deadline passed" } +func (timeoutError) Timeout() bool { return true } +func (timeoutError) Temporary() bool { return true } + +// ClientConfig is how a box or a phone dials: a keepalive well inside the +// NAT timeouts, the shared idle timeout, and datagrams on. +func ClientConfig() *quic.Config { + return &quic.Config{ + HandshakeIdleTimeout: handshakeTimeout, + MaxIdleTimeout: cloudwire.QUICIdleTimeout, + KeepAlivePeriod: cloudwire.QUICKeepAlive, + EnableDatagrams: true, + } +} + +// Dial is a client's side: QUIC to addr with alpn, then the control stream +// with target first on it. Tests and measurements use it; the box's own +// dialler lives in srcfl/ftw. A nil conf is ClientConfig. With a session +// cache in tlsConf and a ticket in it, Dial returns before the handshake is +// done, and what the client writes until then goes as early data. +func Dial(ctx context.Context, addr string, tlsConf *tls.Config, alpn, target string, conf *quic.Config) (*Conn, error) { + tc := tlsConf.Clone() + tc.NextProtos = []string{alpn} + tc.MinVersion = tls.VersionTLS13 + if conf == nil { + conf = ClientConfig() + } + udp, err := net.ResolveUDPAddr("udp", addr) + if err != nil { + return nil, err + } + pc, err := net.ListenUDP("udp", nil) + if err != nil { + return nil, err + } + // Connection ids of its own, so the connection can move to another + // path: a phone from Wi-Fi to cellular. + tr := &quic.Transport{Conn: pc, ConnectionIDLength: 4} + dial := tr.Dial + if tc.ClientSessionCache != nil { + dial = tr.DialEarly + } + qc, err := dial(ctx, udp, tc, conf) + if err != nil { + _ = tr.Close() + return nil, err + } + go func() { + <-qc.Context().Done() + _ = tr.Close() + }() + return open(ctx, qc, target) +} + +// open starts the control stream on a dialled connection. +func open(ctx context.Context, qc *quic.Conn, target string) (*Conn, error) { + str, err := qc.OpenStreamSync(ctx) + if err != nil { + _ = qc.CloseWithError(0, "") + return nil, err + } + if _, err := str.Write(cloudwire.AppendMessage(nil, []byte(target))); err != nil { + _ = qc.CloseWithError(0, "") + return nil, err + } + c := newConn(qc, str, bufio.NewReader(str), true) + if !done(qc) { + c.early, c.earlySent, c.resumed = true, [][]byte{[]byte(target)}, make(chan struct{}) + go c.afterEarly() + } + return c, nil +} + +// afterEarly waits for the handshake. If the service rejected the early +// data, it opens the control stream again and sends what went early. +func (c *Conn) afterEarly() { + ctx := c.qc.Context() + select { + case <-c.qc.HandshakeComplete(): + case <-ctx.Done(): + return + } + defer close(c.resumed) + c.writeMu.Lock() + defer c.writeMu.Unlock() + sent := c.earlySent + c.early, c.earlySent = false, nil + if c.qc.ConnectionState().Used0RTT { + return + } + next, err := c.qc.NextConnection(ctx) + if err != nil { + return + } + str, err := next.OpenStreamSync(ctx) + if err != nil { + c.closeWith(cloudwire.CloseProtocol, "") + return + } + for _, m := range sent { + if _, err := str.Write(cloudwire.AppendMessage(nil, m)); err != nil { + c.closeWith(cloudwire.CloseProtocol, "") + return + } + } + c.qc, c.ctrl, c.r = next, str, bufio.NewReader(str) +} + +// UsedEarlyData reports whether the handshake took the client's early +// data. Meaningful once the handshake is done. +func (c *Conn) UsedEarlyData() bool { return c.qc.ConnectionState().Used0RTT } + +// QUIC is the connection underneath, for tests that move it to a new path. +func (c *Conn) QUIC() *quic.Conn { return c.qc } diff --git a/internal/quicdoor/early.go b/internal/quicdoor/early.go new file mode 100644 index 0000000..7f82652 --- /dev/null +++ b/internal/quicdoor/early.go @@ -0,0 +1,123 @@ +package quicdoor + +import ( + "bytes" + "crypto/tls" + "encoding/binary" + "sync" + "time" + + "github.com/srcfl/ftw/go/protocol/cloudwire" +) + +// Early data (srcfl/ftw docs/protocol-cloud.md, "Early data"). The network can +// replay a client's first flight, and quic-go accepts what it carries. Two +// rules make a replay change nothing: +// +// - A session ticket allows early data for cloudwire.EarlyDataFor after it +// was issued, and resumption without early data after that. +// - A phone's Noise message 1 in early data is answered once for each +// initiator ephemeral key. A second copy waits for a handshake its sender +// cannot finish, and the connection times out with nothing done. +// +// A box sends only its target in early data, and HTTP/3 requests wait for +// the handshake, so neither needs more. + +// earlyTag starts the ticket's entry that says when it was issued. +var earlyTag = []byte("ftw.cloud.early.v1\x00") + +// limitEarlyData makes c's tickets carry their issue time and allow early +// data only while it is under cloudwire.EarlyDataFor. +func limitEarlyData(c *tls.Config, now func() time.Time) { + c.WrapSession = func(cs tls.ConnectionState, ss *tls.SessionState) ([]byte, error) { + stamp := binary.BigEndian.AppendUint64(append([]byte{}, earlyTag...), uint64(now().Unix())) + ss.Extra = append(ss.Extra, stamp) + return c.EncryptTicket(cs, ss) + } + c.UnwrapSession = func(identity []byte, cs tls.ConnectionState) (*tls.SessionState, error) { + ss, err := c.DecryptTicket(identity, cs) + if err != nil || ss == nil { + return ss, err + } + if ss.EarlyData && !earlyStillAllowed(ss.Extra, now()) { + ss.EarlyData = false + } + return ss, nil + } +} + +// earlyStillAllowed reads a ticket's issue time from extra. +func earlyStillAllowed(extra [][]byte, now time.Time) bool { + for _, e := range extra { + if len(e) == len(earlyTag)+8 && bytes.HasPrefix(e, earlyTag) { + issued := time.Unix(int64(binary.BigEndian.Uint64(e[len(earlyTag):])), 0) + return now.Sub(issued) < cloudwire.EarlyDataFor && !issued.After(now.Add(time.Minute)) + } + } + return false +} + +// ephemeralBytes is the initiator's ephemeral key at the front of Noise IK +// message 1, sent in the clear. +const ephemeralBytes = 32 + +// maxEarlySeen bounds the guard's memory: about 48 MB at most. Past it, early +// message 1s wait for the handshake, which costs a round trip and nothing +// else. +const maxEarlySeen = 1 << 20 + +// earlyGuard remembers each message 1 it let through in early data, for as +// long as a ticket allows early data. +type earlyGuard struct { + mu sync.Mutex + seen map[[ephemeralBytes]byte]time.Time + pruned time.Time + now func() time.Time +} + +func newEarlyGuard(now func() time.Time) *earlyGuard { + return &earlyGuard{seen: map[[ephemeralBytes]byte]time.Time{}, now: now} +} + +// first reports whether msg1 is the first message 1 with its ephemeral key, +// and remembers it. False also for a message too short to be message 1, and +// while the guard is full. +func (g *earlyGuard) first(msg1 []byte) bool { + if len(msg1) < ephemeralBytes { + return false + } + var e [ephemeralBytes]byte + copy(e[:], msg1) + g.mu.Lock() + defer g.mu.Unlock() + now := g.now() + if now.Sub(g.pruned) >= time.Minute { + for k, at := range g.seen { + if now.Sub(at) >= cloudwire.EarlyDataFor { + delete(g.seen, k) + } + } + g.pruned = now + } + if _, ok := g.seen[e]; ok || len(g.seen) >= maxEarlySeen { + return false + } + g.seen[e] = now + return true +} + +// remember records msg1 without asking, for a message 1 that came once the +// handshake was done on a connection that used early data: a copy of its +// first flight could still be replayed. +func (g *earlyGuard) remember(msg1 []byte) { + if len(msg1) < ephemeralBytes { + return + } + var e [ephemeralBytes]byte + copy(e[:], msg1) + g.mu.Lock() + defer g.mu.Unlock() + if len(g.seen) < maxEarlySeen { + g.seen[e] = g.now() + } +} diff --git a/internal/quicdoor/early_test.go b/internal/quicdoor/early_test.go new file mode 100644 index 0000000..5d4a663 --- /dev/null +++ b/internal/quicdoor/early_test.go @@ -0,0 +1,68 @@ +package quicdoor + +import ( + "bytes" + "encoding/binary" + "testing" + "time" + + "github.com/srcfl/ftw/go/protocol/cloudwire" +) + +func stamp(at time.Time) []byte { + return binary.BigEndian.AppendUint64(append([]byte{}, earlyTag...), uint64(at.Unix())) +} + +// A ticket allows early data for a day after it was issued, and never one +// without the door's stamp or with a stamp from the future. +func TestATicketAllowsEarlyDataForADay(t *testing.T) { + now := time.Unix(1_790_000_000, 0) + for _, c := range []struct { + name string + extra [][]byte + ok bool + }{ + {"fresh", [][]byte{[]byte("quic-go's own"), stamp(now.Add(-time.Hour))}, true}, + {"a day old", [][]byte{stamp(now.Add(-cloudwire.EarlyDataFor))}, false}, + {"no stamp", [][]byte{[]byte("quic-go's own")}, false}, + {"from the future", [][]byte{stamp(now.Add(time.Hour))}, false}, + {"cut short", [][]byte{stamp(now)[:len(earlyTag)+4]}, false}, + } { + if got := earlyStillAllowed(c.extra, now); got != c.ok { + t.Errorf("%s: %v", c.name, got) + } + } +} + +// Each message 1 passes once by its ephemeral key, for as long as a ticket +// allows early data. +func TestTheGuardLetsEachMessage1ThroughOnce(t *testing.T) { + now := time.Unix(1_790_000_000, 0) + g := newEarlyGuard(func() time.Time { return now }) + m1 := append(bytes.Repeat([]byte{1}, ephemeralBytes), []byte("the rest of message 1")...) + if !g.first(m1) { + t.Fatal("the first copy was refused") + } + again := append(bytes.Repeat([]byte{1}, ephemeralBytes), []byte("other bytes, same key")...) + if g.first(again) { + t.Fatal("a second copy with the same ephemeral key passed") + } + if g.first(m1[:ephemeralBytes-1]) { + t.Fatal("a message too short to be message 1 passed") + } + other := bytes.Repeat([]byte{2}, ephemeralBytes) + g.remember(other) + if g.first(other) { + t.Fatal("a remembered message 1 passed in early data") + } + now = now.Add(cloudwire.EarlyDataFor + time.Minute) + if !g.first(m1) { + t.Fatal("past a ticket's early-data life the key was still held") + } +} + +func TestEarlyDataIsOnUnderTheGuard(t *testing.T) { + if !Config().Allow0RTT { + t.Fatal("0-RTT is off") + } +} diff --git a/internal/quicdoor/quicdoor.go b/internal/quicdoor/quicdoor.go index 8b35817..ab89d51 100644 --- a/internal/quicdoor/quicdoor.go +++ b/internal/quicdoor/quicdoor.go @@ -1,15 +1,23 @@ // Package quicdoor is FTW Cloud's door over QUIC: UDP on the same port as // the TCP listener, with the same certificate. Each connection goes to the -// protocol its TLS handshake negotiated by ALPN. Today that is HTTP/3, for -// the routes that are plain HTTPS: /v1/app/once, /v1/now, /healthz and -// /source. The sockets, /v1/box and /v1/app, stay on TCP. +// protocol its TLS handshake negotiated by ALPN: // -// TLS still ends in ftwcloudd, so r.TLS over HTTP/3 is this process's own -// TLS 1.3 session, and its exporter is the one a client on this connection -// computes. +// h3 HTTP/3, for the plain HTTPS routes: /v1/app/once, +// /v1/now, /healthz and /source +// ftw-cloud-box/1 a box's uplink, served by the same handler as /v1/box +// ftw-cloud-app/1 a phone's app session, as /v1/app +// +// The uplink and the app session keep their messages; Conn carries them on +// a control stream, a bulk stream and QUIC DATAGRAMs (docs/protocol-cloud.md +// in srcfl/ftw, "QUIC"). +// +// TLS still ends in ftwcloudd, so r.TLS over HTTP/3 and Conn.Exporter are +// this process's own TLS 1.3 session, and its exporter is the one a client +// on the same connection computes. package quicdoor import ( + "bufio" "context" "crypto/rand" "crypto/tls" @@ -18,13 +26,16 @@ import ( "log/slog" "net" "net/http" + "net/url" "os" "strconv" "sync" + "sync/atomic" "time" "github.com/quic-go/quic-go" "github.com/quic-go/quic-go/http3" + "github.com/srcfl/ftw/go/protocol/cloudwire" ) // ALPNH3 is HTTP/3. @@ -35,14 +46,21 @@ const ALPNH3 = http3.NextProtoH3 const AltSvcMaxAge = 24 * time.Hour const ( - // idleTimeout closes a quiet connection. A watch posts once per glance - // and a widget reads once per refresh, so nothing here stays open long. - idleTimeout = 30 * time.Second - // handshakeTimeout bounds the TLS handshake. A client that takes longer - // is better served by the WebSocket it races against. + // h3Idle closes a quiet HTTP/3 connection. A watch posts once per + // glance and a widget reads once per refresh, so nothing there stays + // open long. + h3Idle = 30 * time.Second + // handshakeTimeout bounds the TLS handshake, and the target after it. A + // client that takes longer is better served by the WebSocket it races + // against. handshakeTimeout = 5 * time.Second - // maxStreams bounds concurrent requests on one connection. + // maxStreams bounds concurrent requests on one HTTP/3 connection. A box + // or a phone may open one control stream and one bulk stream; Conn + // closes on more. maxStreams = 32 + // maxMessage1 bounds the phone's Noise message 1, which the door reads + // itself; the app session checks it again, more tightly. + maxMessage1 = 64 << 10 ) // retryRate is how many new connections a second the door takes on trust. @@ -79,29 +97,50 @@ func (n *newcomers) retry(net.Addr) bool { return false } -// Door accepts QUIC connections and serves HTTP/3 on them. +// Door accepts QUIC connections and hands each to its protocol. type Door struct { - h3 *http3.Server - log *slog.Logger + h3 *http3.Server + log *slog.Logger + early *earlyGuard + now func() time.Time + // EarlyRepeats counts message 1s in early data the guard held back: a + // replay, or a client that reused an ephemeral key. + EarlyRepeats atomic.Int64 + // Box serves a box's uplink over QUIC and App a phone's session, with + // the same handlers as their WebSockets. A nil one refuses its ALPN. + // Set before Serve. + Box func(conn *Conn, ekm []byte) + App func(conn *Conn, home string) // ResetKey makes stateless resets (RFC 9000, 10.3). After a restart, a // client of the old process gets a reset for its next packet instead // of silence until its idle timeout. Keep it across restarts; nil sends // none. Set before Serve. ResetKey *quic.StatelessResetKey + + mu sync.Mutex + tr *quic.Transport + live map[*Conn]struct{} + closed bool } -// New serves handler over HTTP/3. A nil log writes nothing. +// New serves handler over HTTP/3; a nil handler refuses h3. A nil log +// writes nothing. func New(handler http.Handler, log *slog.Logger) *Door { if log == nil { log = slog.New(slog.DiscardHandler) } - return &Door{log: log, h3: &http3.Server{ - Handler: onlyRequests(handler), - IdleTimeout: idleTimeout, - // quic-go's own log names the peer's address, and an address beside - // a time is what links a household to its traffic. - Logger: nil, - }} + d := &Door{log: log, now: time.Now} + d.early = newEarlyGuard(func() time.Time { return d.now() }) + if handler != nil { + d.h3 = &http3.Server{ + Handler: onlyRequests(handler), + IdleTimeout: h3Idle, + // quic-go's own log names the peer's address, and an address + // beside a time is what links a household to its traffic. + Logger: nil, + } + } + return d } // ResetKeyFile reads the stateless reset key at path, or writes a new one @@ -150,40 +189,53 @@ func PortOf(addr string) (int, error) { func TLSConfig(base *tls.Config) *tls.Config { c := base.Clone() c.MinVersion = tls.VersionTLS13 - c.NextProtos = []string{ALPNH3} + c.NextProtos = []string{ALPNH3, cloudwire.ALPNBox, cloudwire.ALPNApp} return c } -// Config is the QUIC configuration the door listens with. 0-RTT is off: a -// request in early data can be replayed, and no route here has said which -// of its requests survive that. +// Config is the QUIC configuration the door listens with. A connection +// silent for cloudwire.QUICIdleTimeout ends; boxes and phones send keepalives +// well inside it. Datagrams are on for the readable copy's hb and live. +// 0-RTT is on, under the rules in early.go. func Config() *quic.Config { return &quic.Config{ HandshakeIdleTimeout: handshakeTimeout, - MaxIdleTimeout: idleTimeout, + MaxIdleTimeout: cloudwire.QUICIdleTimeout, MaxIncomingStreams: maxStreams, MaxIncomingUniStreams: 8, - Allow0RTT: false, + EnableDatagrams: true, + Allow0RTT: true, } } -// Serve accepts connections on pc until ctx ends, then closes them. pc is -// the caller's, and Serve closes it on its way out. +// Serve accepts connections on pc until ctx ends. Then it takes no more, +// and the live sessions stay until Close, so a drain can end each with its +// own code. pc is the caller's. func (d *Door) Serve(ctx context.Context, pc net.PacketConn, tlsConf *tls.Config) error { tr := &quic.Transport{Conn: pc, StatelessResetKey: d.ResetKey, - VerifySourceAddress: (&newcomers{now: time.Now}).retry} - defer tr.Close() - ln, err := tr.Listen(TLSConfig(tlsConf), Config()) + VerifySourceAddress: (&newcomers{now: d.now}).retry} + d.mu.Lock() + if d.closed { + d.mu.Unlock() + return nil + } + d.tr = tr + d.mu.Unlock() + tc := TLSConfig(tlsConf) + limitEarlyData(tc, d.now) + ln, err := tr.ListenEarly(tc, Config()) if err != nil { return err } defer ln.Close() go func() { <-ctx.Done() + _ = ln.Close() shutdown, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - _ = d.h3.Shutdown(shutdown) - _ = ln.Close() + if d.h3 != nil { + _ = d.h3.Shutdown(shutdown) + } }() for { conn, err := ln.Accept(ctx) @@ -197,17 +249,168 @@ func (d *Door) Serve(ctx context.Context, pc net.PacketConn, tlsConf *tls.Config } } +// Close ends every live session with 1001, going away, so its client dials +// again at once, and then the transport. Call it on the way out, after +// Serve's context is done and after any drain. +func (d *Door) Close() error { + d.mu.Lock() + d.closed = true + tr, live := d.tr, d.live + d.live = nil + d.mu.Unlock() + var wg sync.WaitGroup + for c := range live { + wg.Go(func() { _ = c.Close() }) + } + wg.Wait() + if tr == nil { + return nil + } + return tr.Close() +} + +// track counts c as live until untrack, or reports false once Close has +// begun. +func (d *Door) track(c *Conn) bool { + d.mu.Lock() + defer d.mu.Unlock() + if d.closed { + return false + } + if d.live == nil { + d.live = map[*Conn]struct{}{} + } + d.live[c] = struct{}{} + return true +} + +func (d *Door) untrack(c *Conn) { + d.mu.Lock() + delete(d.live, c) + d.mu.Unlock() +} + func (d *Door) serve(conn *quic.Conn) { - switch conn.ConnectionState().TLS.NegotiatedProtocol { - case ALPNH3: + switch alpn := conn.ConnectionState().TLS.NegotiatedProtocol; { + case alpn == ALPNH3 && d.h3 != nil: + // No route has said which of its requests survive a replay, so a + // request in early data waits for the handshake. + if !d.handshake(conn) { + return + } if err := d.h3.ServeQUICConn(conn); err != nil && !errors.Is(err, http.ErrServerClosed) { d.log.Debug("HTTP/3 connection ended", "err", err) } + case alpn == cloudwire.ALPNBox && d.Box != nil, alpn == cloudwire.ALPNApp && d.App != nil: + d.session(conn, alpn == cloudwire.ALPNBox) default: _ = conn.CloseWithError(quic.ApplicationErrorCode(http3.ErrCodeVersionFallback), "") } } +// session reads the control stream's target and hands the connection to the +// box's or the phone's handler. +func (d *Door) session(qc *quic.Conn, box bool) { + refuse := func(code int) { _ = qc.CloseWithError(quic.ApplicationErrorCode(code), "") } + ctx, cancel := context.WithTimeout(qc.Context(), handshakeTimeout) + defer cancel() + ctrl, err := qc.AcceptStream(ctx) + if err != nil { + refuse(cloudwire.CloseProtocol) + return + } + _ = ctrl.SetReadDeadline(time.Now().Add(handshakeTimeout)) + r := bufio.NewReader(ctrl) + raw, err := cloudwire.ReadMessage(r, cloudwire.MaxTarget) + _ = ctrl.SetReadDeadline(time.Time{}) + if err != nil { + refuse(cloudwire.CloseProtocol) + return + } + target, err := url.Parse(string(raw)) + if err != nil { + refuse(cloudwire.CloseProtocol) + return + } + home := target.Query().Get(cloudwire.AppHomeParam) + if !box && (target.Path != cloudwire.AppPath || !cloudwire.ValidHome(home)) { + refuse(cloudwire.CloseProtocol) + return + } + c := newConn(qc, ctrl, r, box) + if !box { + // Message 1, read here so that a copy replayed in early data stops at + // the door: the second one with an ephemeral key waits for a + // handshake its sender cannot finish. + _ = ctrl.SetReadDeadline(time.Now().Add(handshakeTimeout)) + msg1, err := cloudwire.ReadMessage(r, maxMessage1) + _ = ctrl.SetReadDeadline(time.Time{}) + if err != nil { + refuse(cloudwire.CloseProtocol) + return + } + switch { + case !done(qc): + if !d.early.first(msg1) { + d.EarlyRepeats.Add(1) + if !d.handshake(qc) { + return + } + } + case qc.ConnectionState().Used0RTT: + d.early.remember(msg1) + } + c.pending = [][]byte{msg1} + } + var ekm []byte + if box { + if target.Path != cloudwire.TargetBox { + refuse(cloudwire.CloseProtocol) + return + } + if ekm, err = c.Exporter(); err != nil { + refuse(cloudwire.CloseProtocol) + return + } + } + if !d.track(c) { + _ = c.Close() + return + } + defer d.untrack(c) + if box { + d.Box(c, ekm) + } else { + d.App(c, home) + } + _ = c.Close() +} + +// done reports whether the connection's handshake has completed. +func done(qc *quic.Conn) bool { + select { + case <-qc.HandshakeComplete(): + return true + default: + return false + } +} + +// handshake waits for the handshake, and closes a connection whose +// handshake does not finish in time. +func (d *Door) handshake(qc *quic.Conn) bool { + t := time.NewTimer(handshakeTimeout) + defer t.Stop() + select { + case <-qc.HandshakeComplete(): + return true + case <-qc.Context().Done(): + case <-t.C: + _ = qc.CloseWithError(quic.ApplicationErrorCode(cloudwire.CloseProtocol), "") + } + return false +} + // onlyRequests refuses extended CONNECT. quic-go's HTTP/3 server always // offers it, but nothing here speaks WebSocket over HTTP/3: /v1/box and // /v1/app are WebSockets on TCP. diff --git a/internal/quicdoor/quicdoor_test.go b/internal/quicdoor/quicdoor_test.go index c94efb0..45113a3 100644 --- a/internal/quicdoor/quicdoor_test.go +++ b/internal/quicdoor/quicdoor_test.go @@ -20,6 +20,7 @@ import ( "testing" "time" + "github.com/gorilla/websocket" "github.com/quic-go/quic-go" "github.com/quic-go/quic-go/http3" "github.com/srcfl/ftw/go/protocol/cloudwire" @@ -172,14 +173,6 @@ func TestAltSvcNamesThePort(t *testing.T) { } } -// 0-RTT stays off until a route says which of its requests survive a -// replay. -func TestNoEarlyData(t *testing.T) { - if Config().Allow0RTT { - t.Fatal("0-RTT is on") - } -} - // A deploy or a crash drops the process without a word to its clients. With // the same reset key, the next process on the port answers an old // connection's next packet with a stateless reset, so the client knows at @@ -297,3 +290,66 @@ func TestAClientAskedForARetryStillGetsIn(t *testing.T) { t.Fatalf("answer %q", b) } } + +// SIGTERM stops the door taking connections and leaves the live sessions to +// a drain. Close then ends each with 1001, so its client dials again at +// once, rather than leaving it to its idle timeout. +func TestCloseEndsEachSessionWithGoingAway(t *testing.T) { + cert, pool := testCert(t) + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer pc.Close() + addr := pc.LocalAddr().String() + d := New(nil, nil) + in := make(chan struct{}, 1) + d.Box = func(c *Conn, _ []byte) { + in <- struct{}{} + for { + if _, _, err := c.ReadMessage(); err != nil { + return + } + } + } + ctx, cancel := context.WithCancel(context.Background()) + served := make(chan error, 1) + go func() { served <- d.Serve(ctx, pc, &tls.Config{Certificates: []tls.Certificate{cert}}) }() + + dial := func(within time.Duration) (*Conn, error) { + ctx, cancel := context.WithTimeout(context.Background(), within) + defer cancel() + return Dial(ctx, addr, &tls.Config{RootCAs: pool}, cloudwire.ALPNBox, cloudwire.TargetBox, nil) + } + c, err := dial(5 * time.Second) + if err != nil { + t.Fatal(err) + } + select { + case <-in: + case <-time.After(5 * time.Second): + t.Fatal("the box's handler never ran") + } + + cancel() + if err := <-served; err != nil { + t.Fatal(err) + } + if late, err := dial(500 * time.Millisecond); err == nil { + late.Close() + t.Fatal("the door took a connection after its context ended") + } + if err := c.WriteMessage(websocket.BinaryMessage, []byte(`{}`)); err != nil { + t.Fatalf("the live session ended with the context: %v", err) + } + + if err := d.Close(); err != nil { + t.Fatal(err) + } + _ = c.SetReadDeadline(time.Now().Add(3 * time.Second)) + _, _, err = c.ReadMessage() + var ce *websocket.CloseError + if !errors.As(err, &ce) || ce.Code != websocket.CloseGoingAway { + t.Fatalf("the session ended with %v, not 1001", err) + } +} diff --git a/internal/server/e2e_test.go b/internal/server/e2e_test.go index 377a83a..07dabb0 100644 --- a/internal/server/e2e_test.go +++ b/internal/server/e2e_test.go @@ -22,10 +22,12 @@ import ( "github.com/srcfl/ftw-cloud/internal/apns" "github.com/srcfl/ftw-cloud/internal/appsession" "github.com/srcfl/ftw-cloud/internal/push" + "github.com/srcfl/ftw-cloud/internal/quicdoor" "github.com/srcfl/ftw-cloud/internal/registry" "github.com/srcfl/ftw-cloud/internal/store" "github.com/srcfl/ftw-cloud/internal/token" "github.com/srcfl/ftw-cloud/internal/uplink" + "github.com/srcfl/ftw-cloud/internal/wire" "github.com/srcfl/ftw/go/protocol" "github.com/srcfl/ftw/go/protocol/appwire" "github.com/srcfl/ftw/go/protocol/cloudwire" @@ -65,6 +67,9 @@ type stack struct { up *uplink.Uplink reg *registry.Registry app *appsession.Server + // quicAddr is the stack's QUIC door, once s.quic has started it. + quicAddr string + door *quicdoor.Door } func newStack(t *testing.T) *stack { @@ -117,7 +122,7 @@ type testBox struct { seq uint64 // liveSeq is the seq of the newest live sent, which hb names. liveSeq uint64 - conn *websocket.Conn + conn wire.Conn // heard is what the service told the box after hello_ok. heard chan string start time.Time diff --git a/internal/server/early_e2e_test.go b/internal/server/early_e2e_test.go new file mode 100644 index 0000000..0f986b3 --- /dev/null +++ b/internal/server/early_e2e_test.go @@ -0,0 +1,213 @@ +package server + +import ( + "context" + "crypto/tls" + "net" + "slices" + "sync" + "testing" + "time" + + "github.com/srcfl/ftw-cloud/internal/quicdoor" + "github.com/srcfl/ftw/go/protocol" + "github.com/srcfl/ftw/go/protocol/appwire" + "github.com/srcfl/ftw/go/protocol/cloudwire" +) + +// flightRecorder is a UDP proxy in front of the door that keeps, for each +// client, its first flight as a network could: every Initial and 0-RTT +// packet it sent, the ClientHello and the early data. It holds the door's +// answers back 30 ms, as a real path does, so a client writes its first +// messages before the handshake can finish. +type flightRecorder struct { + t *testing.T + pc net.PacketConn + door *net.UDPAddr + + mu sync.Mutex + flights map[string]*flight + order []string +} + +type flight struct { + up *net.UDPConn + packets [][]byte +} + +func newFlightRecorder(t *testing.T, door string) *flightRecorder { + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = pc.Close() }) + addr, _ := net.ResolveUDPAddr("udp", door) + r := &flightRecorder{t: t, pc: pc, door: addr, flights: map[string]*flight{}} + go r.run() + return r +} + +func (r *flightRecorder) addr() string { return r.pc.LocalAddr().String() } + +func (r *flightRecorder) run() { + buf := make([]byte, 2048) + for { + n, from, err := r.pc.ReadFrom(buf) + if err != nil { + return + } + pkt := append([]byte(nil), buf[:n]...) + r.mu.Lock() + f := r.flights[from.String()] + if f == nil { + up, err := net.DialUDP("udp", nil, r.door) + if err != nil { + r.mu.Unlock() + continue + } + f = &flight{up: up} + r.flights[from.String()] = f + r.order = append(r.order, from.String()) + go func(client net.Addr) { + b := make([]byte, 2048) + for { + n, err := up.Read(b) + if err != nil { + return + } + pkt := append([]byte(nil), b[:n]...) + time.AfterFunc(30*time.Millisecond, func() { _, _ = r.pc.WriteTo(pkt, client) }) + } + }(from) + } + // A long header of type Initial (0) or 0-RTT (1). + if pkt[0]&0x80 != 0 && (pkt[0]>>4)&3 <= 1 { + f.packets = append(f.packets, pkt) + } + r.mu.Unlock() + _, _ = f.up.Write(pkt) + } +} + +// first is the first flight of the n-th client. +func (r *flightRecorder) first(n int) [][]byte { + r.mu.Lock() + defer r.mu.Unlock() + return slices.Clone(r.flights[r.order[n]].packets) +} + +// A phone's second connection carries its Noise message 1, and the box +// session's message 1 inside it, in early data: one round trip sooner. A +// network that replays that flight later, from another address, changes +// nothing: the door holds the copy back, the box hears no handshake and no +// watch, and the phone's live session is not replaced. +func TestAReplayedEarlyFlightChangesNothing(t *testing.T) { + s := newStack(t) + box := s.connectTunBoxQUIC() + key, _ := appwire.GenerateKeyPair() + box.roster(2, grantOff, nil, key.Public()) + rec := newFlightRecorder(t, s.quic()) + cache := tls.NewLRUClientSessionCache(4) + dial := func(cache tls.ClientSessionCache) *quicdoor.Conn { + t.Helper() + tc := s.clientTLS() + tc.ClientSessionCache = cache + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + c, err := quicdoor.Dial(ctx, rec.addr(), tc, cloudwire.ALPNApp, cloudwire.AppTarget(box.home), nil) + if err != nil { + t.Fatal(err) + } + return c + } + now := cloudwire.Now{Hz: cloudwire.HzLooking} + + // The first connection holds no ticket; it leaves with one. + first := newTunPhone(t, key) + first.session(dial(cache), s.noise.Public(), box.home, now) + if first.tr == nil { + t.Fatalf("the first session did not open: %d", first.code) + } + first.open(box.static.Public(), 21, nil) + time.Sleep(100 * time.Millisecond) + _ = first.conn.Close() + + // The second resumes and sends message 1 in early data. + second := newTunPhone(t, key) + conn := dial(cache) + second.session(conn, s.noise.Public(), box.home, cloudwire.Now{Hz: cloudwire.HzLooking, + Tun: &cloudwire.AppTun{Sid: 22, D: second.boxMessage1(box.static.Public(), nil)}}) + if second.tr == nil { + t.Fatalf("the early session did not open: %d", second.code) + } + if !conn.UsedEarlyData() { + t.Fatal("the second connection did not use early data") + } + second.boxMessage2(second.next().D) + if ack, _ := second.cmd(22, "early"); ack != protocol.MsgCmdAck { + t.Fatal("no ack on the early session") + } + flightOfSecond := rec.first(1) + _ = conn.Close() + + // The phone's live session now, over the WebSocket, which the replay + // must not replace. + live := s.dialPhone(box.home, key, cloudwire.Now{}, "") + if live.tr == nil { + t.Fatal("the live session did not open") + } + live.open(box.static.Public(), 23, nil) + // Wait out quic-go's memory of the closed connection, so the replay + // reaches the door as a connection of its own. + time.Sleep(1500 * time.Millisecond) + + handshakes := box.handshakes() + box.mu.Lock() + heard := len(box.heard) + box.mu.Unlock() + repeats := s.door.EarlyRepeats.Load() + + replay, err := net.DialUDP("udp", nil, mustUDP(t, s.quic())) + if err != nil { + t.Fatal(err) + } + defer replay.Close() + for _, p := range flightOfSecond { + if _, err := replay.Write(p); err != nil { + t.Fatal(err) + } + } + // The replayer cannot finish the handshake; give the door every chance + // to act on what it got. + time.Sleep(1500 * time.Millisecond) + + if got := s.door.EarlyRepeats.Load(); got != repeats+1 { + t.Fatalf("the door held back %d copies of message 1, want 1: the replay never reached it", got-repeats) + } + if got := box.handshakes(); got != handshakes { + t.Fatalf("the replay reached the box: %d handshakes", got-handshakes) + } + box.mu.Lock() + since := slices.Clone(box.heard[heard:]) + box.mu.Unlock() + for _, typ := range since { + if typ == cloudwire.MsgWatch || typ == cloudwire.MsgTun { + t.Fatalf("the box heard %q after the replay", typ) + } + } + if ack, res := live.cmd(23, "after the replay"); ack != protocol.MsgCmdAck || res != protocol.CmdApplied { + t.Fatalf("the live session after the replay: %q %q", ack, res) + } + if live.ended != nil { + t.Fatalf("the live session ended: %v", live.ended) + } +} + +func mustUDP(t *testing.T, addr string) *net.UDPAddr { + t.Helper() + a, err := net.ResolveUDPAddr("udp", addr) + if err != nil { + t.Fatal(err) + } + return a +} diff --git a/internal/server/quic_e2e_test.go b/internal/server/quic_e2e_test.go new file mode 100644 index 0000000..4470a8b --- /dev/null +++ b/internal/server/quic_e2e_test.go @@ -0,0 +1,486 @@ +package server + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "errors" + "net" + "slices" + "testing" + "time" + + "github.com/gorilla/websocket" + "github.com/quic-go/quic-go" + "github.com/srcfl/ftw-cloud/internal/quicdoor" + "github.com/srcfl/ftw-cloud/internal/store" + "github.com/srcfl/ftw-cloud/internal/wire" + "github.com/srcfl/ftw/go/protocol" + "github.com/srcfl/ftw/go/protocol/appwire" + "github.com/srcfl/ftw/go/protocol/cloudwire" +) + +// The uplink and the app session over QUIC, through the whole service as +// ftwcloudd wires it: the same handlers as the WebSockets, a control stream, +// a bulk stream for the tunnel's lane 1, and datagrams for hb and live. + +// quic starts the stack's QUIC door, with the uplink and the app session +// served by the stack's own handlers, and returns its address. +func (s *stack) quic() string { + s.t.Helper() + if s.quicAddr != "" { + return s.quicAddr + } + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + s.t.Fatal(err) + } + door := quicdoor.New(s.srv.Config.Handler, nil) + s.door = door + door.Box = func(c *quicdoor.Conn, ekm []byte) { s.up.ServeConn(c, ekm) } + door.App = func(c *quicdoor.Conn, home string) { s.app.ServeConn(c, home) } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + _ = door.Serve(ctx, pc, &tls.Config{Certificates: s.srv.TLS.Certificates}) + }() + s.t.Cleanup(func() { + cancel() + <-done + }) + s.quicAddr = pc.LocalAddr().String() + return s.quicAddr +} + +func (s *stack) clientTLS() *tls.Config { + pool := x509.NewCertPool() + pool.AddCert(s.srv.Certificate()) + return &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS13} +} + +// dialQUIC opens a client connection to the stack's door. +func (s *stack) dialQUIC(alpn, target string) (*quicdoor.Conn, error) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + return quicdoor.Dial(ctx, s.quic(), s.clientTLS(), alpn, target, nil) +} + +// connectTunBoxQUIC is connectTunBox over QUIC. +func (s *stack) connectTunBoxQUIC() *tunBox { + s.t.Helper() + key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + static, _ := appwire.GenerateKeyPair() + pk, _ := cloudwire.PublicKeyBytesOf(&key.PublicKey) + b := &tunBox{t: s.t, key: key, home: cloudwire.HomeID(pk), static: static, allowed: map[string][]byte{}, + sessions: map[string]*boxSide{}, purged: make(chan cloudwire.Purged, 4), closed: make(chan struct{})} + conn, err := s.dialQUIC(cloudwire.ALPNBox, cloudwire.TargetBox) + if err != nil { + s.t.Fatal(err) + } + ekm, err := conn.Exporter() + if err != nil { + s.t.Fatal(err) + } + b.link(conn, ekm) + return b +} + +// dialPhoneQUIC is dialPhone over QUIC. +func (s *stack) dialPhoneQUIC(home string, key appwire.KeyPair, now cloudwire.Now) *tunPhone { + s.t.Helper() + p := newTunPhone(s.t, key) + conn, err := s.dialQUIC(cloudwire.ALPNApp, cloudwire.AppTarget(home)) + if err != nil { + s.t.Fatal(err) + } + p.session(conn, s.noise.Public(), home, now) + return p +} + +// say sends the service one message of the app session. +func (p *tunPhone) say(t string, body any) { + p.t.Helper() + env, _ := cloudwire.Unsigned(t, body) + frame, _ := cloudwire.EncodeFrame(env) + msg, _ := p.tr.Encrypt(frame) + if err := p.conn.WriteMessage(websocket.BinaryMessage, msg); err != nil { + p.t.Fatal(err) + } +} + +func (p *tunPhone) kindsOf(t string) []int { + p.kmu.Lock() + defer p.kmu.Unlock() + return slices.Clone(p.kinds[t]) +} + +// A command crosses the service over QUIC on both legs, and over QUIC on one +// leg and a WebSocket on the other: the handlers are the same. +func TestACommandCrossesQUIC(t *testing.T) { + for _, legs := range []struct { + name string + boxQUIC bool + phoneQUI bool + }{{"both QUIC", true, true}, {"box QUIC, phone WebSocket", true, false}, {"box WebSocket, phone QUIC", false, true}} { + t.Run(legs.name, func(t *testing.T) { + s := newStack(t) + var box *tunBox + if legs.boxQUIC { + box = s.connectTunBoxQUIC() + } else { + box = s.connectTunBox() + } + key, _ := appwire.GenerateKeyPair() + box.roster(2, grantOff, nil, key.Public()) + eventually(t, "the roster held", func() bool { + _, err := s.st.For(storePrincipal(box.home, key)) + return err == nil + }) + var phone *tunPhone + if legs.phoneQUI { + phone = s.dialPhoneQUIC(box.home, key, cloudwire.Now{Hz: cloudwire.HzLooking}) + } else { + phone = s.dialPhone(box.home, key, cloudwire.Now{Hz: cloudwire.HzLooking}, "") + } + if phone.tr == nil { + t.Fatalf("the app session did not open: %d", phone.code) + } + phone.open(box.static.Public(), 7, nil) + if ack, res := phone.cmd(7, "c1"); ack != protocol.MsgCmdAck || res != protocol.CmdApplied { + t.Fatalf("over %s: ack %q, result %q", legs.name, ack, res) + } + }) + } +} + +// The box's bulk stream reaches the phone's: a lane 1 frame of the phone's +// session never waits behind lane 0 or holds it up. Lane 0 stays on the +// control stream. +func TestBulkTravelsOnTheBulkStream(t *testing.T) { + s := newStack(t) + box := s.connectTunBoxQUIC() + key, _ := appwire.GenerateKeyPair() + box.roster(2, grantOff, nil, key.Public()) + phone := s.dialPhoneQUIC(box.home, key, cloudwire.Now{}) + if phone.tr == nil { + t.Fatalf("the app session did not open: %d", phone.code) + } + phone.open(box.static.Public(), 9, nil) + dev := cloudwire.DeviceID(key.Public()) + + bulk := bytes.Repeat([]byte{0xb7}, 4096+appwire.TransportOverhead) + body, _ := cloudwire.Marshal(cloudwire.Tun{Dev: dev, Sid: 9, D: bulk}) + box.writeAs(wire.Bulk, cloudwire.Envelope{T: cloudwire.MsgTun, B: body}) + got := phone.next() + if !bytes.Equal(got.D, bulk) { + t.Fatalf("bulk arrived as %d bytes", len(got.D)) + } + kinds := phone.kindsOf(cloudwire.MsgTun) + // The box's answer to message 1 came on the control stream, the bulk + // frame on the bulk stream. + if !slices.Equal(kinds, []int{websocket.BinaryMessage, wire.Bulk}) { + t.Fatalf("tun kinds at the phone: %v", kinds) + } + + // Anything but a tun on the bulk stream breaks the protocol. + box.writeAs(wire.Bulk, mustSeal(t, box, cloudwire.MsgHB, cloudwire.HB{Header: box.header(), Clock: cloudwire.ClockNTP})) + waitClosed(t, box) +} + +// With the readable copy on, hb and a live without sources reach the phone +// as datagrams; a live that carries its source table comes on the stream. +func TestHbAndLiveTravelAsDatagrams(t *testing.T) { + s := newStack(t) + box := s.connectTunBoxQUIC() + key, _ := appwire.GenerateKeyPair() + box.roster(2, grantServe, nil, key.Public()) + phone := s.dialPhoneQUIC(box.home, key, cloudwire.Now{Hz: cloudwire.HzLooking}) + if phone.tr == nil { + t.Fatalf("the app session did not open: %d", phone.code) + } + + full := cloudwire.Live{Header: box.header(), Fields: map[string]int64{"2": 1800}, SrcRev: 1, Ages: map[string]int64{"m": 300}, + Sources: map[string]cloudwire.Source{"m": {Kind: "driver", State: protocol.SourceLive, StaleAfterMs: 10_000}}, + FieldSources: map[string][]string{"2": {"m"}}} + box.writeAs(wire.Datagram, mustSeal(t, box, cloudwire.MsgLive, full)) + // A live with its table may not go as a datagram: the uplink closes. + waitClosed(t, box) + + box = s.connectTunBoxQUICAs(box.key, box.static) + box.roster(3, grantServe, nil, key.Public()) + box.send(cloudwire.MsgLive, full) + box.writeAs(wire.Datagram, mustSeal(t, box, cloudwire.MsgLive, + cloudwire.Live{Header: box.header(), Fields: map[string]int64{"2": 1900}, SrcRev: 1, Ages: map[string]int64{"m": 100}})) + box.writeAs(wire.Datagram, mustSeal(t, box, cloudwire.MsgHB, cloudwire.HB{Header: box.header(), Clock: cloudwire.ClockNTP, SrcRev: 1})) + + phone = s.dialPhoneQUIC(box.home, key, cloudwire.Now{Hz: cloudwire.HzLooking}) + if phone.tr == nil { + t.Fatalf("the app session did not open again: %d", phone.code) + } + // Until the phone speaks, it may not hold message 2 yet: everything + // comes on the control stream, behind it. + box.writeAs(wire.Datagram, mustSeal(t, box, cloudwire.MsgHB, cloudwire.HB{Header: box.header(), Clock: cloudwire.ClockNTP, SrcRev: 1})) + eventually(t, "an hb before the phone spoke", func() bool { return len(phone.kindsOf(cloudwire.MsgHB)) >= 1 }) + if hb := phone.kindsOf(cloudwire.MsgHB); hb[0] != websocket.BinaryMessage { + t.Fatalf("before the phone spoke, an hb came as %d", hb[0]) + } + phone.say(cloudwire.MsgWatch, cloudwire.Watch{Hz: cloudwire.HzLooking}) + time.Sleep(50 * time.Millisecond) + box.writeAs(wire.Datagram, mustSeal(t, box, cloudwire.MsgLive, + cloudwire.Live{Header: box.header(), Fields: map[string]int64{"2": 2000}, SrcRev: 1, Ages: map[string]int64{"m": 50}})) + box.writeAs(wire.Datagram, mustSeal(t, box, cloudwire.MsgHB, cloudwire.HB{Header: box.header(), Clock: cloudwire.ClockNTP, SrcRev: 1})) + box.send(cloudwire.MsgLive, full2(box)) + eventually(t, "two lives and an hb at the phone", func() bool { + return len(phone.kindsOf(cloudwire.MsgLive)) >= 2 && len(phone.kindsOf(cloudwire.MsgHB)) >= 1 + }) + lives := phone.kindsOf(cloudwire.MsgLive) + if lives[0] != wire.Datagram || lives[len(lives)-1] != websocket.BinaryMessage { + t.Fatalf("live kinds at the phone: %v", lives) + } + if hb := phone.kindsOf(cloudwire.MsgHB); hb[len(hb)-1] != wire.Datagram { + t.Fatalf("hb kinds at the phone: %v", hb) + } +} + +// A datagram that arrives after a later message of its class is dropped: +// the service never holds an older reading than it has. +func TestALateDatagramIsDropped(t *testing.T) { + s := newStack(t) + box := s.connectTunBoxQUIC() + key, _ := appwire.GenerateKeyPair() + box.roster(2, grantServe, nil, key.Public()) + early := mustSeal(t, box, cloudwire.MsgLive, cloudwire.Live{Header: box.header(), Fields: map[string]int64{"2": 1}, SrcRev: 1, Ages: map[string]int64{}}) + box.send(cloudwire.MsgLive, cloudwire.Live{Header: box.header(), Fields: map[string]int64{"2": 2}, SrcRev: 1, Ages: map[string]int64{}}) + box.writeAs(wire.Datagram, early) + box.send(cloudwire.MsgHB, cloudwire.HB{Header: box.header(), Clock: cloudwire.ClockNTP, SrcRev: 1}) + waitHeldN(t, s, box.home, key, 2) + snap := heldSnapshot(t, s, box.home, key) + for _, raw := range snap.Envelopes { + env, _ := cloudwire.DecodeEnvelope(raw) + if env.T != cloudwire.MsgLive { + continue + } + var l cloudwire.Live + _ = cloudwire.Unmarshal(env.B, &l) + if l.Fields["2"] != 2 { + t.Fatalf("the service holds the late live: %v", l.Fields) + } + } +} + +// The box binds hello to the TLS session inside QUIC. Its exporter is the +// service's, so hello_ok comes; one from another session closes the uplink. +func TestHelloOverQUICIsBoundToItsTLSSession(t *testing.T) { + s := newStack(t) + key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + pk, _ := cloudwire.PublicKeyBytesOf(&key.PublicKey) + home := cloudwire.HomeID(pk) + for _, bound := range []bool{true, false} { + conn, err := s.dialQUIC(cloudwire.ALPNBox, cloudwire.TargetBox) + if err != nil { + t.Fatal(err) + } + ekm, _ := conn.Exporter() + if !bound { + ekm = bytes.Repeat([]byte{1}, cloudwire.EKMBytes) + } + b := &tunBox{t: t, key: key, home: home, conn: conn} + var ch cloudwire.Challenge + if env := b.read(); env.T != cloudwire.MsgChallenge || cloudwire.Unmarshal(env.B, &ch) != nil { + t.Fatalf("first message %q", env.T) + } + b.send(cloudwire.MsgHello, cloudwire.Hello{Header: b.header(), V: cloudwire.Version, PK: pk, Nonce: ch.Nonce, + EKM: ekm, BoxNonce: bytes.Repeat([]byte{4}, cloudwire.NonceBytes), Build: "e2e", Clock: cloudwire.ClockNTP, + Caps: []string{cloudwire.MsgRoster}, RosterSeq: 1}) + _ = conn.SetReadDeadline(time.Now().Add(3 * time.Second)) + _, msg, err := conn.ReadMessage() + if bound { + payload, _ := cloudwire.DecodeFrame(msg) + env, _ := cloudwire.DecodeEnvelope(payload) + var ok cloudwire.HelloOK + if err != nil || env.T != cloudwire.MsgHelloOK || cloudwire.Unmarshal(env.B, &ok) != nil || !bytes.Equal(ok.EKM, ekm) { + t.Fatalf("a bound hello got %q, %v", env.T, err) + } + } else if err == nil { + t.Fatal("a hello bound to another TLS session was answered") + } + _ = conn.Close() + } +} + +// A refusal crosses QUIC as the WebSocket's close code. +func TestARefusalOverQUICCarriesItsCode(t *testing.T) { + s := newStack(t) + box := s.connectTunBoxQUIC() + stranger, _ := appwire.GenerateKeyPair() + known, _ := appwire.GenerateKeyPair() + box.roster(2, grantOff, nil, known.Public()) + phone := s.dialPhoneQUIC(box.home, stranger, cloudwire.Now{}) + if phone.tr != nil || phone.code != cloudwire.CloseDenied { + t.Fatalf("a phone not in the roster: open %v, code %d", phone.tr != nil, phone.code) + } + conn, err := s.dialQUIC(cloudwire.ALPNApp, "/v1/app?home=not-a-home") + if err != nil { + t.Fatal(err) + } + _ = conn.SetReadDeadline(time.Now().Add(3 * time.Second)) + _, _, err = conn.ReadMessage() + var ce *websocket.CloseError + if !errors.As(err, &ce) || ce.Code != cloudwire.CloseProtocol { + t.Fatalf("a bad home: %v", err) + } +} + +// A phone that opens a second stream, or a box that opens a second bulk +// stream, breaks the protocol. +func TestOneStreamOfEachKind(t *testing.T) { + s := newStack(t) + box := s.connectTunBoxQUIC() + key, _ := appwire.GenerateKeyPair() + box.roster(2, grantOff, nil, key.Public()) + phone := s.dialPhoneQUIC(box.home, key, cloudwire.Now{}) + if phone.tr == nil { + t.Fatal("the app session did not open") + } + str, err := phone.conn.(*quicdoor.Conn).QUIC().OpenUniStream() + if err != nil { + t.Fatal(err) + } + _, _ = str.Write(cloudwire.AppendMessage(nil, []byte("x"))) + select { + case <-phone.conn.(*quicdoor.Conn).Context().Done(): + case <-time.After(3 * time.Second): + t.Fatal("a phone's unidirectional stream was let through") + } + + qc := box.conn.(*quicdoor.Conn).QUIC() + for range 2 { + str, err := qc.OpenUniStream() + if err != nil { + t.Fatal(err) + } + _, _ = str.Write([]byte{0}) + } + waitClosed(t, box) +} + +// The phone moves to a new address mid-session, as from Wi-Fi to cellular: +// the QUIC connection follows, and the box session inside goes on with the +// same keys and sid. No handshake, no new tunnel. +func TestThePhoneSessionSurvivesANetworkSwitch(t *testing.T) { + s := newStack(t) + box := s.connectTunBoxQUIC() + key, _ := appwire.GenerateKeyPair() + box.roster(2, grantOff, nil, key.Public()) + phone := s.dialPhoneQUIC(box.home, key, cloudwire.Now{Hz: cloudwire.HzLooking}) + if phone.tr == nil { + t.Fatal("the app session did not open") + } + phone.open(box.static.Public(), 11, nil) + if ack, _ := phone.cmd(11, "before"); ack != protocol.MsgCmdAck { + t.Fatal("no ack before the switch") + } + qc := phone.conn.(*quicdoor.Conn).QUIC() + before := qc.LocalAddr().String() + handshakes := box.handshakes() + + // The new network: another UDP socket, so another source address. + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + tr := &quic.Transport{Conn: pc} + t.Cleanup(func() { _ = tr.Close() }) + path, err := qc.AddPath(tr) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if err := path.Probe(ctx); err != nil { + t.Fatal(err) + } + if err := path.Switch(); err != nil { + t.Fatal(err) + } + if ack, res := phone.cmd(11, "after"); ack != protocol.MsgCmdAck || res != protocol.CmdApplied { + t.Fatalf("after the switch: ack %q, result %q", ack, res) + } + if after := qc.LocalAddr().String(); after == before || after != pc.LocalAddr().String() { + t.Fatalf("the session runs from %s, was %s, the new network is %s", after, before, pc.LocalAddr()) + } + if got := box.handshakes(); got != handshakes { + t.Fatalf("the box ran %d handshakes across the switch", got-handshakes) + } +} + +var grantServe = cloudwire.Grant{ID: bytes.Repeat([]byte{8}, cloudwire.GrantIDBytes), Scopes: protocol.CloudScopes, + Purposes: []string{protocol.CloudPurposeServe}, Controller: cloudwire.ControllerSourceful} + +// connectTunBoxQUICAs is the same box dialling again over QUIC. +func (s *stack) connectTunBoxQUICAs(key *ecdsa.PrivateKey, static appwire.KeyPair) *tunBox { + s.t.Helper() + pk, _ := cloudwire.PublicKeyBytesOf(&key.PublicKey) + b := &tunBox{t: s.t, key: key, home: cloudwire.HomeID(pk), static: static, allowed: map[string][]byte{}, + sessions: map[string]*boxSide{}, purged: make(chan cloudwire.Purged, 4), closed: make(chan struct{})} + conn, err := s.dialQUIC(cloudwire.ALPNBox, cloudwire.TargetBox) + if err != nil { + s.t.Fatal(err) + } + ekm, _ := conn.Exporter() + b.link(conn, ekm) + return b +} + +func storePrincipal(home string, key appwire.KeyPair) store.Principal { + return store.Principal{Kind: store.Device, Home: home, DeviceKey: key.Public()} +} + +func mustSeal(t *testing.T, b *tunBox, typ string, body any) cloudwire.Envelope { + t.Helper() + env, err := cloudwire.Seal(b.key, b.home, typ, body) + if err != nil { + t.Fatal(err) + } + return env +} + +// full2 is another live that carries its source table. +func full2(b *tunBox) cloudwire.Live { + return cloudwire.Live{Header: b.header(), Fields: map[string]int64{"2": 2100}, SrcRev: 1, Ages: map[string]int64{"m": 10}, + Sources: map[string]cloudwire.Source{"m": {Kind: "driver", State: protocol.SourceLive, StaleAfterMs: 10_000}}, + FieldSources: map[string][]string{"2": {"m"}}} +} + +// waitClosed waits for the service to end the box's uplink. +func waitClosed(t *testing.T, b *tunBox) { + t.Helper() + select { + case <-b.closed: + case <-time.After(3 * time.Second): + t.Fatal("the uplink stayed open") + } +} + +func waitHeldN(t *testing.T, s *stack, home string, key appwire.KeyPair, n int) { + t.Helper() + waitHeld(t, s.st, home, key, n) +} + +func heldSnapshot(t *testing.T, s *stack, home string, key appwire.KeyPair) store.Snapshot { + t.Helper() + x, err := s.st.For(storePrincipal(home, key)) + if err != nil { + t.Fatal(err) + } + snap, err := x.Snapshot() + if err != nil { + t.Fatal(err) + } + return snap +} diff --git a/internal/server/tunnel_e2e_test.go b/internal/server/tunnel_e2e_test.go index 1140a3a..fd7e1f8 100644 --- a/internal/server/tunnel_e2e_test.go +++ b/internal/server/tunnel_e2e_test.go @@ -15,6 +15,7 @@ import ( "time" "github.com/gorilla/websocket" + "github.com/srcfl/ftw-cloud/internal/wire" "github.com/srcfl/ftw/go/protocol" "github.com/srcfl/ftw/go/protocol/appwire" "github.com/srcfl/ftw/go/protocol/cloudwire" @@ -38,7 +39,7 @@ type tunBox struct { key *ecdsa.PrivateKey home string static appwire.KeyPair - conn *websocket.Conn + conn wire.Conn wmu sync.Mutex seq uint64 @@ -51,6 +52,14 @@ type tunBox struct { heard []string purged chan cloudwire.Purged closed chan struct{} + // hsCount counts the phones' handshakes the box ran. + hsCount int +} + +func (b *tunBox) handshakes() int { + b.mu.Lock() + defer b.mu.Unlock() + return b.hsCount } type boxSide struct { @@ -79,10 +88,19 @@ func (s *stack) connectTunBoxAs(key *ecdsa.PrivateKey, static appwire.KeyPair) * if err != nil { s.t.Fatal(err) } - s.t.Cleanup(func() { _ = conn.Close() }) - b.conn = conn cs := conn.NetConn().(*tls.Conn).ConnectionState() ekm, _ := cs.ExportKeyingMaterial(cloudwire.EKMLabel, nil, cloudwire.EKMBytes) + b.link(conn, ekm) + return b +} + +// link runs the uplink's handshake on conn, bound to ekm, and starts the +// box serving its phones. +func (b *tunBox) link(conn wire.Conn, ekm []byte) { + b.t.Cleanup(func() { _ = conn.Close() }) + b.conn = conn + pk, _ := cloudwire.PublicKeyBytesOf(&b.key.PublicKey) + s := b var ch cloudwire.Challenge if env := b.read(); env.T != cloudwire.MsgChallenge || cloudwire.Unmarshal(env.B, &ch) != nil { s.t.Fatalf("first message %q", env.T) @@ -94,7 +112,6 @@ func (s *stack) connectTunBoxAs(key *ecdsa.PrivateKey, static appwire.KeyPair) * s.t.Fatalf("second message %q, want hello_ok", env.T) } go b.serve() - return b } func (b *tunBox) header() cloudwire.Header { @@ -117,12 +134,16 @@ func (b *tunBox) read() cloudwire.Envelope { return env } -func (b *tunBox) write(env cloudwire.Envelope) { +func (b *tunBox) write(env cloudwire.Envelope) { b.writeAs(websocket.BinaryMessage, env) } + +// writeAs sends env the way kind says: over QUIC, on the bulk stream or as +// a datagram. +func (b *tunBox) writeAs(kind int, env cloudwire.Envelope) { raw, _ := env.Encode() frame, _ := cloudwire.EncodeFrame(raw) b.wmu.Lock() defer b.wmu.Unlock() - _ = b.conn.WriteMessage(websocket.BinaryMessage, frame) + _ = b.conn.WriteMessage(kind, frame) } func (b *tunBox) send(t string, body any) { @@ -239,6 +260,9 @@ func (b *tunBox) inner(t cloudwire.Tun) { if err != nil { return } + b.mu.Lock() + b.hsCount++ + b.mu.Unlock() code, err := hs.ReadMessage(t.D) if err != nil { b.tun(cloudwire.Tun{Dev: t.Dev, Sid: t.Sid}) @@ -276,12 +300,17 @@ func (b *tunBox) endsSeen() []cloudwire.Tun { type tunPhone struct { t *testing.T key appwire.KeyPair - conn *websocket.Conn + conn wire.Conn tr *appwire.Transport snap cloudwire.Snapshot inner *appwire.Transport in *appwire.InitiatorState tuns chan cloudwire.AppTun + // envs is every envelope that is not a tun, and kinds how each message + // came: over QUIC, by stream, bulk stream or datagram. + envs chan cloudwire.Envelope + kmu sync.Mutex + kinds map[string][]int code int // ended is how the stream ended, set before tuns closes. ended error @@ -291,7 +320,7 @@ type tunPhone struct { // message 2. func (s *stack) dialPhone(home string, key appwire.KeyPair, now cloudwire.Now, origin string) *tunPhone { s.t.Helper() - p := &tunPhone{t: s.t, key: key, tuns: make(chan cloudwire.AppTun, 64)} + p := newTunPhone(s.t, key) pool := x509.NewCertPool() pool.AddCert(s.srv.Certificate()) d := websocket.Dialer{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS13}} @@ -306,9 +335,22 @@ func (s *stack) dialPhone(home string, key appwire.KeyPair, now cloudwire.Now, o } return p } - s.t.Cleanup(func() { _ = conn.Close() }) + p.session(conn, s.noise.Public(), home, now) + return p +} + +func newTunPhone(t *testing.T, key appwire.KeyPair) *tunPhone { + return &tunPhone{t: t, key: key, tuns: make(chan cloudwire.AppTun, 64), envs: make(chan cloudwire.Envelope, 64), + kinds: map[string][]int{}} +} + +// session runs the app session's handshake on conn and starts reading. +func (p *tunPhone) session(conn wire.Conn, service []byte, home string, now cloudwire.Now) { + s := p + key := p.key + p.t.Cleanup(func() { _ = conn.Close() }) p.conn = conn - init, err := appwire.NewInitiator(appwire.InitiatorOptions{StaticKey: key, RemoteStatic: s.noise.Public(), + init, err := appwire.NewInitiator(appwire.InitiatorOptions{StaticKey: key, RemoteStatic: service, Prologue: cloudwire.AppPrologue(home)}) if err != nil { s.t.Fatal(err) @@ -323,7 +365,7 @@ func (s *stack) dialPhone(home string, key appwire.KeyPair, now cloudwire.Now, o if ok := asClose(err, &ce); ok { p.code = ce.Code } - return p + return } reply, err := init.ReadMessage(msg2) if err != nil { @@ -337,7 +379,7 @@ func (s *stack) dialPhone(home string, key appwire.KeyPair, now cloudwire.Now, o go func() { for { _ = conn.SetReadDeadline(time.Time{}) - _, msg, err := conn.ReadMessage() + kind, msg, err := conn.ReadMessage() if err != nil { p.ended = err close(p.tuns) @@ -349,14 +391,21 @@ func (s *stack) dialPhone(home string, key appwire.KeyPair, now cloudwire.Now, o } payload, _ := cloudwire.DecodeFrame(frame) env, _ := cloudwire.DecodeEnvelope(payload) + p.kmu.Lock() + p.kinds[env.T] = append(p.kinds[env.T], kind) + p.kmu.Unlock() if env.T == cloudwire.MsgTun { var t cloudwire.AppTun _ = cloudwire.Unmarshal(env.B, &t) p.tuns <- t + continue + } + select { + case p.envs <- env: + default: } } }() - return p } func asClose(err error, ce **websocket.CloseError) bool { diff --git a/internal/store/session.go b/internal/store/session.go index 6a86062..606159f 100644 --- a/internal/store/session.go +++ b/internal/store/session.go @@ -23,10 +23,13 @@ type Out struct { At time.Duration } -// TunOut is one tun for a phone and when it arrived. +// TunOut is one tun for a phone and when it arrived. Bulk is how it came +// over QUIC, a lane 1 frame on the box's bulk stream, so the phone's goes +// the same way; it is never on the wire. type TunOut struct { cloudwire.AppTun - At time.Duration + At time.Duration + Bulk bool } // epoch starts the clock Out and TunOut are stamped with. @@ -155,7 +158,10 @@ func (s *Session) Ended(sid uint32) { // Tun hands one tun from the box to the session of the device it names. // False when no session of that device is open. A full queue drops it. -func (x *Handle) Tun(t cloudwire.Tun) bool { +func (x *Handle) Tun(t cloudwire.Tun) bool { return x.TunBy(t, false) } + +// TunBy is Tun for a tun that came on the box's bulk stream, when bulk. +func (x *Handle) TunBy(t cloudwire.Tun, bulk bool) bool { if x.kind != Box { return false } @@ -166,7 +172,7 @@ func (x *Handle) Tun(t cloudwire.Tun) bool { continue } select { - case sess.tun <- TunOut{AppTun: cloudwire.AppTun{Sid: t.Sid, D: t.D}, At: stamp()}: + case sess.tun <- TunOut{AppTun: cloudwire.AppTun{Sid: t.Sid, D: t.D}, At: stamp(), Bulk: bulk}: default: dropPhoneTun.Inc() } diff --git a/internal/uplink/uplink.go b/internal/uplink/uplink.go index 5172e96..4d89401 100644 --- a/internal/uplink/uplink.go +++ b/internal/uplink/uplink.go @@ -31,6 +31,7 @@ import ( "github.com/srcfl/ftw-cloud/internal/registry" "github.com/srcfl/ftw-cloud/internal/shard" "github.com/srcfl/ftw-cloud/internal/store" + "github.com/srcfl/ftw-cloud/internal/wire" "github.com/srcfl/ftw-cloud/internal/wsclose" "github.com/srcfl/ftw/go/protocol" "github.com/srcfl/ftw/go/protocol/cloudwire" @@ -281,6 +282,13 @@ func (u *Uplink) ServeHTTP(w http.ResponseWriter, r *http.Request) { if err != nil { return } + u.ServeConn(conn, ekm) +} + +// ServeConn serves one box's uplink on conn, a WebSocket or QUIC (package +// quicdoor). ekm is the exporter of the TLS session conn rides on, which the +// box's hello must carry. It closes conn. +func (u *Uplink) ServeConn(conn wire.Conn, ekm []byte) { defer conn.Close() conn.SetReadLimit(maxBoxFrame) boxSockets.Add(1) @@ -402,7 +410,7 @@ func verifiedRoster(h registry.Home) (*cloudwire.Roster, bool) { // boxConn is one box's connection. type boxConn struct { u *Uplink - conn *websocket.Conn + conn wire.Conn ekm []byte // write sends one frame; tests replace it. write func([]byte) error @@ -425,6 +433,11 @@ type boxConn struct { // pinger pings the box every pingEvery until stopped. pinger atomic.Pointer[time.Timer] stopped atomic.Bool + + // newest is the highest seq of each newest-wins class on this + // connection. Over QUIC an hb or a live may come as a datagram, out of + // order; one older than what the service holds is dropped. + newest map[string]uint64 } // bucket is a token bucket. @@ -503,6 +516,9 @@ func (c *boxConn) run() error { if err != nil { return err } + if kind != websocket.BinaryMessage { + return fmt.Errorf("%w: hello off the control stream", errProtocol) + } hello, err := c.hello(env, nonce) if err != nil { return err @@ -594,8 +610,14 @@ func (c *boxConn) run() error { if err != nil { return err } + switch { + case kind == wire.Datagram && !cloudwire.Datagram(env): + return fmt.Errorf("%w: a %s as a datagram", errProtocol, env.T) + case kind == wire.Bulk && env.T != cloudwire.MsgTun: + return fmt.Errorf("%w: a %s on the bulk stream", errProtocol, env.T) + } if env.T == cloudwire.MsgTun && !c.purged { - if err := c.fromBox(env); err != nil { + if err := c.fromBox(env, kind == wire.Bulk); err != nil { return err } continue @@ -633,8 +655,9 @@ func (c *boxConn) hello(env cloudwire.Envelope, nonce []byte) (cloudwire.Hello, // fromBox routes one tun from the box to the phone whose session it is. It // is not signed and never read: d is the phone's and the box's. A phone that -// has gone gets its session ended at the box. -func (c *boxConn) fromBox(env cloudwire.Envelope) error { +// has gone gets its session ended at the box. bulk is a tun from the box's +// bulk stream, which goes on the phone's. +func (c *boxConn) fromBox(env cloudwire.Envelope, bulk bool) error { var t cloudwire.Tun if len(env.Sig) != 0 || cloudwire.Unmarshal(env.B, &t) != nil || len(t.D) > cloudwire.TunDataMax || t.Dev == "" { return fmt.Errorf("%w: a tun the service cannot route", errProtocol) @@ -655,7 +678,7 @@ func (c *boxConn) fromBox(env cloudwire.Envelope) error { } tunFramesUp.Inc() tunBytesUp.Add(int64(len(t.D))) - if !c.handle.Tun(t) && !t.Ended() && c.tun { + if !c.handle.TunBy(t, bulk) && !t.Ended() && c.tun { return c.send(tun{Dev: t.Dev, Sid: t.Sid}) } return nil @@ -691,9 +714,9 @@ func (c *boxConn) allow() bool { } // parseUplink reads one box frame. Only binary lane 1 frames carry the -// protocol. +// protocol, on the control stream, the bulk stream or as a datagram. func parseUplink(kind int, msg []byte) (cloudwire.Envelope, error) { - if kind != websocket.BinaryMessage { + if kind != websocket.BinaryMessage && kind != wire.Bulk && kind != wire.Datagram { return cloudwire.Envelope{}, fmt.Errorf("%w: a text message", errProtocol) } payload, err := cloudwire.DecodeFrame(msg) @@ -745,6 +768,9 @@ func (c *boxConn) apply(env cloudwire.Envelope) (stop bool, err error) { // The readable copy is off: the service holds no readings. return false, nil } + if !c.newer(env.T, h.Seq) { + return false, nil + } raw, err := env.Encode() if err != nil { return true, err @@ -758,6 +784,23 @@ func (c *boxConn) apply(env cloudwire.Envelope) (stop bool, err error) { return false, nil } +// newer reports whether a message of a newest-wins class is newer than any +// of its class on this connection, and remembers it. Over a WebSocket every +// one is; over QUIC a datagram can arrive after a later message. +func (c *boxConn) newer(t string, seq uint64) bool { + if t != cloudwire.MsgLive && t != cloudwire.MsgHB { + return true + } + if c.newest == nil { + c.newest = map[string]uint64{} + } + if seq <= c.newest[t] { + return false + } + c.newest[t] = seq + return true +} + // carriesTable reports whether a live body holds its source table. func carriesTable(body []byte) bool { var l struct { diff --git a/internal/uplink/uplink_test.go b/internal/uplink/uplink_test.go index 81e9de5..438a763 100644 --- a/internal/uplink/uplink_test.go +++ b/internal/uplink/uplink_test.go @@ -414,6 +414,6 @@ func FuzzBoxTun(f *testing.F) { handle, _ := st.For(store.Principal{Kind: store.Box, Home: home}) c := &boxConn{u: u, home: home, pub: &key.PublicKey, handle: handle, tun: true, write: func([]byte) error { return nil }, lim: newLimits(time.Now())} - _ = c.fromBox(cloudwire.Envelope{T: cloudwire.MsgTun, B: body}) + _ = c.fromBox(cloudwire.Envelope{T: cloudwire.MsgTun, B: body}, false) }) } diff --git a/internal/wire/wire.go b/internal/wire/wire.go new file mode 100644 index 0000000..6e04271 --- /dev/null +++ b/internal/wire/wire.go @@ -0,0 +1,65 @@ +// Package wire is what the uplink and the app session need of a connection, +// so that the same handlers serve a box or a phone over a WebSocket and over +// QUIC (package quicdoor). A *websocket.Conn is a Conn as it is. +// +// QUIC adds two ways for a message to travel beside the control stream, each +// a message type of its own. A handler asks for one with Kind, which falls +// back to a binary message on a connection that lacks it, so it never has to +// know which transport it serves. +package wire + +import ( + "time" + + "github.com/gorilla/websocket" +) + +// Conn is a message connection: gorilla/websocket's method set. +type Conn interface { + ReadMessage() (messageType int, p []byte, err error) + WriteMessage(messageType int, data []byte) error + WriteControl(messageType int, data []byte, deadline time.Time) error + SetReadDeadline(t time.Time) error + SetWriteDeadline(t time.Time) error + SetReadLimit(limit int64) + SetPongHandler(h func(appData string) error) + Close() error +} + +// Message types beyond the WebSocket's. A Conn that reads one says how the +// message came; one that is asked to write one sends it that way. +const ( + // Bulk is a message on the bulk stream: a tun carrying a lane 1 frame of + // the phone's session, which must not hold up lane 0. + Bulk = 0x100 + // Datagram is a QUIC DATAGRAM: never sent again, the newest wins. + Datagram = 0x101 +) + +// Carrier is what a Conn that offers more than binary messages reports. +type Carrier interface { + // Carries reports whether the connection can send messages of this + // type now. A datagram needs the peer to have offered datagrams. + Carries(messageType int) bool + // Transport is the registry's name for it: quic or websocket. + Transport() string +} + +// Kind is the message type to write a message of the wanted type with on +// conn: the wanted one where the connection carries it, a binary message +// anywhere else. +func Kind(conn Conn, want int) int { + if want == websocket.BinaryMessage { + return want + } + if c, ok := conn.(Carrier); ok && c.Carries(want) { + return want + } + return websocket.BinaryMessage +} + +// Data reports whether a message type read from a Conn carries the +// protocol: a binary message, or one of QUIC's. +func Data(messageType int) bool { + return messageType == websocket.BinaryMessage || messageType == Bulk || messageType == Datagram +} diff --git a/internal/wsclose/wsclose.go b/internal/wsclose/wsclose.go index c875546..413ebc4 100644 --- a/internal/wsclose/wsclose.go +++ b/internal/wsclose/wsclose.go @@ -5,7 +5,8 @@ // such as 4421 with the right shard or 4503 with the spread. // // A Set tracks every socket a handler holds from the upgrade on, so a drain -// reaches those still in their handshake too. +// reaches those still in their handshake too. A socket is a WebSocket or a +// QUIC connection (package quicdoor), whose close is its application error. package wsclose import ( @@ -14,6 +15,8 @@ import ( "time" "github.com/gorilla/websocket" + + "github.com/srcfl/ftw-cloud/internal/wire" ) // Wait is how long a close waits for the peer's. @@ -21,7 +24,7 @@ const Wait = time.Second // Socket is one tracked socket. type Socket struct { - Conn *websocket.Conn + Conn wire.Conn sent atomic.Bool } @@ -58,7 +61,7 @@ type Set struct { } // Add tracks a socket from its upgrade on. -func (t *Set) Add(c *websocket.Conn) *Socket { +func (t *Set) Add(c wire.Conn) *Socket { s := &Socket{Conn: c} t.mu.Lock() defer t.mu.Unlock() From c607a18936499e65b175237bc0de4ce7b8d6240b Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Fri, 16 Oct 2026 07:23:14 +0200 Subject: [PATCH 3/4] fix(quicdoor): the review's fixes for the QUIC sessions - B1: a phone's session waits for the QUIC handshake, the first copy of an early flight too, so a flight the network dropped and replays later replaces nothing, spends no pairing code and asks for no watch. The ephemeral-key guard is gone; EarlyHeld counts sessions that waited. - B2: the door offers only the ALPNs it serves, so with the sessions off a client's handshake for them fails at once. - S2: a late live whose srcRev is above the table held still gives its table (store.PutTable), and the newest values stay. - S4: flow-control windows of 256 KB a stream and 512 KB a connection, at most 20,000 connections and 1,000 handshakes at once. A full door closes after the handshake with 4429, so the client reads the code. - Tests: a dropped early flight replayed after the phone dialled again, a door without the sessions, an early box target's exporter, a full door, a late table. Co-Authored-By: Claude Opus 5.5 --- README.md | 8 +- internal/quicdoor/conn.go | 8 -- internal/quicdoor/early.go | 79 ++-------------- internal/quicdoor/early_test.go | 30 +----- internal/quicdoor/quicdoor.go | 131 +++++++++++++++++++------- internal/quicdoor/quicdoor_test.go | 141 ++++++++++++++++++++++++++++ internal/server/early_e2e_test.go | 142 ++++++++++++++++++----------- internal/server/quic_e2e_test.go | 40 ++++++++ internal/store/store.go | 18 ++++ internal/uplink/uplink.go | 38 ++++++-- 10 files changed, 429 insertions(+), 206 deletions(-) diff --git a/README.md b/README.md index ed1a09b..a4f55d0 100644 --- a/README.md +++ b/README.md @@ -99,9 +99,11 @@ to it yet. except a WebSocket upgrade's. The uplink and the app session run over QUIC with the WebSockets' handlers (`internal/wire`): a control stream, a bulk stream for the tunnel's lane 1, and datagrams for `hb` and a `live` - without its table. Clients race QUIC against the WebSocket. Early data - is answered once per Noise message 1. A phone that moves from Wi-Fi to - cellular keeps its connection and its box session. + without its table. Clients race QUIC against the WebSocket, and the door + offers only the protocols it serves. A box's target may come in early + data; a phone's session waits for the handshake, so a replay changes + nothing. A phone that moves from Wi-Fi to cellular keeps its connection + and its box session. - **Alerts (C4).** A box's signed `event` goes to its phones through APNs, wrapped unchanged: the phone checks the signature, writes its own words and picks the buttons from the verified kind. Every alert has the same aps, diff --git a/internal/quicdoor/conn.go b/internal/quicdoor/conn.go index f5512a4..445f9c4 100644 --- a/internal/quicdoor/conn.go +++ b/internal/quicdoor/conn.go @@ -37,9 +37,6 @@ type Conn struct { limit atomic.Int64 start sync.Once in chan inbound - // pending holds messages the door read before handing the connection - // over; ReadMessage returns them first. - pending [][]byte mu sync.Mutex deadline time.Time @@ -107,11 +104,6 @@ func (c *Conn) Carries(messageType int) bool { // ReadMessage returns the next message from any of the connection's paths, // with the type it came by. func (c *Conn) ReadMessage() (int, []byte, error) { - if len(c.pending) > 0 { - m := c.pending[0] - c.pending = c.pending[1:] - return websocket.BinaryMessage, m, nil - } c.start.Do(c.readers) for { c.mu.Lock() diff --git a/internal/quicdoor/early.go b/internal/quicdoor/early.go index 7f82652..b51c803 100644 --- a/internal/quicdoor/early.go +++ b/internal/quicdoor/early.go @@ -4,24 +4,22 @@ import ( "bytes" "crypto/tls" "encoding/binary" - "sync" "time" "github.com/srcfl/ftw/go/protocol/cloudwire" ) // Early data (srcfl/ftw docs/protocol-cloud.md, "Early data"). The network can -// replay a client's first flight, and quic-go accepts what it carries. Two +// replay a client's first flight, and quic-go accepts what it carries. These // rules make a replay change nothing: // // - A session ticket allows early data for cloudwire.EarlyDataFor after it // was issued, and resumption without early data after that. -// - A phone's Noise message 1 in early data is answered once for each -// initiator ephemeral key. A second copy waits for a handshake its sender -// cannot finish, and the connection times out with nothing done. -// -// A box sends only its target in early data, and HTTP/3 requests wait for -// the handshake, so neither needs more. +// - A phone's session waits for the handshake (Door.session), which the +// sender of a replay cannot finish. +// - A box sends only its target in early data, and the challenge it may +// get back changes nothing. +// - HTTP/3 requests wait for the handshake. // earlyTag starts the ticket's entry that says when it was issued. var earlyTag = []byte("ftw.cloud.early.v1\x00") @@ -56,68 +54,3 @@ func earlyStillAllowed(extra [][]byte, now time.Time) bool { } return false } - -// ephemeralBytes is the initiator's ephemeral key at the front of Noise IK -// message 1, sent in the clear. -const ephemeralBytes = 32 - -// maxEarlySeen bounds the guard's memory: about 48 MB at most. Past it, early -// message 1s wait for the handshake, which costs a round trip and nothing -// else. -const maxEarlySeen = 1 << 20 - -// earlyGuard remembers each message 1 it let through in early data, for as -// long as a ticket allows early data. -type earlyGuard struct { - mu sync.Mutex - seen map[[ephemeralBytes]byte]time.Time - pruned time.Time - now func() time.Time -} - -func newEarlyGuard(now func() time.Time) *earlyGuard { - return &earlyGuard{seen: map[[ephemeralBytes]byte]time.Time{}, now: now} -} - -// first reports whether msg1 is the first message 1 with its ephemeral key, -// and remembers it. False also for a message too short to be message 1, and -// while the guard is full. -func (g *earlyGuard) first(msg1 []byte) bool { - if len(msg1) < ephemeralBytes { - return false - } - var e [ephemeralBytes]byte - copy(e[:], msg1) - g.mu.Lock() - defer g.mu.Unlock() - now := g.now() - if now.Sub(g.pruned) >= time.Minute { - for k, at := range g.seen { - if now.Sub(at) >= cloudwire.EarlyDataFor { - delete(g.seen, k) - } - } - g.pruned = now - } - if _, ok := g.seen[e]; ok || len(g.seen) >= maxEarlySeen { - return false - } - g.seen[e] = now - return true -} - -// remember records msg1 without asking, for a message 1 that came once the -// handshake was done on a connection that used early data: a copy of its -// first flight could still be replayed. -func (g *earlyGuard) remember(msg1 []byte) { - if len(msg1) < ephemeralBytes { - return - } - var e [ephemeralBytes]byte - copy(e[:], msg1) - g.mu.Lock() - defer g.mu.Unlock() - if len(g.seen) < maxEarlySeen { - g.seen[e] = g.now() - } -} diff --git a/internal/quicdoor/early_test.go b/internal/quicdoor/early_test.go index 5d4a663..334c948 100644 --- a/internal/quicdoor/early_test.go +++ b/internal/quicdoor/early_test.go @@ -1,7 +1,6 @@ package quicdoor import ( - "bytes" "encoding/binary" "testing" "time" @@ -34,34 +33,7 @@ func TestATicketAllowsEarlyDataForADay(t *testing.T) { } } -// Each message 1 passes once by its ephemeral key, for as long as a ticket -// allows early data. -func TestTheGuardLetsEachMessage1ThroughOnce(t *testing.T) { - now := time.Unix(1_790_000_000, 0) - g := newEarlyGuard(func() time.Time { return now }) - m1 := append(bytes.Repeat([]byte{1}, ephemeralBytes), []byte("the rest of message 1")...) - if !g.first(m1) { - t.Fatal("the first copy was refused") - } - again := append(bytes.Repeat([]byte{1}, ephemeralBytes), []byte("other bytes, same key")...) - if g.first(again) { - t.Fatal("a second copy with the same ephemeral key passed") - } - if g.first(m1[:ephemeralBytes-1]) { - t.Fatal("a message too short to be message 1 passed") - } - other := bytes.Repeat([]byte{2}, ephemeralBytes) - g.remember(other) - if g.first(other) { - t.Fatal("a remembered message 1 passed in early data") - } - now = now.Add(cloudwire.EarlyDataFor + time.Minute) - if !g.first(m1) { - t.Fatal("past a ticket's early-data life the key was still held") - } -} - -func TestEarlyDataIsOnUnderTheGuard(t *testing.T) { +func TestEarlyDataIsOn(t *testing.T) { if !Config().Allow0RTT { t.Fatal("0-RTT is off") } diff --git a/internal/quicdoor/quicdoor.go b/internal/quicdoor/quicdoor.go index ab89d51..ef9bf05 100644 --- a/internal/quicdoor/quicdoor.go +++ b/internal/quicdoor/quicdoor.go @@ -58,9 +58,20 @@ const ( // or a phone may open one control stream and one bulk stream; Conn // closes on more. maxStreams = 32 - // maxMessage1 bounds the phone's Noise message 1, which the door reads - // itself; the app session checks it again, more tightly. - maxMessage1 = 64 << 10 + // streamWindow and connWindow are the most a peer may have in flight to + // the door, on one stream and on one connection. Boxes and phones send + // little; the readable copy's bulk goes the other way. + streamWindow = 256 << 10 + connWindow = 512 << 10 +) + +// maxConns and maxHandshakes bound what the door holds at once: every +// connection, and those still in their handshake. Past either a new one is +// closed with 4429, busy, and its client tries the WebSocket or waits. +// Variables so a test can lower them. +var ( + maxConns int64 = 20_000 + maxHandshakes int64 = 1_000 ) // retryRate is how many new connections a second the door takes on trust. @@ -99,13 +110,15 @@ func (n *newcomers) retry(net.Addr) bool { // Door accepts QUIC connections and hands each to its protocol. type Door struct { - h3 *http3.Server - log *slog.Logger - early *earlyGuard - now func() time.Time - // EarlyRepeats counts message 1s in early data the guard held back: a - // replay, or a client that reused an ephemeral key. - EarlyRepeats atomic.Int64 + h3 *http3.Server + log *slog.Logger + now func() time.Time + // EarlyHeld counts phone sessions whose target came in early data and + // waited for the handshake before the session opened. + EarlyHeld atomic.Int64 + // conns and handshakes count what the door holds now (maxConns, + // maxHandshakes). + conns, handshakes atomic.Int64 // Box serves a box's uplink over QUIC and App a phone's session, with // the same handlers as their WebSockets. A nil one refuses its ALPN. // Set before Serve. @@ -130,7 +143,6 @@ func New(handler http.Handler, log *slog.Logger) *Door { log = slog.New(slog.DiscardHandler) } d := &Door{log: log, now: time.Now} - d.early = newEarlyGuard(func() time.Time { return d.now() }) if handler != nil { d.h3 = &http3.Server{ Handler: onlyRequests(handler), @@ -186,13 +198,30 @@ func PortOf(addr string) (int, error) { // TLSConfig is base for QUIC: TLS 1.3 and this door's protocols. It keeps // base's certificate source, so ACME renews both listeners at once. -func TLSConfig(base *tls.Config) *tls.Config { +func TLSConfig(base *tls.Config, protos ...string) *tls.Config { c := base.Clone() c.MinVersion = tls.VersionTLS13 - c.NextProtos = []string{ALPNH3, cloudwire.ALPNBox, cloudwire.ALPNApp} + c.NextProtos = protos return c } +// protos are the ALPNs the door serves. One it does not serve is not +// offered, so a client's handshake for it fails at once and the client +// takes the WebSocket, rather than winning a race and losing the session. +func (d *Door) protos() []string { + var p []string + if d.h3 != nil { + p = append(p, ALPNH3) + } + if d.Box != nil { + p = append(p, cloudwire.ALPNBox) + } + if d.App != nil { + p = append(p, cloudwire.ALPNApp) + } + return p +} + // Config is the QUIC configuration the door listens with. A connection // silent for cloudwire.QUICIdleTimeout ends; boxes and phones send keepalives // well inside it. Datagrams are on for the readable copy's hb and live. @@ -205,6 +234,11 @@ func Config() *quic.Config { MaxIncomingUniStreams: 8, EnableDatagrams: true, Allow0RTT: true, + + InitialStreamReceiveWindow: streamWindow, + MaxStreamReceiveWindow: streamWindow, + InitialConnectionReceiveWindow: connWindow, + MaxConnectionReceiveWindow: connWindow, } } @@ -221,7 +255,7 @@ func (d *Door) Serve(ctx context.Context, pc net.PacketConn, tlsConf *tls.Config } d.tr = tr d.mu.Unlock() - tc := TLSConfig(tlsConf) + tc := TLSConfig(tlsConf, d.protos()...) limitEarlyData(tc, d.now) ln, err := tr.ListenEarly(tc, Config()) if err != nil { @@ -245,10 +279,50 @@ func (d *Door) Serve(ctx context.Context, pc net.PacketConn, tlsConf *tls.Config } return err } + if !d.admit(conn) { + continue + } go d.serve(conn) } } +// admit counts conn against maxConns and maxHandshakes, or closes it when +// the door holds enough. +func (d *Door) admit(conn *quic.Conn) bool { + busy := quic.ApplicationErrorCode(cloudwire.CloseBusy) + if !done(conn) && d.handshakes.Load() >= maxHandshakes { + // Too many handshakes at once: shed this one now. A close this + // early loses its code (RFC 9000 §10.2.3), so its client counts QUIC + // as failing here and takes the WebSocket. + _ = conn.CloseWithError(busy, "") + return false + } + if d.conns.Load() >= maxConns { + // After the handshake the client reads 4429, busy, and waits. + go func() { + select { + case <-conn.HandshakeComplete(): + case <-conn.Context().Done(): + } + _ = conn.CloseWithError(busy, "") + }() + return false + } + d.conns.Add(1) + context.AfterFunc(conn.Context(), func() { d.conns.Add(-1) }) + if !done(conn) { + d.handshakes.Add(1) + go func() { + select { + case <-conn.HandshakeComplete(): + case <-conn.Context().Done(): + } + d.handshakes.Add(-1) + }() + } + return true +} + // Close ends every live session with 1001, going away, so its client dials // again at once, and then the transport. Call it on the way out, after // Serve's context is done and after any drain. @@ -338,29 +412,18 @@ func (d *Door) session(qc *quic.Conn, box bool) { return } c := newConn(qc, ctrl, r, box) + // A phone's message 1 may come in early data, which the network can + // replay. Its session opens only once the handshake is done, which the + // sender of a replay cannot do, so nothing is replaced, spent, forwarded + // or asked for until then: for the first copy too, which matters when + // the network dropped the original and the phone dialled again. if !box { - // Message 1, read here so that a copy replayed in early data stops at - // the door: the second one with an ephemeral key waits for a - // handshake its sender cannot finish. - _ = ctrl.SetReadDeadline(time.Now().Add(handshakeTimeout)) - msg1, err := cloudwire.ReadMessage(r, maxMessage1) - _ = ctrl.SetReadDeadline(time.Time{}) - if err != nil { - refuse(cloudwire.CloseProtocol) - return + if !done(qc) { + d.EarlyHeld.Add(1) } - switch { - case !done(qc): - if !d.early.first(msg1) { - d.EarlyRepeats.Add(1) - if !d.handshake(qc) { - return - } - } - case qc.ConnectionState().Used0RTT: - d.early.remember(msg1) + if !d.handshake(qc) { + return } - c.pending = [][]byte{msg1} } var ekm []byte if box { diff --git a/internal/quicdoor/quicdoor_test.go b/internal/quicdoor/quicdoor_test.go index 45113a3..fc67fc4 100644 --- a/internal/quicdoor/quicdoor_test.go +++ b/internal/quicdoor/quicdoor_test.go @@ -16,6 +16,7 @@ import ( "net/http" "net/http/httptest" "os" + "strings" "sync" "testing" "time" @@ -353,3 +354,143 @@ func TestCloseEndsEachSessionWithGoingAway(t *testing.T) { t.Fatalf("the session ended with %v, not 1001", err) } } + +// A door that does not serve the sessions does not offer their ALPNs, so a +// client's handshake for one fails at once and it takes the WebSocket, +// rather than winning the race and losing the session. +func TestTheDoorOffersOnlyWhatItServes(t *testing.T) { + cert, pool := testCert(t) + addr := serve(t, http.NotFoundHandler(), cert) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + start := time.Now() + c, err := Dial(ctx, addr, &tls.Config{RootCAs: pool}, cloudwire.ALPNBox, cloudwire.TargetBox, nil) + if err == nil { + c.Close() + t.Fatal("a door without the box's handler finished a handshake for it") + } + if took := time.Since(start); took > time.Second { + t.Fatalf("the refused handshake took %v", took) + } + if !strings.Contains(err.Error(), "no_application_protocol") && !strings.Contains(err.Error(), "0x178") { + t.Fatalf("refused with %v, not for the protocol", err) + } +} + +// A box that dials again sends its target as early data, and its hello must +// bind to the exporter of that same TLS session. The door hands the box's +// handler the exporter before the handshake ends; it is the client's. +func TestABoxsEarlyTargetGetsItsSessionsExporter(t *testing.T) { + cert, pool := testCert(t) + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer pc.Close() + d := New(nil, nil) + ekms := make(chan []byte, 2) + d.Box = func(c *Conn, ekm []byte) { + ekms <- ekm + for { + if _, _, err := c.ReadMessage(); err != nil { + return + } + } + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { _ = d.Serve(ctx, pc, &tls.Config{Certificates: []tls.Certificate{cert}}) }() + defer d.Close() + + cache := tls.NewLRUClientSessionCache(4) + dial := func() *Conn { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + c, err := Dial(ctx, pc.LocalAddr().String(), &tls.Config{RootCAs: pool, ClientSessionCache: cache}, + cloudwire.ALPNBox, cloudwire.TargetBox, nil) + if err != nil { + t.Fatal(err) + } + <-c.QUIC().HandshakeComplete() + return c + } + first := dial() + <-ekms + time.Sleep(100 * time.Millisecond) // the ticket + _ = first.Close() + + second := dial() + defer second.Close() + if !second.UsedEarlyData() { + t.Fatal("the second connection used no early data") + } + var got []byte + select { + case got = <-ekms: + case <-time.After(3 * time.Second): + t.Fatal("the box's handler never ran") + } + want, err := second.Exporter() + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Fatal("the door's exporter for an early target is not the client's") + } +} + +// Past maxConns the door closes a new connection with 4429. +func TestAFullDoorSaysItIsBusy(t *testing.T) { + old := maxConns + maxConns = 1 + t.Cleanup(func() { maxConns = old }) + cert, pool := testCert(t) + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer pc.Close() + d := New(nil, nil) + d.Box = func(c *Conn, _ []byte) { + for { + if _, _, err := c.ReadMessage(); err != nil { + return + } + } + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { _ = d.Serve(ctx, pc, &tls.Config{Certificates: []tls.Certificate{cert}}) }() + defer d.Close() + dial := func() *Conn { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + c, err := Dial(ctx, pc.LocalAddr().String(), &tls.Config{RootCAs: pool}, cloudwire.ALPNBox, cloudwire.TargetBox, nil) + if err != nil { + t.Fatal(err) + } + return c + } + first := dial() + defer first.Close() + time.Sleep(100 * time.Millisecond) + // The close may reach the client while it dials or once it reads. + dctx, dcancel := context.WithTimeout(context.Background(), 3*time.Second) + defer dcancel() + second, err := Dial(dctx, pc.LocalAddr().String(), &tls.Config{RootCAs: pool}, cloudwire.ALPNBox, cloudwire.TargetBox, nil) + if err == nil { + defer second.Close() + _ = second.SetReadDeadline(time.Now().Add(3 * time.Second)) + _, _, err = second.ReadMessage() + } + var ae *quic.ApplicationError + var ce *websocket.CloseError + switch { + case errors.As(err, &ae) && ae.ErrorCode == quic.ApplicationErrorCode(cloudwire.CloseBusy): + case errors.As(err, &ce) && ce.Code == cloudwire.CloseBusy: + default: + t.Fatalf("the second connection ended with %v, not 4429", err) + } +} diff --git a/internal/server/early_e2e_test.go b/internal/server/early_e2e_test.go index 0f986b3..2bf24df 100644 --- a/internal/server/early_e2e_test.go +++ b/internal/server/early_e2e_test.go @@ -1,6 +1,7 @@ package server import ( + "bytes" "context" "crypto/tls" "net" @@ -24,6 +25,9 @@ type flightRecorder struct { t *testing.T pc net.PacketConn door *net.UDPAddr + // dropNext makes the next new client's packets go nowhere, as a + // network that records a flight and drops it. + dropNext bool mu sync.Mutex flights map[string]*flight @@ -33,6 +37,7 @@ type flightRecorder struct { type flight struct { up *net.UDPConn packets [][]byte + dropped bool } func newFlightRecorder(t *testing.T, door string) *flightRecorder { @@ -59,6 +64,12 @@ func (r *flightRecorder) run() { pkt := append([]byte(nil), buf[:n]...) r.mu.Lock() f := r.flights[from.String()] + if f == nil && r.dropNext { + r.dropNext = false + f = &flight{dropped: true} + r.flights[from.String()] = f + r.order = append(r.order, from.String()) + } if f == nil { up, err := net.DialUDP("udp", nil, r.door) if err != nil { @@ -85,10 +96,19 @@ func (r *flightRecorder) run() { f.packets = append(f.packets, pkt) } r.mu.Unlock() - _, _ = f.up.Write(pkt) + if !f.dropped { + _, _ = f.up.Write(pkt) + } } } +// drop makes the next new client's flight go nowhere. +func (r *flightRecorder) drop() { + r.mu.Lock() + r.dropNext = true + r.mu.Unlock() +} + // first is the first flight of the n-th client. func (r *flightRecorder) first(n int) [][]byte { r.mu.Lock() @@ -96,19 +116,21 @@ func (r *flightRecorder) first(n int) [][]byte { return slices.Clone(r.flights[r.order[n]].packets) } -// A phone's second connection carries its Noise message 1, and the box -// session's message 1 inside it, in early data: one round trip sooner. A -// network that replays that flight later, from another address, changes -// nothing: the door holds the copy back, the box hears no handshake and no -// watch, and the phone's live session is not replaced. -func TestAReplayedEarlyFlightChangesNothing(t *testing.T) { +// A network records a phone's early flight and drops it. The phone dials +// again, over the WebSocket, and only then does the network replay the +// flight. That changes nothing: the door holds the copy for a handshake its +// sender cannot finish, so the live session is not replaced, the box hears +// no handshake and no watch, and a pairing code in a flight stays unspent. +func TestADroppedEarlyFlightReplayedLaterChangesNothing(t *testing.T) { s := newStack(t) box := s.connectTunBoxQUIC() - key, _ := appwire.GenerateKeyPair() - box.roster(2, grantOff, nil, key.Public()) + known, _ := appwire.GenerateKeyPair() + code := bytes.Repeat([]byte{0x42}, 16) + box.roster(2, grantOff, code, known.Public()) + waitRoster(t, s, box.home, known) rec := newFlightRecorder(t, s.quic()) cache := tls.NewLRUClientSessionCache(4) - dial := func(cache tls.ClientSessionCache) *quicdoor.Conn { + dial := func() *quicdoor.Conn { t.Helper() tc := s.clientTLS() tc.ClientSessionCache = cache @@ -120,72 +142,82 @@ func TestAReplayedEarlyFlightChangesNothing(t *testing.T) { } return c } - now := cloudwire.Now{Hz: cloudwire.HzLooking} - - // The first connection holds no ticket; it leaves with one. - first := newTunPhone(t, key) - first.session(dial(cache), s.noise.Public(), box.home, now) - if first.tr == nil { - t.Fatalf("the first session did not open: %d", first.code) + // ticket makes one connection that completes its handshake and keeps + // the ticket it leaves with. + ticket := func() { + t.Helper() + c := dial() + select { + case <-c.QUIC().HandshakeComplete(): + case <-time.After(3 * time.Second): + t.Fatal("no handshake") + } + time.Sleep(100 * time.Millisecond) + _ = c.Close() } - first.open(box.static.Public(), 21, nil) - time.Sleep(100 * time.Millisecond) - _ = first.conn.Close() - - // The second resumes and sends message 1 in early data. - second := newTunPhone(t, key) - conn := dial(cache) - second.session(conn, s.noise.Public(), box.home, cloudwire.Now{Hz: cloudwire.HzLooking, - Tun: &cloudwire.AppTun{Sid: 22, D: second.boxMessage1(box.static.Public(), nil)}}) - if second.tr == nil { - t.Fatalf("the early session did not open: %d", second.code) + // early dials with the ticket and sends message 1 in early data into a + // network that drops it. + early := func(key appwire.KeyPair, now cloudwire.Now) { + t.Helper() + rec.drop() + p := newTunPhone(t, key) + c := dial() + p.session(c, s.noise.Public(), box.home, now) + if p.tr != nil { + t.Fatal("a dropped flight opened a session") + } } - if !conn.UsedEarlyData() { - t.Fatal("the second connection did not use early data") + // carriesEarlyData reports whether a flight holds a 0-RTT packet: a long + // header of type 1. + carriesEarlyData := func(flight [][]byte) bool { + return slices.ContainsFunc(flight, func(p []byte) bool { return p[0]&0x80 != 0 && (p[0]>>4)&3 == 1 }) } - second.boxMessage2(second.next().D) - if ack, _ := second.cmd(22, "early"); ack != protocol.MsgCmdAck { - t.Fatal("no ack on the early session") + + ticket() + pairing, _ := appwire.GenerateKeyPair() + early(pairing, cloudwire.Now{Hz: cloudwire.HzLooking, Pair: cloudwire.PairingHash(box.home, code)}) + flightOfPairing := rec.first(1) + ticket() + early(known, cloudwire.Now{Hz: cloudwire.HzLooking, + Tun: &cloudwire.AppTun{Sid: 31, D: newTunPhone(t, known).boxMessage1(box.static.Public(), nil)}}) + flightOfKnown := rec.first(3) + if !carriesEarlyData(flightOfPairing) || !carriesEarlyData(flightOfKnown) { + t.Fatal("a recorded flight carried no early data") } - flightOfSecond := rec.first(1) - _ = conn.Close() - // The phone's live session now, over the WebSocket, which the replay - // must not replace. - live := s.dialPhone(box.home, key, cloudwire.Now{}, "") + // The known phone dials again, over the WebSocket: its live session. + live := s.dialPhone(box.home, known, cloudwire.Now{}, "") if live.tr == nil { t.Fatal("the live session did not open") } live.open(box.static.Public(), 23, nil) - // Wait out quic-go's memory of the closed connection, so the replay - // reaches the door as a connection of its own. - time.Sleep(1500 * time.Millisecond) handshakes := box.handshakes() box.mu.Lock() heard := len(box.heard) box.mu.Unlock() - repeats := s.door.EarlyRepeats.Load() - - replay, err := net.DialUDP("udp", nil, mustUDP(t, s.quic())) - if err != nil { - t.Fatal(err) - } - defer replay.Close() - for _, p := range flightOfSecond { - if _, err := replay.Write(p); err != nil { + held := s.door.EarlyHeld.Load() + for _, flight := range [][][]byte{flightOfKnown, flightOfPairing} { + replay, err := net.DialUDP("udp", nil, mustUDP(t, s.quic())) + if err != nil { t.Fatal(err) } + defer replay.Close() + for _, p := range flight { + if _, err := replay.Write(p); err != nil { + t.Fatal(err) + } + } } // The replayer cannot finish the handshake; give the door every chance // to act on what it got. time.Sleep(1500 * time.Millisecond) - if got := s.door.EarlyRepeats.Load(); got != repeats+1 { - t.Fatalf("the door held back %d copies of message 1, want 1: the replay never reached it", got-repeats) + if got := s.door.EarlyHeld.Load() - held; got != 2 { + t.Fatalf("the door held %d replayed sessions for the handshake, want 2: a replay never reached it", got) } if got := box.handshakes(); got != handshakes { - t.Fatalf("the replay reached the box: %d handshakes", got-handshakes) + t.Fatalf("a replay reached the box: %d handshakes", got-handshakes) } box.mu.Lock() since := slices.Clone(box.heard[heard:]) @@ -201,6 +233,10 @@ func TestAReplayedEarlyFlightChangesNothing(t *testing.T) { if live.ended != nil { t.Fatalf("the live session ended: %v", live.ended) } + // The pairing code is still there for the phone that holds it. + if p := s.dialPhone(box.home, pairing, cloudwire.Now{Hz: 1, Pair: cloudwire.PairingHash(box.home, code)}, ""); p.tr == nil { + t.Fatalf("the replay spent the pairing code: refused with %d", p.code) + } } func mustUDP(t *testing.T, addr string) *net.UDPAddr { diff --git a/internal/server/quic_e2e_test.go b/internal/server/quic_e2e_test.go index 4470a8b..29996c0 100644 --- a/internal/server/quic_e2e_test.go +++ b/internal/server/quic_e2e_test.go @@ -273,6 +273,46 @@ func TestALateDatagramIsDropped(t *testing.T) { } } +// Over QUIC a live without its table can overtake the one that carries it. +// The service still takes the late table, and keeps the newer values: a +// phone's snapshot holds both, so its sources do not go stale for want of a +// table. +func TestALateTableStillCounts(t *testing.T) { + s := newStack(t) + box := s.connectTunBoxQUIC() + key, _ := appwire.GenerateKeyPair() + box.roster(2, grantServe, nil, key.Public()) + table := map[string]cloudwire.Source{"m": {Kind: "driver", State: protocol.SourceLive, StaleAfterMs: 10_000}} + box.send(cloudwire.MsgLive, cloudwire.Live{Header: box.header(), Fields: map[string]int64{"2": 1}, Sources: table, SrcRev: 1, Ages: map[string]int64{}}) + withTable := mustSeal(t, box, cloudwire.MsgLive, cloudwire.Live{Header: box.header(), Fields: map[string]int64{"2": 2}, + Sources: table, FieldSources: map[string][]string{"2": {"m"}}, SrcRev: 2, Ages: map[string]int64{}}) + box.writeAs(wire.Datagram, mustSeal(t, box, cloudwire.MsgLive, cloudwire.Live{Header: box.header(), Fields: map[string]int64{"2": 3}, SrcRev: 2, Ages: map[string]int64{}})) + time.Sleep(100 * time.Millisecond) + box.writeAs(websocket.BinaryMessage, withTable) + box.send(cloudwire.MsgHB, cloudwire.HB{Header: box.header(), Clock: cloudwire.ClockNTP, SrcRev: 2}) + waitHeldN(t, s, box.home, key, 3) + var newest int64 + var tableRev uint64 + for _, raw := range heldSnapshot(t, s, box.home, key).Envelopes { + env, _ := cloudwire.DecodeEnvelope(raw) + if env.T != cloudwire.MsgLive { + continue + } + var l cloudwire.Live + _ = cloudwire.Unmarshal(env.B, &l) + if l.Sources != nil { + tableRev = max(tableRev, l.SrcRev) + } + newest = l.Fields["2"] + } + if tableRev != 2 { + t.Fatalf("the service holds table %d, not the late one", tableRev) + } + if newest != 3 { + t.Fatalf("the newest live in the snapshot holds %d, not the overtaking one's 3", newest) + } +} + // The box binds hello to the TLS session inside QUIC. Its exporter is the // service's, so hello_ok comes; one from another session closes the uplink. func TestHelloOverQUICIsBoundToItsTLSSession(t *testing.T) { diff --git a/internal/store/store.go b/internal/store/store.go index 7912e14..5b1d930 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -364,6 +364,24 @@ func (x *Handle) Put(t string, env []byte, seq uint64, carriesTable bool) error return nil } +// PutTable keeps a live that came after a newer one but carries a newer +// source table: it becomes the table held, and the newest values stay. The +// phones reading take its table the same way. +func (x *Handle) PutTable(env []byte, seq uint64) { + if x.kind != Box { + return + } + x.h.mu.Lock() + defer x.h.mu.Unlock() + x.h.full = &Held{T: cloudwire.MsgLive, Envelope: env, Seq: seq} + at := stamp() + for sess := range x.h.sessions { + if sess.x.mayRead(cloudwire.MsgLive) { + sess.deliver(env, at) + } + } +} + // Roster returns the roster held, and its number, 0 when none. func (x *Handle) Roster() (*cloudwire.Roster, uint64) { x.h.mu.Lock() diff --git a/internal/uplink/uplink.go b/internal/uplink/uplink.go index 4d89401..384032e 100644 --- a/internal/uplink/uplink.go +++ b/internal/uplink/uplink.go @@ -436,8 +436,10 @@ type boxConn struct { // newest is the highest seq of each newest-wins class on this // connection. Over QUIC an hb or a live may come as a datagram, out of - // order; one older than what the service holds is dropped. - newest map[string]uint64 + // order; one older than what the service holds is dropped. tableRev is + // the srcRev of the newest table held from this connection. + newest map[string]uint64 + tableRev uint64 } // bucket is a token bucket. @@ -768,14 +770,33 @@ func (c *boxConn) apply(env cloudwire.Envelope) (stop bool, err error) { // The readable copy is off: the service holds no readings. return false, nil } + rev, table := uint64(0), false + if env.T == cloudwire.MsgLive { + rev, table = tableOf(env.B) + } if !c.newer(env.T, h.Seq) { + // Over QUIC a live without its table can overtake the one that + // carries it. The late one still brings a newer table, and nothing + // else (docs/protocol-cloud.md, "Datagrams"). + if !table || rev <= c.tableRev { + return false, nil + } + raw, err := env.Encode() + if err != nil { + return true, err + } + c.tableRev = rev + c.handle.PutTable(raw, h.Seq) return false, nil } raw, err := env.Encode() if err != nil { return true, err } - if err := c.handle.Put(env.T, raw, h.Seq, env.T == cloudwire.MsgLive && carriesTable(env.B)); err != nil { + if table { + c.tableRev = max(c.tableRev, rev) + } + if err := c.handle.Put(env.T, raw, h.Seq, table); err != nil { return true, err } if c.u.OnHeld != nil { @@ -801,12 +822,17 @@ func (c *boxConn) newer(t string, seq uint64) bool { return true } -// carriesTable reports whether a live body holds its source table. -func carriesTable(body []byte) bool { +// tableOf reports whether a live body holds its source table, and the +// table's srcRev. +func tableOf(body []byte) (uint64, bool) { var l struct { Sources map[string]cbor.RawMessage `cbor:"sources"` + SrcRev uint64 `cbor:"srcRev"` + } + if cloudwire.Unmarshal(body, &l) != nil || len(l.Sources) == 0 { + return 0, false } - return cloudwire.Unmarshal(body, &l) == nil && len(l.Sources) > 0 + return l.SrcRev, true } // roster keeps a signed roster whose grant and scopes are within what FTW From 6d7dcd417678a13e97763444c2b5d09fe1fa716b Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Fri, 16 Oct 2026 10:39:26 +0200 Subject: [PATCH 4/4] chore: pin go/protocol to srcfl/ftw master after #1653 (8fdc990), with main merged Co-Authored-By: Claude Opus 5.5 --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index f76725a..537b92b 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/fxamacker/cbor/v2 v2.9.4 github.com/gorilla/websocket v1.5.3 github.com/quic-go/quic-go v0.63.0 - github.com/srcfl/ftw/go/protocol v0.0.0-20261016050141-c49da79dfc79 + github.com/srcfl/ftw/go/protocol v0.0.0-20261016083708-8fdc990b05ac golang.org/x/crypto v0.57.0 golang.org/x/sys v0.48.0 modernc.org/sqlite v1.59.0 diff --git a/go.sum b/go.sum index aec6aad..c844831 100644 --- a/go.sum +++ b/go.sum @@ -52,10 +52,8 @@ github.com/quic-go/quic-go v0.63.0 h1:LIFGHI4PFUhhw2dDD1ARHdCff143ffMHwZtbnbuJ78 github.com/quic-go/quic-go v0.63.0/go.mod h1:RAro2j2yN9a9EiPACLHT9IB2NXCvGQmmo/alT0yYI0w= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/srcfl/ftw/go/protocol v0.0.0-20261015212100-f024251f3248 h1:56VRLOp7pcN9FyU6ncNri5y1nYCRlG6zD+ZKko46HTA= -github.com/srcfl/ftw/go/protocol v0.0.0-20261015212100-f024251f3248/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= -github.com/srcfl/ftw/go/protocol v0.0.0-20261016050141-c49da79dfc79 h1:BnMVCZe7r6zmaCQeiImMeZ5AlJOqL52ok8ef8s39UKo= -github.com/srcfl/ftw/go/protocol v0.0.0-20261016050141-c49da79dfc79/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= +github.com/srcfl/ftw/go/protocol v0.0.0-20261016083708-8fdc990b05ac h1:SrPUWxe26lQYr+UxxRvmm3rCqYmUrMaCpJ3dIQJ8Z4E= +github.com/srcfl/ftw/go/protocol v0.0.0-20261016083708-8fdc990b05ac/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=