The uplink and the app session over QUIC, with 0-RTT a replay cannot use (QUIC steps 2–3)

README.md

@@ -93,12 +93,17 @@ to it yet.
   most 4 KiB, the answer within 10 s.
 - Limits per home: 16 sessions, one per device, 256 envelopes queued per
   phone, and a small message rate from each phone.
-- **HTTP/3.** The plain HTTPS routes, `/v1/app/once`, `/v1/now`,
-  `/healthz` and `/source`, also answer over HTTP/3 on UDP 443, with the
-  same certificate (`internal/quicdoor`). TCP responses say so in
-  `Alt-Svc`, except a WebSocket upgrade's. The sockets stay on TCP. A read
-  on a new connection takes two round trips instead of three; under loss
-  the gap is larger (`scripts/netem-once.sh`, [deploy/http3.md](deploy/http3.md)).
+- **QUIC.** UDP 443 carries what TCP 443 does, with the same certificate
+  (`internal/quicdoor`, [deploy/http3.md](deploy/http3.md)). The plain
+  HTTPS routes answer over HTTP/3, and TCP responses say so in `Alt-Svc`,
+  except a WebSocket upgrade's. The uplink and the app session run over
+  QUIC with the WebSockets' handlers (`internal/wire`): a control stream, a
+  bulk stream for the tunnel's lane 1, and datagrams for `hb` and a `live`
+  without its table. Clients race QUIC against the WebSocket, and the door
+  offers only the protocols it serves. A box's target may come in early
+  data; a phone's session waits for the handshake, so a replay changes
+  nothing. A phone that moves from Wi-Fi to cellular keeps its connection
+  and its box session.
 - **Alerts (C4).** A box's signed `event` goes to its phones through APNs,
   wrapped unchanged: the phone checks the signature, writes its own words and
   picks the buttons from the verified kind. Every alert has the same aps,

cmd/ftwcloudd/main.go

@@ -100,6 +100,7 @@ func serve(args []string) error {
 	restartRate := fs.Int("restart-rate", defaultRestartRate, "sockets a second the next instance can take after a restart; sets the spread of the redial")
 	restartMin := fs.Duration("restart-spread-min", 15*time.Second, "the shortest spread: longer than the port stays closed while the next container starts")
 	h3 := fs.Bool("h3", true, "also serve HTTP/3 on the same address over UDP, and say so in Alt-Svc")
+	quicSessions := fs.Bool("quic-sessions", true, "also serve /v1/box and /v1/app over QUIC on the same address over UDP")
 	_ = fs.Parse(args)
 
 	harden()
@@ -233,22 +234,33 @@ func serve(args []string) error {
 		tlsConfig.NextProtos = append(tlsConfig.NextProtos, "acme-tls/1")
 	}
 
-	// HTTP/3 on UDP, beside TCP, for the plain HTTPS routes. The sockets
-	// stay on TCP. The UDP port is bound before Alt-Svc says it is there,
-	// and a port that will not bind leaves TCP serving alone: QUIC is the
-	// fast path, never the only one.
+	// QUIC on UDP, beside TCP: HTTP/3 for the plain HTTPS routes, and the
+	// uplink and the app session with the handlers their WebSockets have.
+	// The UDP port is bound before Alt-Svc says it is there, and a port that
+	// will not bind leaves TCP serving alone: QUIC is the fast path, never
+	// the only one.
 	var door *quicdoor.Door
 	var udp net.PacketConn
-	if *h3 {
+	if *h3 || *quicSessions {
 		port, err := quicdoor.PortOf(*listen)
 		if err != nil {
 			return err
 		}
 		if udp, err = net.ListenPacket("udp", *listen); err != nil {
-			log.Error("HTTP/3 is off: UDP did not bind; TCP serves alone", "err", err)
+			log.Error("QUIC is off: UDP did not bind; TCP serves alone", "err", err)
 		} else {
-			srv.AltSvc = quicdoor.AltSvc(port)
-			door = quicdoor.New(srv.Handler(), log)
+			var h3Handler http.Handler
+			if *h3 {
+				srv.AltSvc = quicdoor.AltSvc(port)
+				h3Handler = srv.Handler()
+			}
+			door = quicdoor.New(h3Handler, log)
+			if *quicSessions {
+				door.Box = func(c *quicdoor.Conn, ekm []byte) { up.ServeConn(c, ekm) }
+				if srv.App != nil {
+					door.App = func(c *quicdoor.Conn, home string) { srv.App.ServeConn(c, home) }
+				}
+			}
 			if door.ResetKey, err = quicdoor.ResetKeyFile(filepath.Join(*stateDir, "quic-reset.key")); err != nil {
 				log.Warn("no stateless resets: a client waits out its idle timeout after a restart", "err", err)
 			}
@@ -333,15 +345,19 @@ func serve(args []string) error {
 		stopPush()
 	}()
 	if door != nil {
+		// On the way out the live QUIC sessions end with 1001. Silence would
+		// leave each client to its idle timeout.
+		defer door.Close()
 		go func() {
 			if err := door.Serve(ctx, udp, tlsConfig); err != nil {
-				log.Error("HTTP/3 stopped; TCP serves on", "err", err)
+				log.Error("QUIC stopped; TCP serves on", "err", err)
 			}
 		}()
 	}
 
 	pub, _ := cloudwire.PublicKeyBytesOf(&key.PublicKey)
-	log.Info("ftwcloudd listening", "addr", *listen, "h3", door != nil, "shard", shards.Self(), "shards", shards.Shards(),
+	log.Info("ftwcloudd listening", "addr", *listen, "h3", srv.AltSvc != "", "quic_sessions", door != nil && door.Box != nil,
+		"shard", shards.Self(), "shards", shards.Shards(),
 		"service_key", base64.RawURLEncoding.EncodeToString(pub), "noise_key", base64.RawURLEncoding.EncodeToString(noise.Public()))
 	err = httpSrv.ListenAndServeTLS("", "")
 	if err != nil && !errors.Is(err, http.ErrServerClosed) {

deploy/bootstrap.sh

@@ -174,8 +174,9 @@ if ! [[ $rev =~ ^[0-9a-f]{40}$ ]] || ! git merge-base --is-ancestor "$rev" origi
 fi
 git checkout --quiet --detach "$rev"
 
-# Only the image line changes; what keys.sh set, and FTW_CLOUD_H3=false set
-# by hand to turn HTTP/3 off (deploy/http3.md), stay.
+# Only the image line changes; what keys.sh set, and FTW_CLOUD_H3=false or
+# FTW_CLOUD_QUIC_SESSIONS=false set by hand to turn QUIC off
+# (deploy/http3.md), stay.
 touch "$env"
 sed -i '/^FTW_CLOUD_IMAGE=/d' "$env"
 printf 'FTW_CLOUD_IMAGE=%s\n' "$image" >> "$env"

deploy/compose.yml

@@ -20,6 +20,7 @@ services:
     command:
       - -listen=:443
       - -h3=${FTW_CLOUD_H3:-true}
+      - -quic-sessions=${FTW_CLOUD_QUIC_SESSIONS:-true}
       - -domain=cloud.ftw.energy
       - -state=/state
       - -service-key=/run/secrets/ftw-cloud-service.key

deploy/http3.md

@@ -1,10 +1,16 @@
-# HTTP/3 on UDP 443
+# QUIC on UDP 443
 
-`ftwcloudd` serves HTTP/3 on UDP 443 beside TCP 443, with the same
-certificate, for the routes that are plain HTTPS: `/v1/app/once`, `/v1/now`,
-`/healthz` and `/source`. The sockets, `/v1/box` and `/v1/app`, stay on TCP.
-TCP responses carry `Alt-Svc: h3=":443"; ma=86400`, except a WebSocket
-upgrade's.
+`ftwcloudd` serves QUIC on UDP 443 beside TCP 443, with the same
+certificate:
+
+- HTTP/3 (`h3`) for the routes that are plain HTTPS: `/v1/app/once`,
+  `/v1/now`, `/healthz` and `/source`. TCP responses carry
+  `Alt-Svc: h3=":443"; ma=86400`, except a WebSocket upgrade's.
+- The uplink (`ftw-cloud-box/1`) and the app session (`ftw-cloud-app/1`),
+  with the WebSockets' handlers and messages (srcfl/ftw
+  `docs/protocol-cloud.md`, "QUIC"). Boxes and phones race QUIC against the
+  WebSocket and fall back to it on their own, so shutting UDP never cuts
+  anyone off; it only makes them slower.
 
 What it buys, measured by `scripts/netem-once.sh` (a watch's one-shot read,
 40 ms round trip): a read on a new connection takes two round trips instead
@@ -69,11 +75,12 @@ Nothing restarts on it; it is there to look at after a deploy.
 
 ## Turning it off
 
-Without a release: on the host, add `FTW_CLOUD_H3=false` to
-`/etc/ftw-cloud/compose.env` and run `systemctl restart ftw-cloud.service`.
-`ftw-cloud-deploy` keeps that line. The service then stops listening on UDP
-and stops sending `Alt-Svc`. Wait a day, the `ma` in `Alt-Svc`, before you
-shut UDP 443 in the security group:
+Without a release: on the host, add `FTW_CLOUD_H3=false`,
+`FTW_CLOUD_QUIC_SESSIONS=false` or both to `/etc/ftw-cloud/compose.env` and
+run `systemctl restart ftw-cloud.service`. `ftw-cloud-deploy` keeps those
+lines. With both off the service stops listening on UDP and stops sending
+`Alt-Svc`. Wait a day, the `ma` in `Alt-Svc`, before you shut UDP 443 in
+the security group:
 
 ```bash
 aws ec2 revoke-security-group-ingress --region eu-north-1 --group-id sg-098992ad382ecf94a \
@@ -93,12 +100,22 @@ aws ec2 revoke-security-group-ingress --region eu-north-1 --group-id sg-098992ad
   knows at once, not after its idle timeout
   (`TestARestartResetsTheOldProcesssConnections`). Keep the file with the
   rest of `/state`; a new key only costs one idle timeout per client.
+  On SIGTERM the door stops taking connections, and on the way out it ends
+  each live session with 1001, going away
+  (`TestCloseEndsEachSessionWithGoingAway`). The drain, which closes them
+  first with 4503 and a spread, goes between the two.
 - **Floods.** Past 200 new connections a second the door asks each to prove
   its address with a Retry, which costs a real client one round trip and a
-  spoofed one everything. A handshake gets 5 s. A UDP port that will not
-  bind turns HTTP/3 off and leaves TCP serving.
-- **No 0-RTT.** A request in early data can be replayed, so the door refuses
-  early data until a route says which of its requests survive a replay.
+  spoofed one everything. A handshake gets 5 s, and the door holds at most
+  a set number of handshakes and connections at once. A UDP port that will
+  not bind turns QUIC off and leaves TCP serving.
+- **0-RTT.** The sessions take early data under the protocol's rules: a
+  box's target may get its `challenge` before the handshake ends, and a
+  phone's session waits for the handshake, so a replayed flight changes
+  nothing. A ticket allows early data for a day. HTTP/3 requests in early
+  data wait for the handshake. A restart draws new ticket keys.
+- **A UDP load balancer** in front would have to keep each QUIC connection
+  on one host by its connection id; with one host there is none.
 - **No DNS record for HTTP/3.** An `HTTPS` record with `alpn=h3` would tell
   every client, the sockets' too, to try QUIC on the first connection.
   `Alt-Svc` reaches only clients of the plain routes.

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/fxamacker/cbor/v2 v2.9.4
 	github.com/gorilla/websocket v1.5.3
 	github.com/quic-go/quic-go v0.63.0
-	github.com/srcfl/ftw/go/protocol v0.0.0-20261015212100-f024251f3248
+	github.com/srcfl/ftw/go/protocol v0.0.0-20261016050141-c49da79dfc79
 	golang.org/x/crypto v0.57.0
 	golang.org/x/sys v0.48.0
 	modernc.org/sqlite v1.59.0

go.sum

@@ -54,6 +54,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/srcfl/ftw/go/protocol v0.0.0-20261015212100-f024251f3248 h1:56VRLOp7pcN9FyU6ncNri5y1nYCRlG6zD+ZKko46HTA=
 github.com/srcfl/ftw/go/protocol v0.0.0-20261015212100-f024251f3248/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU=
+github.com/srcfl/ftw/go/protocol v0.0.0-20261016050141-c49da79dfc79 h1:BnMVCZe7r6zmaCQeiImMeZ5AlJOqL52ok8ef8s39UKo=
+github.com/srcfl/ftw/go/protocol v0.0.0-20261016050141-c49da79dfc79/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU=
 github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
 github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=

internal/appsession/appsession.go

@@ -31,6 +31,7 @@ import (
 	"github.com/srcfl/ftw-cloud/internal/registry"
 	"github.com/srcfl/ftw-cloud/internal/shard"
 	"github.com/srcfl/ftw-cloud/internal/store"
+	"github.com/srcfl/ftw-cloud/internal/wire"
 	"github.com/srcfl/ftw-cloud/internal/wsclose"
 	"github.com/srcfl/ftw/go/protocol/appwire"
 	"github.com/srcfl/ftw/go/protocol/cloudwire"
@@ -200,9 +201,19 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	if err != nil {
 		return
 	}
+	s.ServeConn(conn, home)
+}
+
+// ServeConn runs one phone's session on conn, a WebSocket or QUIC (package
+// quicdoor), for home. It closes conn.
+func (s *Server) ServeConn(conn wire.Conn, home string) {
 	defer conn.Close()
 	appSockets.Add(1)
 	defer appSockets.Add(-1)
+	if !cloudwire.ValidHome(home) {
+		_ = conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(cloudwire.CloseProtocol, ""), time.Now().Add(time.Second))
+		return
+	}
 	conn.SetReadLimit(maxPhoneMessage)
 	sock := s.sockets.Add(conn)
 	defer s.sockets.Remove(sock)
@@ -269,7 +280,7 @@ var errProtocol = errors.New("appsession: the phone broke the protocol")
 // serve runs one session and returns the close code to end it with. Once the
 // session's reader runs, reading is set to a channel that closes when it
 // stops, and only that reader may read the socket.
-func (s *Server) serve(conn *websocket.Conn, home string, reading *<-chan struct{}) (int, error) {
+func (s *Server) serve(conn wire.Conn, home string, reading *<-chan struct{}) (int, error) {
 	start := time.Now()
 	_ = conn.SetReadDeadline(time.Now().Add(handshakeTimeout))
 	kind, msg1, err := conn.ReadMessage()
@@ -445,7 +456,7 @@ func (s *Server) watchOnce(home string) {
 // phoneConn is one phone's session after the handshake.
 type phoneConn struct {
 	s    *Server
-	conn *websocket.Conn
+	conn wire.Conn
 	tr   *appwire.Transport
 	dev  *store.Handle
 	home string
@@ -476,6 +487,11 @@ type phoneConn struct {
 	tunMu  sync.Mutex
 	tunSid uint32
 	hasTun bool
+
+	// heard is set by the phone's first transport message, which proves it
+	// holds message 2. Until then everything goes on the control stream,
+	// behind message 2: over QUIC a datagram could overtake it.
+	heard atomic.Bool
 }
 
 func (c *phoneConn) setLooking(on bool) {
@@ -503,26 +519,43 @@ func (c *phoneConn) stopLooking() {
 
 // write sends one WebSocket message. Only the session's own goroutine
 // writes; pings go through WriteControl, which may run beside it.
-func (c *phoneConn) write(msg []byte) error {
+func (c *phoneConn) write(msg []byte) error { return c.writeAs(websocket.BinaryMessage, msg) }
+
+// writeAs sends one message the way kind says, over QUIC; a WebSocket
+// carries every kind as a binary message.
+func (c *phoneConn) writeAs(kind int, msg []byte) error {
 	if err := c.conn.SetWriteDeadline(time.Now().Add(writeTimeout)); err != nil {
 		return err
 	}
-	return c.conn.WriteMessage(websocket.BinaryMessage, msg)
+	return c.conn.WriteMessage(wire.Kind(c.conn, kind), msg)
 }
 
 // forward sends one box envelope as the box sent it. An envelope too large
 // for a Noise message cannot reach a phone; it is dropped, and the phone sees
 // the gap through the next hb.
-func (c *phoneConn) forward(env []byte) error {
+func (c *phoneConn) forward(env []byte) error { return c.forwardAs(websocket.BinaryMessage, env) }
+
+// forwardAs is forward over the path kind names. Over QUIC an hb, or a live
+// without its source table, goes as a datagram: the newest wins, and a
+// lost one shows as a gap in the phone's freshness rules.
+func (c *phoneConn) forwardAs(kind int, env []byte) error {
 	frame, err := cloudwire.EncodeFrame(env)
 	if err != nil || len(frame)+appwire.TransportOverhead > maxNoiseMessage {
 		return nil
 	}
+	switch {
+	case !c.heard.Load():
+		kind = websocket.BinaryMessage
+	case kind == websocket.BinaryMessage && wire.Kind(c.conn, wire.Datagram) == wire.Datagram:
+		if e, err := cloudwire.DecodeEnvelope(env); err == nil && cloudwire.Datagram(e) {
+			kind = wire.Datagram
+		}
+	}
 	msg, err := c.tr.Encrypt(frame)
 	if err != nil {
 		return err
 	}
-	return c.write(msg)
+	return c.writeAs(kind, msg)
 }
 
 // run forwards until the store ends the session, the phone leaves or breaks
@@ -557,7 +590,11 @@ func (c *phoneConn) run(sess *store.Session) (int, error) {
 			if err != nil {
 				return 0, err
 			}
-			if err := c.forward(env); err != nil {
+			kind := websocket.BinaryMessage
+			if t.Bulk {
+				kind = wire.Bulk
+			}
+			if err := c.forwardAs(kind, env); err != nil {
 				return 0, err
 			}
 			tunUp.Observe(t.Waited())
@@ -598,6 +635,7 @@ func (c *phoneConn) readLoop() (int, error) {
 		if err != nil {
 			return cloudwire.CloseProtocol, err
 		}
+		c.heard.Store(true)
 		if code, err := c.apply(frame); err != nil {
 			return code, err
 		}