From 8239ba74bbc1d8f21df76da547d461b93c67968e Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Thu, 19 Nov 2026 03:23:14 -0500 Subject: [PATCH 1/6] fix(containerlab): Pin Cilium's devices to eth0 and the bonds With kube-proxy replacement and no device list, Cilium adopts every interface with a global address. A galactic-tap host TAP carries one, the guest's gateway, so Cilium registered the tenant address as a node address and attached its programs to the TAP with tcx, which deleted every legacy tc filter there: galactic-cni's egress chain and tp_deliver. The TAP then forwarded without its traffic policy. Pin the devices as production deployments pin them to their uplinks. In a Kind node that is eth0 and the bonds; eth+ would also match the bond members, whose legacy usid_ingress filters Cilium would delete the same way. --- deploy/containerlab/scripts/deploy-cni.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/deploy/containerlab/scripts/deploy-cni.sh b/deploy/containerlab/scripts/deploy-cni.sh index 2b9f0382..02872234 100755 --- a/deploy/containerlab/scripts/deploy-cni.sh +++ b/deploy/containerlab/scripts/deploy-cni.sh @@ -60,6 +60,16 @@ for site in dfw sjc iad; do # node silently kept serving the old config instead of applying the fix # above. `set -e` inside the inner script plus install-or-upgrade makes # reruns actually idempotent. + # devices pins the interfaces Cilium attaches its host programs to, as + # production deployments pin them to their uplink bonds and primary + # Ethernet interfaces. Left to + # auto-detection, Cilium also adopts every interface with a global address: + # a galactic-tap host TAP (it carries the guest's gateway), galactic-vip0. + # Its tcx attach then deletes every legacy tc filter there, including + # galactic-cni's egress chain and tp_deliver (docs/handoffs.md). Here the + # pin is eth0 and the bonds: eth+ would also match the bonds' members, + # eth1 to eth4, whose legacy tc filters (usid_ingress) Cilium would delete + # the same way. # k8sServiceHost/k8sServicePort point the agent at the API server directly. # Without them it uses the kubernetes Service, which Cilium itself # load-balances, so an API server restart can leave the site unable to @@ -79,6 +89,7 @@ for site in dfw sjc iad; do --set ipam.operator.clusterPoolIPv6MaskSize=${CILIUM_POD_CIDR_V6_MASK_SIZE} \ --set cni.exclusive=false \ --set kubeProxyReplacement=true \ + --set devices='eth0 bond+' \ --set k8sServiceHost=${node} \ --set k8sServicePort=6443 \ --set tunnelProtocol=vxlan \ From d0ead1301973c3e3570c32e62227c1383de81b6f Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Thu, 19 Nov 2026 03:23:14 -0500 Subject: [PATCH 2/6] test(handoffs): Add the Local handoff prototypes and handoff-probe tests/handoffs/probe holds prototype programs for the three Local handoffs the Worker qualification plan lists: public receive XDP_PASS into tc and a tenant TAP, tc into an internal veth pair with XDP on its peer and the reverse tc receive, and a guard for a pod-to-host routed link. handoff.c includes the uSID datapath source, so its policy decisions run the product's tp_capture and tp_decide; the loader swaps every uSID map for galactic-cni's pin and refuses to create one. handoff-probe places the programs on a scratch rig with the existing loaders (xdpdispatch, natattach, attach, egressroutemap) and removes them. build:ebpf generates the probe, and the unit and root test runs cover it; a root test loads every program through the verifier against freshly pinned uSID maps. --- .gitignore | 4 + Taskfile.yaml | 3 +- scripts/ci.sh | 4 +- tests/handoffs/cmd/handoff-probe/main.go | 793 +++++++++++++++++++++++ tests/handoffs/probe/doc.go | 15 + tests/handoffs/probe/handoff.c | 624 ++++++++++++++++++ tests/handoffs/probe/load.go | 179 +++++ tests/handoffs/probe/load_root_test.go | 96 +++ tests/handoffs/probe/stats_test.go | 36 + 9 files changed, 1751 insertions(+), 3 deletions(-) create mode 100644 tests/handoffs/cmd/handoff-probe/main.go create mode 100644 tests/handoffs/probe/doc.go create mode 100644 tests/handoffs/probe/handoff.c create mode 100644 tests/handoffs/probe/load.go create mode 100644 tests/handoffs/probe/load_root_test.go create mode 100644 tests/handoffs/probe/stats_test.go diff --git a/.gitignore b/.gitignore index a85008cb..a5348439 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,10 @@ internal/plumbing/ebpf/xmitdrops/xmitdrops_bpfel.go internal/plumbing/ebpf/xmitdrops/xmitdrops_bpfel.o internal/plumbing/ebpf/xmitdrops/xmitdrops_bpfeb.go internal/plumbing/ebpf/xmitdrops/xmitdrops_bpfeb.o +tests/handoffs/probe/handoff_bpfel.go +tests/handoffs/probe/handoff_bpfel.o +tests/handoffs/probe/handoff_bpfeb.go +tests/handoffs/probe/handoff_bpfeb.o # Same convention as above, for the containerlab-only pass-through XDP # program (deploy/containerlab/Taskfile.yaml's build:lab-xdp-passthrough, diff --git a/Taskfile.yaml b/Taskfile.yaml index 14a20f3c..46982680 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -122,7 +122,7 @@ tasks: cmds: - python3 hack/docs/generate_api.py --check build:ebpf: - desc: Regenerate the eBPF uSID, edge gateway DSR, NAT egress, XDP dispatcher and XDP transmit drop programs (requires clang) + desc: Regenerate the eBPF uSID, edge gateway DSR, NAT egress, XDP dispatcher, XDP transmit drop and Local handoff probe programs (requires clang) run: once deps: [require-clang] cmds: @@ -131,6 +131,7 @@ tasks: - go generate ./internal/plumbing/ebpf/natprog/... - go generate ./internal/plumbing/ebpf/xdpdispatch/... - go generate ./internal/plumbing/ebpf/xmitdrops/... + - go generate ./tests/handoffs/probe/... require-clang: internal: true run: once diff --git a/scripts/ci.sh b/scripts/ci.sh index 4bdfe3a7..d7a12523 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -6,7 +6,7 @@ COMMAND="${1:-}" case "$COMMAND" in unittest) echo "--- Running Go unit tests" - go test -v -race -coverprofile=coverage.out ./api/... ./cmd/... ./internal/... ./tests/edgeconformance/edgeprobe/... + go test -v -race -coverprofile=coverage.out ./api/... ./cmd/... ./internal/... ./tests/edgeconformance/edgeprobe/... ./tests/handoffs/... ;; unittest-root) @@ -24,7 +24,7 @@ case "$COMMAND" in # differently once actually run as root. echo "--- Discovering root-gated packages" mapfile -t pkgs < <( - grep -rl 'requireRoot(t)' --include='*_test.go' ./cmd ./internal 2>/dev/null \ + grep -rl 'requireRoot(t)' --include='*_test.go' ./cmd ./internal ./tests/handoffs 2>/dev/null \ | xargs -n1 dirname | sort -u | sed 's#^\./#./#' ) if [ "${#pkgs[@]}" -eq 0 ]; then diff --git a/tests/handoffs/cmd/handoff-probe/main.go b/tests/handoffs/cmd/handoff-probe/main.go new file mode 100644 index 00000000..6199b0a8 --- /dev/null +++ b/tests/handoffs/cmd/handoff-probe/main.go @@ -0,0 +1,793 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Command handoff-probe places the #1444 prototype handoff programs on a +// scratch rig on a lab node, and removes them. It runs in the node's own +// network and mount namespaces (docker exec), next to galactic-cni, whose +// pinned maps the prototypes' policy decisions read. +// +// It uses the product loaders as they are: xdpdispatch for the dispatcher +// slot, natattach (xdpattach) for XDP on the NAT side of the internal veth +// pair, attach for tc ingress, and egressroutemap for the Local pass-through +// entries. It writes nothing outside its own pin directory except tc filters +// and XDP links on the interfaces its configuration names, and pass-through +// entries under its own owner, all of which teardown removes. +// +// Usage: +// +// handoff-probe check GALACTIC_PIN_DIR +// handoff-probe setup CONFIG +// handoff-probe renew CONFIG +// handoff-probe counters CONFIG +// handoff-probe teardown CONFIG +// handoff-probe leftovers VRF... +// handoff-probe mode IFACE... +// +// check loads the programs against the running datapath and removes what it +// loaded; it and setup exit 3 when that datapath does not match this +// checkout, so a caller can stop at that precondition. leftovers lists +// anything a run left behind: the probe's pin directory, tc filters and its +// pass-through entries in the named VRFs, and exits 1 when there is any. +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "strings" + "time" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/link" + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" + + "go.datum.net/galactic/internal/plumbing/ebpf/attach" + "go.datum.net/galactic/internal/plumbing/ebpf/egressroutemap" + "go.datum.net/galactic/internal/plumbing/ebpf/natattach" + "go.datum.net/galactic/internal/plumbing/ebpf/xdpdispatch" + "go.datum.net/galactic/internal/plumbing/ebpf/xdpmode" + "go.datum.net/galactic/tests/handoffs/probe" +) + +// passThroughOwner marks the probe's egress_route_table pass-through entries, +// so teardown removes its own and never another writer's. +const passThroughOwner egressroutemap.PassThroughOwner = 0x48 + +// tapFilterName names the probe's filter on a tenant TAP's tc ingress, after +// galactic-cni's own two. +const tapFilterName = "galactic_handoff_probe" + +// exitMismatch is setup's exit status when the datapath differs. +const exitMismatch = 3 + +// Config is what the rig script writes for one rig cell. +type Config struct { + PinDir string `json:"pinDir"` + GalacticPinDir string `json:"galacticPinDir"` + Mode string `json:"mode"` // native or generic + // PeerMode is the mode of XDP on the NAT side of the internal veth + // pairs, when it differs from Mode. + PeerMode string `json:"peerMode,omitempty"` + // XDP lists the receiving interfaces the dispatcher is attached to; TC + // the ones hp_tc_tenant_rx is attached to. They differ only for a VLAN: + // XDP on its lower device, tc on the VLAN device. + XDP []string `json:"xdp"` + TC []string `json:"tc"` + VLANID uint32 `json:"vlanID,omitempty"` + VIPs []VIP `json:"vips"` + Taps []Tap `json:"taps"` + Pairs []Pair `json:"pairs"` + Guards []Guard `json:"guards"` + PassThrough []PassThrough `json:"passThrough"` +} + +// VIP is a public address delivered to a tenant TAP (local.tap). +type VIP struct { + VIP string `json:"vip"` + Tap string `json:"tap"` + Guest string `json:"guest"` + // Decide makes the tc program decide policy on the PublicService path. + // False leaves it to tp_deliver. + Decide bool `json:"decide"` +} + +// Tap is a tenant TAP whose guest replies and NAT flows the probe sends on. +type Tap struct { + Tap string `json:"tap"` + DUT string `json:"dut"` + Pair string `json:"pair"` // tenant end of the internal veth pair + Peer string `json:"peer"` // NAT end + VIP string `json:"vip"` + ClientPrefix string `json:"clientPrefix"` // a /64 +} + +// Pair is one internal veth pair (local.veth_pair). +type Pair struct { + Pair string `json:"pair"` + Peer string `json:"peer"` + Tap string `json:"tap"` + Guest string `json:"guest"` + Shard string `json:"shard"` + // Redirect makes the peer's XDP forward with XDP_REDIRECT; false passes + // the translated packet to the kernel. + Redirect bool `json:"redirect"` + // VLAN, when set, is an egress VLAN device. A redirect toward it pushes + // the tag in XDP and goes to its lower device instead. + VLAN *VLANEgress `json:"vlan,omitempty"` +} + +// VLANEgress names a VLAN device, its lower device and its VLAN ID. +type VLANEgress struct { + Device string `json:"device"` + Lower string `json:"lower"` + VID uint32 `json:"vid"` +} + +// Guard is the host end of a routed link and the /127 it carries. +type Guard struct { + Iface string `json:"iface"` + Prefix string `json:"prefix"` +} + +// PassThrough is a Local destination in a tenant VRF that usid_egress must +// leave to the kernel rather than send to a NAT shard. +type PassThrough struct { + VRF string `json:"vrf"` + Prefix string `json:"prefix"` +} + +func main() { + if len(os.Args) < 3 { + fmt.Fprintln(os.Stderr, "usage: handoff-probe setup|renew|counters|teardown CONFIG | mode IFACE...") + os.Exit(2) + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + var err error + switch os.Args[1] { + case "mode": + err = selectMode(os.Args[2:]) + case "check": + err = check(os.Args[2]) + case "leftovers": + err = leftovers(os.Args[2:]) + case "setup", "renew", "counters", "teardown": + var cfg Config + if cfg, err = readConfig(os.Args[2]); err != nil { + break + } + switch os.Args[1] { + case "setup": + err = setup(ctx, cfg) + case "renew": + err = renew(ctx, cfg) + case "counters": + err = counters(cfg) + case "teardown": + err = teardown(ctx, cfg) + } + default: + err = fmt.Errorf("unknown command %q", os.Args[1]) + } + if err != nil { + fmt.Fprintln(os.Stderr, "handoff-probe:", err) + if errors.Is(err, probe.ErrDatapathMismatch) { + os.Exit(exitMismatch) + } + os.Exit(1) + } +} + +// defaultPinDir is where the lab scripts put the probe's pins. +const defaultPinDir = "/sys/fs/bpf/galactic-handoff-proof" + +func check(galacticPinDir string) error { + dir := defaultPinDir + "-check" + objs, err := probe.Load(galacticPinDir, dir) + if objs != nil { + _ = objs.Close() + } + if rmErr := os.RemoveAll(dir); err == nil { + err = rmErr + } + if err == nil { + fmt.Println("the prototype programs load against the running datapath") + } + return err +} + +func leftovers(vrfs []string) error { + var found []string + if _, err := os.Stat(defaultPinDir); err == nil { + found = append(found, "pin directory "+defaultPinDir) + } + links, err := netlink.LinkList() + if err != nil { + return err + } + for _, l := range links { + name := l.Attrs().Name + if strings.HasPrefix(name, "hp-") || strings.HasPrefix(name, "hpn") || strings.HasPrefix(name, "hpr") || + strings.HasPrefix(name, "hpv") { + found = append(found, "interface "+name) + } + filters, err := netlink.FilterList(l, netlink.HANDLE_MIN_INGRESS) + if err != nil { + continue + } + for _, f := range filters { + if bf, ok := f.(*netlink.BpfFilter); ok && bf.Name == tapFilterName { + found = append(found, "tc filter "+tapFilterName+" on "+name) + } + } + } + if len(vrfs) > 0 { + table, closer, err := egressroutemap.OpenPinnedEgressRouteTable(attach.PinDir) + if err != nil { + return err + } + defer closer.Close() //nolint:errcheck // our own descriptor + for _, name := range vrfs { + l, err := netlink.LinkByName(name) + if err != nil { + continue + } + vrf, ok := l.(*netlink.Vrf) + if !ok { + continue + } + entries, err := table.Entries(vrf.Table) + if err != nil { + return err + } + for prefix, e := range entries { + if e.PassThrough && e.Owner == passThroughOwner { + found = append(found, fmt.Sprintf("pass-through %s in %s", prefix, name)) + } + } + } + } + for _, f := range found { + fmt.Println(f) + } + if len(found) > 0 { + return fmt.Errorf("%d leftovers", len(found)) + } + fmt.Println("no leftovers") + return nil +} + +func readConfig(path string) (Config, error) { + var cfg Config + raw, err := os.ReadFile(path) + if err != nil { + return cfg, err + } + if err := json.Unmarshal(raw, &cfg); err != nil { + return cfg, fmt.Errorf("parse %s: %w", path, err) + } + if cfg.PinDir == "" || cfg.GalacticPinDir == "" { + return cfg, errors.New("config needs pinDir and galacticPinDir") + } + return cfg, nil +} + +func (c Config) mode() (xdpmode.Mode, error) { return parseMode(c.Mode) } + +func (c Config) peerMode() (xdpmode.Mode, error) { + if c.PeerMode == "" { + return c.mode() + } + return parseMode(c.PeerMode) +} + +func parseMode(m string) (xdpmode.Mode, error) { + switch m { + case "native": + return xdpmode.Native, nil + case "generic": + return xdpmode.Generic, nil + } + return 0, fmt.Errorf("mode %q is neither native nor generic", m) +} + +func (c Config) mapDir() string { return filepath.Join(c.PinDir, "maps") } +func (c Config) dispatchDir() string { return filepath.Join(c.PinDir, "dispatch") } +func (c Config) linkDir() string { return filepath.Join(c.PinDir, "links") } + +// selectMode prints what xdpmode.Select decides for the interfaces, for the +// run's record. +func selectMode(ifaces []string) error { + d, err := xdpmode.Select(ifaces...) + if err != nil { + return err + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{ + "mode": d.Mode.String(), "reason": d.Reason, "drivers": d.Drivers, "skipped": d.Skipped, + }) +} + +func setup(ctx context.Context, cfg Config) error { + mode, err := cfg.mode() + if err != nil { + return err + } + objs, err := probe.Load(cfg.GalacticPinDir, cfg.mapDir()) + if err != nil { + return err + } + defer objs.Close() //nolint:errcheck // pins and filters outlive this process + + if err := writeRows(objs, cfg); err != nil { + return err + } + if err := passThrough(cfg, true); err != nil { + return err + } + + // The receiving device: the dispatcher's slot program, then tc. + if len(cfg.XDP) > 0 { + if err := fillDispatcher(ctx, cfg, mode, objs.Programs.XDPPublic); err != nil { + return err + } + if err := attach.Attach(objs.Programs.TCTenantRx, cfg.TC); err != nil { + return fmt.Errorf("attach hp_tc_tenant_rx: %w", err) + } + } + for _, t := range cfg.Taps { + if err := attachTapFilter(objs.Programs.TCTapRx, t.Tap); err != nil { + return err + } + } + if len(cfg.Pairs) > 0 { + var peers, pairs []string + for _, p := range cfg.Pairs { + peers = append(peers, p.Peer) + pairs = append(pairs, p.Pair) + } + peerMode, err := cfg.peerMode() + if err != nil { + return err + } + links, err := natattach.Attach(objs.Programs.XDPPairPeer, peers, peerMode) + if err != nil { + return fmt.Errorf("attach hp_xdp_pair_peer: %w", err) + } + if err := os.MkdirAll(cfg.linkDir(), 0o755); err != nil { + return err + } + for i, l := range links { + if err := l.Pin(filepath.Join(cfg.linkDir(), "peer-"+peers[i])); err != nil { + return fmt.Errorf("pin XDP link of %s: %w", peers[i], err) + } + _ = l.Close() + } + if err := attach.Attach(objs.Programs.TCPairRx, pairs); err != nil { + return fmt.Errorf("attach hp_tc_pair_rx: %w", err) + } + } + if len(cfg.Guards) > 0 { + var names []string + for _, g := range cfg.Guards { + names = append(names, g.Iface) + } + if err := attach.Attach(objs.Programs.TCLinkGuard, names); err != nil { + return fmt.Errorf("attach hp_tc_link_guard: %w", err) + } + } + return nil +} + +func fillDispatcher(ctx context.Context, cfg Config, mode xdpmode.Mode, prog *ebpf.Program) error { + d, err := xdpdispatch.Open(ctx, cfg.dispatchDir(), mode) + if err != nil { + return fmt.Errorf("open dispatcher: %w", err) + } + defer d.Close() //nolint:errcheck // our own descriptors + + for _, name := range cfg.XDP { + l, err := netlink.LinkByName(name) + if err != nil { + return err + } + locked, err := d.Lock(ctx) + if err != nil { + return err + } + bounced, err := locked.EnsureLink(l.Attrs().Index) + if err == nil { + err = locked.SetRole(l.Attrs().Index, xdpdispatch.RolePublicLB) + } + if err == nil && bounced { + err = waitBondMember(name, 30*time.Second) + } + locked.Unlock() + if err != nil { + return fmt.Errorf("dispatcher on %s in %s mode: %w", name, mode, err) + } + } + locked, err := d.Lock(ctx) + if err != nil { + return err + } + defer locked.Unlock() + return locked.Fill(xdpdispatch.SlotGatewayLB, prog) +} + +// waitBondMember waits until a bond member that an XDP attach reset is back +// in its aggregate. Interfaces that are not bond members return at once. +func waitBondMember(name string, timeout time.Duration) error { + const distributing = 0x20 // LACP actor state bit + deadline := time.Now().Add(timeout) + for { + l, err := netlink.LinkByName(name) + if err != nil { + return err + } + slave, ok := l.Attrs().Slave.(*netlink.BondSlave) + if !ok { + return nil + } + if slave.MiiStatus == netlink.BondLinkUp && slave.AdActorOperPortState&distributing != 0 { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("bond member %s did not rejoin its aggregate within %s", name, timeout) + } + time.Sleep(200 * time.Millisecond) + } +} + +// attachTapFilter adds hp_tc_tap_rx to a tenant TAP's tc ingress at the +// priority after galactic-cni's last filter there, so it sees only what +// usid_egress passed on. +func attachTapFilter(prog *ebpf.Program, tap string) error { + l, err := netlink.LinkByName(tap) + if err != nil { + return err + } + filters, err := netlink.FilterList(l, netlink.HANDLE_MIN_INGRESS) + if err != nil { + return fmt.Errorf("list tc ingress filters on %s: %w", tap, err) + } + var last uint16 + for _, f := range filters { + if bf, ok := f.(*netlink.BpfFilter); ok && bf.Name != tapFilterName && bf.Priority > last { + last = bf.Priority + } + } + if last == 0 { + return fmt.Errorf("%s carries no galactic-cni tc ingress filter, so it is not a managed tenant TAP", tap) + } + filter := &netlink.BpfFilter{ + FilterAttrs: netlink.FilterAttrs{ + LinkIndex: l.Attrs().Index, + Parent: netlink.HANDLE_MIN_INGRESS, + Handle: netlink.MakeHandle(0, 1), + Protocol: unix.ETH_P_ALL, + Priority: last + 1, + }, + Fd: prog.FD(), + Name: tapFilterName, + DirectAction: true, + } + if err := netlink.FilterReplace(filter); err != nil { + return fmt.Errorf("attach hp_tc_tap_rx to %s: %w", tap, err) + } + return nil +} + +func ifindex(name string) (uint32, net.HardwareAddr, error) { + l, err := netlink.LinkByName(name) + if err != nil { + return 0, nil, err + } + return uint32(l.Attrs().Index), l.Attrs().HardwareAddr, nil +} + +func addr16(s string) ([16]byte, error) { + var out [16]byte + ip := net.ParseIP(s) + if ip == nil || ip.To4() != nil { + return out, fmt.Errorf("%q is not an IPv6 address", s) + } + copy(out[:], ip.To16()) + return out, nil +} + +func writeRows(objs *probe.Objects, cfg Config) error { + m := objs.Maps + if err := m.Cfg.Put(uint32(0), probe.HandoffHpCfgValue{VlanId: cfg.VLANID}); err != nil { + return err + } + for _, write := range []func(probe.Maps, Config) error{writeVIPs, writeTaps, writePairs, writeGuards} { + if err := write(m, cfg); err != nil { + return err + } + } + return nil +} + +func writeVIPs(m probe.Maps, cfg Config) error { + for _, v := range cfg.VIPs { + vip, err := addr16(v.VIP) + if err != nil { + return err + } + guest, err := addr16(v.Guest) + if err != nil { + return err + } + tap, _, err := ifindex(v.Tap) + if err != nil { + return err + } + val := probe.HandoffHpVipValue{TapIfindex: tap, Guest: guest} + if v.Decide { + val.Decide = 1 + } + if err := m.VIP.Put(probe.HandoffHpIn6{A: vip}, val); err != nil { + return fmt.Errorf("hp_vip %s: %w", v.VIP, err) + } + } + return nil +} + +func writeTaps(m probe.Maps, cfg Config) error { + for _, t := range cfg.Taps { + tap, _, err := ifindex(t.Tap) + if err != nil { + return err + } + dut, _, err := ifindex(t.DUT) + if err != nil { + return err + } + val := probe.HandoffHpTapValue{DutIfindex: dut} + if val.Vip, err = addr16(t.VIP); err != nil { + return err + } + _, client, err := net.ParseCIDR(t.ClientPrefix) + if err != nil { + return err + } + copy(val.ClientPrefix[:], client.IP.To16()[:8]) + if t.Pair != "" { + pair, pairMAC, err := ifindex(t.Pair) + if err != nil { + return err + } + _, peerMAC, err := ifindex(t.Peer) + if err != nil { + return err + } + val.PairIfindex = pair + copy(val.PairSmac[:], pairMAC) + copy(val.PairDmac[:], peerMAC) + } + if err := m.Tap.Put(tap, val); err != nil { + return fmt.Errorf("hp_tap %s: %w", t.Tap, err) + } + } + return nil +} + +func writePairs(m probe.Maps, cfg Config) error { + for _, p := range cfg.Pairs { + pair, pairMAC, err := ifindex(p.Pair) + if err != nil { + return err + } + peer, peerMAC, err := ifindex(p.Peer) + if err != nil { + return err + } + tap, _, err := ifindex(p.Tap) + if err != nil { + return err + } + guest, err := addr16(p.Guest) + if err != nil { + return err + } + shard, err := addr16(p.Shard) + if err != nil { + return err + } + snat := probe.HandoffHpSnatValue{Guest: guest, Shard: shard} + if p.Redirect { + snat.Redirect = 1 + } + if p.VLAN != nil { + if snat.VlanIfindex, _, err = ifindex(p.VLAN.Device); err != nil { + return err + } + if snat.LowerIfindex, _, err = ifindex(p.VLAN.Lower); err != nil { + return err + } + snat.Vid = p.VLAN.VID + } + if err := m.SNAT.Put(peer, snat); err != nil { + return err + } + sv := probe.HandoffHpShardValue{PeerIfindex: peer, Guest: guest} + copy(sv.Smac[:], peerMAC) + copy(sv.Dmac[:], pairMAC) + if err := m.Shard.Put(probe.HandoffHpIn6{A: shard}, sv); err != nil { + return err + } + if err := m.PairRx.Put(pair, probe.HandoffHpPairRxValue{TapIfindex: tap, Guest: guest}); err != nil { + return err + } + } + return nil +} + +func writeGuards(m probe.Maps, cfg Config) error { + for _, g := range cfg.Guards { + idx, _, err := ifindex(g.Iface) + if err != nil { + return err + } + _, prefix, err := net.ParseCIDR(g.Prefix) + if err != nil { + return err + } + if ones, _ := prefix.Mask.Size(); ones != 127 { + return fmt.Errorf("guard prefix %s is not a /127", g.Prefix) + } + var v probe.HandoffHpGuardValue + copy(v.Prefix[:], prefix.IP.To16()) + if err := m.Guard.Put(idx, v); err != nil { + return err + } + } + return nil +} + +// passThrough adds or removes the probe's Local pass-through entries. +func passThrough(cfg Config, add bool) error { + if len(cfg.PassThrough) == 0 { + return nil + } + table, closer, err := egressroutemap.OpenPinnedEgressRouteTable(cfg.GalacticPinDir) + if err != nil { + return err + } + defer closer.Close() //nolint:errcheck // our own descriptor + + var errs []error + for _, p := range cfg.PassThrough { + l, err := netlink.LinkByName(p.VRF) + if err != nil { + if !add { + continue + } + return err + } + vrf, ok := l.(*netlink.Vrf) + if !ok { + return fmt.Errorf("%s is not a VRF", p.VRF) + } + _, prefix, err := net.ParseCIDR(p.Prefix) + if err != nil { + return err + } + if add { + if err := table.RegisterPassThroughAs(vrf.Table, prefix, passThroughOwner); err != nil { + return err + } + continue + } + if _, err := table.UnregisterPassThroughAs(vrf.Table, prefix, passThroughOwner); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +func renew(ctx context.Context, cfg Config) error { + d, err := xdpdispatch.OpenForSlots(ctx, cfg.dispatchDir()) + if err != nil { + return err + } + defer d.Close() //nolint:errcheck // our own descriptors + return d.Renew(xdpdispatch.SlotGatewayLB) +} + +func counters(cfg Config) error { + m, err := ebpf.LoadPinnedMap(filepath.Join(cfg.mapDir(), "hp_stats"), nil) + if err != nil { + return err + } + defer m.Close() //nolint:errcheck // our own descriptor + stats, err := probe.Stats(m) + if err != nil { + return err + } + return json.NewEncoder(os.Stdout).Encode(stats) +} + +// teardown removes everything setup placed. It is safe to run on a partial +// setup and to run twice. +func teardown(ctx context.Context, cfg Config) error { + var errs []error + for _, t := range cfg.Taps { + errs = append(errs, removeFilter(t.Tap, tapFilterName)) + } + tc := make([]string, 0, len(cfg.TC)+len(cfg.Pairs)+len(cfg.Guards)) + tc = append(tc, cfg.TC...) + for _, p := range cfg.Pairs { + tc = append(tc, p.Pair) + } + for _, g := range cfg.Guards { + tc = append(tc, g.Iface) + } + if len(tc) > 0 { + errs = append(errs, attach.Detach(tc)) + } + for _, p := range cfg.Pairs { + path := filepath.Join(cfg.linkDir(), "peer-"+p.Peer) + if l, err := link.LoadPinnedLink(path, nil); err == nil { + errs = append(errs, l.Unpin(), l.Close()) + } + } + if _, err := os.Stat(cfg.dispatchDir()); err == nil { + errs = append(errs, releaseDispatcher(ctx, cfg)) + } + errs = append(errs, passThrough(cfg, false)) + errs = append(errs, os.RemoveAll(cfg.PinDir)) + return errors.Join(errs...) +} + +func releaseDispatcher(ctx context.Context, cfg Config) error { + mode, err := cfg.mode() + if err != nil { + return err + } + d, err := xdpdispatch.Open(ctx, cfg.dispatchDir(), mode) + if err != nil { + return err + } + defer d.Close() //nolint:errcheck // our own descriptors + locked, err := d.Lock(ctx) + if err != nil { + return err + } + defer locked.Unlock() + errs := []error{locked.Clear(xdpdispatch.SlotGatewayLB)} + for _, name := range cfg.XDP { + l, err := netlink.LinkByName(name) + if err != nil { + continue // the rig removed it already, and the kernel the link + } + errs = append(errs, locked.Release(l.Attrs().Index)) + } + return errors.Join(errs...) +} + +func removeFilter(iface, name string) error { + l, err := netlink.LinkByName(iface) + if err != nil { + return nil // gone with its filters + } + filters, err := netlink.FilterList(l, netlink.HANDLE_MIN_INGRESS) + if err != nil { + return err + } + for _, f := range filters { + if bf, ok := f.(*netlink.BpfFilter); ok && bf.Name == name { + if err := netlink.FilterDel(f); err != nil { + return fmt.Errorf("remove %s from %s: %w", name, iface, err) + } + } + } + return nil +} diff --git a/tests/handoffs/probe/doc.go b/tests/handoffs/probe/doc.go new file mode 100644 index 00000000..1fe07f19 --- /dev/null +++ b/tests/handoffs/probe/doc.go @@ -0,0 +1,15 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package probe holds the prototype eBPF programs for the Local kernel +// handoffs (#1444) and the loader the lab's handoff-probe uses to place them +// on a scratch rig. It is test code and is never deployed. +// +// handoff.c includes the uSID datapath source, so its policy decisions run +// the product's own tp_capture and tp_decide. Load replaces every uSID map +// those programs use with the map galactic-cni pinned, and fails rather than +// create one. +package probe + +//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cflags "-O2 -g -Wall -Wno-unused-function -idirafter /usr/include/x86_64-linux-gnu -idirafter /usr/include/aarch64-linux-gnu" -target bpfel,bpfeb -type hp_vip_value -type hp_shard_value -type hp_tap_value -type hp_snat_value -type hp_pair_rx_value -type hp_guard_value -type hp_cfg_value -type hp_in6 Handoff handoff.c diff --git a/tests/handoffs/probe/handoff.c b/tests/handoffs/probe/handoff.c new file mode 100644 index 00000000..413bee99 --- /dev/null +++ b/tests/handoffs/probe/handoff.c @@ -0,0 +1,624 @@ +//go:build ignore + +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// handoff.c holds the prototype programs for the Local kernel handoffs that +// docs/worker/qualification.md lists (#1444). It is test code: the lab's +// handoff-probe loads it on a scratch rig and never on a production hook. +// +// It includes the uSID datapath source so the prototypes decide traffic +// policy with the product's own tp_capture and tp_decide, against the maps +// galactic-cni pinned. The loader replaces every map these programs use with +// that pin and refuses to create one. +// +// Programs, by handoff: +// +// local.tap hp_xdp_public (dispatcher slot on the receiving device) +// sets metadata and returns XDP_PASS; hp_tc_tenant_rx (tc +// ingress of the same device) delivers to the tenant TAP. +// hp_tc_tap_rx sends the guest's reply back out. +// local.veth_pair hp_tc_tap_rx sends the guest's NAT flow into the tenant +// end of the pair; hp_xdp_pair_peer (XDP on the peer) +// translates and forwards it. On return hp_xdp_public +// redirects into the peer and hp_tc_pair_rx (tc ingress of +// the tenant end) delivers it to the TAP. +// local.routed_link +// hp_tc_link_guard (tc ingress of the host end) admits only +// the link's own source addresses. + +#include "../../../internal/plumbing/ebpf/prog/usid.c" + +static long (*hp_xdp_adjust_meta)(void *ctx, int delta) = (void *) BPF_FUNC_xdp_adjust_meta; +static long (*hp_xdp_adjust_head)(void *ctx, int delta) = (void *) BPF_FUNC_xdp_adjust_head; + +#define HP_INLINE inline __attribute__((always_inline)) + +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define HP_HTONS(x) __builtin_bswap16(x) +#else +#define HP_HTONS(x) (x) +#endif + +#define HP_ETH_P_IPV6 0x86DD +#define HP_ETH_P_8021Q 0x8100 +#define HP_IPPROTO_TCP 6 +#define HP_IPPROTO_UDP 17 +#define HP_IPPROTO_ICMPV6 58 +#define HP_META_MAGIC 0x48503434 // "HP44" + +// Counters, read by handoff-probe counters. Append only; the Go side names +// them by index. +enum hp_stat { + HP_STAT_XDP_VIP = 0, + HP_STAT_XDP_META_SET, + HP_STAT_XDP_META_FAIL, + HP_STAT_XDP_SHARD, + HP_STAT_XDP_SHARD_VLAN_POP, + HP_STAT_TC_META_OK, + HP_STAT_TC_META_MISSING, + HP_STAT_TC_TENANT_DENY, + HP_STAT_TC_TENANT_REDIRECT, + HP_STAT_TAP_REPLY, + HP_STAT_TAP_REPLY_DENY, + HP_STAT_TAP_NAT, + HP_STAT_TAP_NAT_DENY, + HP_STAT_PEER_SNAT, + HP_STAT_PEER_REDIRECT, + HP_STAT_PEER_PASS, + HP_STAT_PAIR_RX, + HP_STAT_PAIR_RX_DENY, + HP_STAT_GUARD_PASS, + HP_STAT_GUARD_DROP, + HP_STAT_MAX_SEEN_LEN, + HP_STAT_PEER_VLAN_PUSH, + HP_STAT_COUNT = 32, +}; + +struct hp_vip_value { + __u32 tap_ifindex; + // decide is 1 when hp_tc_tenant_rx decides the receiver's policy on the + // PublicService path and marks the packet decided, 0 when it leaves the + // decision to tp_deliver, which takes the Private path. + __u32 decide; + __u8 guest[16]; +}; + +struct hp_shard_value { + __u32 peer_ifindex; + __u32 pad; + __u8 guest[16]; + __u8 smac[6]; + __u8 dmac[6]; +}; + +struct hp_tap_value { + __u32 dut_ifindex; + __u32 pair_ifindex; + __u8 vip[16]; + __u8 client_prefix[8]; // a /64 + __u8 pair_smac[6]; + __u8 pair_dmac[6]; +}; + +struct hp_snat_value { + __u32 redirect; // 1: XDP_REDIRECT through bpf_fib_lookup; 0: XDP_PASS + // When bpf_fib_lookup answers vlan_ifindex, a VLAN device, the program + // pushes the vid tag itself and redirects to lower_ifindex, the VLAN's + // lower device: a VLAN device has no XDP transmit. + __u32 vlan_ifindex; + __u32 lower_ifindex; + __u32 vid; + __u8 guest[16]; + __u8 shard[16]; +}; + +struct hp_pair_rx_value { + __u32 tap_ifindex; + __u32 pad; + __u8 guest[16]; +}; + +struct hp_guard_value { + __u8 prefix[16]; // a /127 +}; + +struct hp_cfg_value { + __u32 vlan_id; // nonzero: hp_xdp_public parses this 802.1Q tag + __u32 pad; +}; + +struct hp_meta { + __u32 magic; + __u32 tap_ifindex; +}; + +struct hp_in6 { + __u8 a[16]; +}; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 16); + __type(key, struct hp_in6); + __type(value, struct hp_vip_value); +} hp_vip SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 16); + __type(key, struct hp_in6); + __type(value, struct hp_shard_value); +} hp_shard SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 16); + __type(key, __u32); + __type(value, struct hp_tap_value); +} hp_tap SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 16); + __type(key, __u32); + __type(value, struct hp_snat_value); +} hp_snat SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 16); + __type(key, __u32); + __type(value, struct hp_pair_rx_value); +} hp_pair_rx SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 16); + __type(key, __u32); + __type(value, struct hp_guard_value); +} hp_guard SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct hp_cfg_value); +} hp_cfg SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, HP_STAT_COUNT); + __type(key, __u32); + __type(value, __u64); +} hp_stats SEC(".maps"); + +static HP_INLINE void hp_count(__u32 stat) +{ + __u64 *v = bpf_map_lookup_elem(&hp_stats, &stat); + + if (v) + __sync_fetch_and_add(v, 1); +} + +static HP_INLINE void hp_seen_len(__u64 len) +{ + __u32 key = HP_STAT_MAX_SEEN_LEN; + __u64 *v = bpf_map_lookup_elem(&hp_stats, &key); + + if (v && len > *v) + *v = len; +} + +static HP_INLINE int hp_eq16(const __u8 *a, const __u8 *b) +{ + const __u64 *x = (const __u64 *) a; + const __u64 *y = (const __u64 *) b; + + return x[0] == y[0] && x[1] == y[1]; +} + +static HP_INLINE void hp_copy16(__u8 *dst, const __u8 *src) +{ + __builtin_memcpy(dst, src, 16); +} + +static HP_INLINE __u16 hp_fold(__s64 sum) +{ + __u64 s = (__u64) sum; + + s = (s & 0xffffffff) + (s >> 32); + s = (s & 0xffff) + (s >> 16); + s = (s & 0xffff) + (s >> 16); + return (__u16) s; +} + +// hp_l4_csum_off returns the offset of the L4 checksum from the start of the +// L4 header, or -1 for a protocol it does not rewrite. +static HP_INLINE int hp_l4_csum_off(__u8 nexthdr) +{ + switch (nexthdr) { + case HP_IPPROTO_TCP: + return 16; + case HP_IPPROTO_UDP: + return 6; + case HP_IPPROTO_ICMPV6: + return 2; + } + return -1; +} + +// hp_xdp_rewrite_addr replaces the 16-byte address at addr, inside an IPv6 +// header at ip6, with to, and updates the L4 checksum that covers it through +// the pseudo-header. The checksum is updated as a complete one, which is +// what a receive path carries. +static HP_INLINE int hp_xdp_rewrite_addr(struct xdp_md *ctx, struct usid_ip6hdr *ip6, __u8 *addr, const __u8 *to) +{ + void *end = (void *) (long) ctx->data_end; + int off = hp_l4_csum_off(ip6->nexthdr); + __u8 from[16]; + __u8 next[16]; + + hp_copy16(from, addr); + hp_copy16(next, to); + if (off >= 0) { + __u16 *csum = (void *) (ip6 + 1) + off; + + if ((void *) (csum + 1) > end) + return -1; + if (!(ip6->nexthdr == HP_IPPROTO_UDP && *csum == 0)) { + __u32 seed = (~(__u32) *csum) & 0xffff; + __s64 sum = bpf_csum_diff((__be32 *) from, 16, (__be32 *) next, 16, seed); + + *csum = ~hp_fold(sum); + } + } + hp_copy16(addr, next); + return 0; +} + +// hp_xdp_public is the dispatcher slot program on the device under test. A +// packet for a VIP gets metadata naming its TAP and is passed to the kernel, +// where hp_tc_tenant_rx takes it. A packet for a shard address is the return +// half of a NAT flow: it is translated back to the guest and redirected into +// the NAT side of the internal veth pair. +SEC("xdp") +int hp_xdp_public(struct xdp_md *ctx) +{ + void *data = (void *) (long) ctx->data; + void *end = (void *) (long) ctx->data_end; + struct usid_ethhdr *eth = data; + __u32 l3 = sizeof(*eth); + __u32 zero = 0; + int tagged = 0; + + if ((void *) (eth + 1) > end) + return XDP_PASS; + hp_seen_len((__u64) (end - data)); + if (eth->h_proto == HP_HTONS(HP_ETH_P_8021Q)) { + struct hp_cfg_value *cfg = bpf_map_lookup_elem(&hp_cfg, &zero); + __be16 *tci = data + sizeof(*eth); // TCI, then the inner EtherType + + if (!cfg || !cfg->vlan_id || (void *) (tci + 2) > end) + return XDP_PASS; + if ((HP_HTONS(tci[0]) & 0x0fff) != cfg->vlan_id || tci[1] != HP_HTONS(HP_ETH_P_IPV6)) + return XDP_PASS; + l3 += 4; + tagged = 1; + } else if (eth->h_proto != HP_HTONS(HP_ETH_P_IPV6)) { + return XDP_PASS; + } + + struct usid_ip6hdr *ip6 = data + l3; + + if ((void *) (ip6 + 1) > end) + return XDP_PASS; + + struct hp_in6 dst; + + hp_copy16(dst.a, ip6->daddr); + + struct hp_vip_value *vip = bpf_map_lookup_elem(&hp_vip, &dst); + + if (vip) { + hp_count(HP_STAT_XDP_VIP); + if (hp_xdp_adjust_meta(ctx, -(int) sizeof(struct hp_meta)) == 0) { + struct hp_meta *meta = (void *) (long) ctx->data_meta; + + if ((void *) (meta + 1) <= (void *) (long) ctx->data) { + meta->magic = HP_META_MAGIC; + meta->tap_ifindex = vip->tap_ifindex; + hp_count(HP_STAT_XDP_META_SET); + } + } else { + hp_count(HP_STAT_XDP_META_FAIL); + } + return XDP_PASS; + } + + struct hp_shard_value *shard = bpf_map_lookup_elem(&hp_shard, &dst); + + if (!shard) + return XDP_PASS; + if (hp_xdp_rewrite_addr(ctx, ip6, ip6->daddr, shard->guest)) + return XDP_PASS; + if (tagged) { + // Drop the tag in place: move both MAC addresses forward over it. + __u8 macs[12]; + + __builtin_memcpy(macs, data, 12); + if ((void *) ((__u8 *) data + 16) > end) + return XDP_PASS; + __builtin_memcpy((__u8 *) data + 4, macs, 12); + if (hp_xdp_adjust_head(ctx, 4)) + return XDP_PASS; + data = (void *) (long) ctx->data; + end = (void *) (long) ctx->data_end; + eth = data; + if ((void *) (eth + 1) > end) + return XDP_PASS; + hp_count(HP_STAT_XDP_SHARD_VLAN_POP); + } + __builtin_memcpy(eth->h_source, shard->smac, 6); + __builtin_memcpy(eth->h_dest, shard->dmac, 6); + hp_count(HP_STAT_XDP_SHARD); + return bpf_redirect(shard->peer_ifindex, 0); +} + +static HP_INLINE struct usid_ip6hdr *hp_skb_ip6(struct __sk_buff *skb) +{ + void *data = (void *) (long) skb->data; + void *end = (void *) (long) skb->data_end; + struct usid_ethhdr *eth = data; + struct usid_ip6hdr *ip6 = (void *) (eth + 1); + + if ((void *) (ip6 + 1) > end || eth->h_proto != HP_HTONS(HP_ETH_P_IPV6)) + return 0; + return ip6; +} + +// hp_tc_tenant_rx runs at tc ingress of the device under test, after +// hp_xdp_public passed a VIP packet. It takes the TAP from the metadata, or +// from the VIP when the metadata did not survive, decides the receiver's +// policy and delivers to the TAP through the kernel neighbor. +SEC("tc") +int hp_tc_tenant_rx(struct __sk_buff *skb) +{ + if (bpf_skb_pull_data(skb, sizeof(struct usid_ethhdr) + sizeof(struct usid_ip6hdr))) + return TC_ACT_UNSPEC; + + __u32 meta_tap = 0; + struct hp_meta *meta = (void *) (long) skb->data_meta; + + if ((void *) (meta + 1) <= (void *) (long) skb->data && meta->magic == HP_META_MAGIC) + meta_tap = meta->tap_ifindex; + + struct usid_ip6hdr *ip6 = hp_skb_ip6(skb); + + if (!ip6) + return TC_ACT_UNSPEC; + + struct hp_in6 dst; + + hp_copy16(dst.a, ip6->daddr); + + struct hp_vip_value *vip = bpf_map_lookup_elem(&hp_vip, &dst); + + if (!vip) + return TC_ACT_UNSPEC; + if (meta_tap == vip->tap_ifindex) + hp_count(HP_STAT_TC_META_OK); + else + hp_count(HP_STAT_TC_META_MISSING); + + __u32 tap = vip->tap_ifindex; + + if (vip->decide) { + tp_capture(skb, TP_DIR_INGRESS, tap, 0); + if (tp_decide(tap, TP_DIR_INGRESS, TP_PATH_PUBLIC_SERVICE)) { + hp_count(HP_STAT_TC_TENANT_DENY); + return TC_ACT_SHOT; + } + skb->mark |= tp_ingress_mark(); + } + + struct bpf_redir_neigh nh = {}; + + nh.nh_family = USID_AF_INET6; + __builtin_memcpy(nh.ipv6_nh, vip->guest, 16); + hp_count(HP_STAT_TC_TENANT_REDIRECT); + return bpf_redirect_neigh(tap, &nh, sizeof(nh), 0); +} + +// hp_tc_tap_rx runs at tc ingress of the tenant TAP, after usid_egress has +// passed the packet through its Local pass-through entry. A reply from the +// VIP leaves through the device under test; any other packet to the client +// prefix is a NAT flow and enters the tenant end of the internal veth pair. +SEC("tc") +int hp_tc_tap_rx(struct __sk_buff *skb) +{ + __u32 ifindex = skb->ifindex; + struct hp_tap_value *v = bpf_map_lookup_elem(&hp_tap, &ifindex); + + if (!v) + return TC_ACT_UNSPEC; + if (bpf_skb_pull_data(skb, sizeof(struct usid_ethhdr) + sizeof(struct usid_ip6hdr))) + return TC_ACT_UNSPEC; + + struct usid_ip6hdr *ip6 = hp_skb_ip6(skb); + + if (!ip6) + return TC_ACT_UNSPEC; + if (*(__u64 *) ip6->daddr != *(__u64 *) v->client_prefix) + return TC_ACT_UNSPEC; + + if (hp_eq16(ip6->saddr, v->vip)) { + tp_capture(skb, TP_DIR_EGRESS, ifindex, 0); + if (tp_decide(ifindex, TP_DIR_EGRESS, TP_PATH_PUBLIC_SERVICE)) { + hp_count(HP_STAT_TAP_REPLY_DENY); + return TC_ACT_SHOT; + } + hp_count(HP_STAT_TAP_REPLY); + return bpf_redirect_neigh(v->dut_ifindex, 0, 0, 0); + } + + tp_capture(skb, TP_DIR_EGRESS, ifindex, 0); + if (tp_decide(ifindex, TP_DIR_EGRESS, TP_PATH_PUBLIC_NAT)) { + hp_count(HP_STAT_TAP_NAT_DENY); + return TC_ACT_SHOT; + } + + void *data = (void *) (long) skb->data; + void *end = (void *) (long) skb->data_end; + struct usid_ethhdr *eth = data; + + if ((void *) (eth + 1) > end) + return TC_ACT_SHOT; + __builtin_memcpy(eth->h_source, v->pair_smac, 6); + __builtin_memcpy(eth->h_dest, v->pair_dmac, 6); + hp_count(HP_STAT_TAP_NAT); + return bpf_redirect(v->pair_ifindex, 0); +} + +// hp_xdp_pair_peer is XDP on the NAT side of the internal veth pair. It +// stands in for the NAT shard: it translates the guest's source to the +// shard's address and forwards the packet toward the client. +SEC("xdp") +int hp_xdp_pair_peer(struct xdp_md *ctx) +{ + __u32 ifindex = ctx->ingress_ifindex; + struct hp_snat_value *v = bpf_map_lookup_elem(&hp_snat, &ifindex); + void *data = (void *) (long) ctx->data; + void *end = (void *) (long) ctx->data_end; + struct usid_ethhdr *eth = data; + struct usid_ip6hdr *ip6 = (void *) (eth + 1); + + if (!v || (void *) (ip6 + 1) > end || eth->h_proto != HP_HTONS(HP_ETH_P_IPV6)) + return XDP_PASS; + hp_seen_len((__u64) (end - data)); + if (!hp_eq16(ip6->saddr, v->guest)) + return XDP_PASS; + if (hp_xdp_rewrite_addr(ctx, ip6, ip6->saddr, v->shard)) + return XDP_PASS; + hp_count(HP_STAT_PEER_SNAT); + if (!v->redirect) { + hp_count(HP_STAT_PEER_PASS); + return XDP_PASS; + } + + struct bpf_fib_lookup fib = {}; + + fib.family = USID_AF_INET6; + fib.ifindex = ifindex; + __builtin_memcpy(fib.ipv6_src, ip6->saddr, 16); + __builtin_memcpy(fib.ipv6_dst, ip6->daddr, 16); + if (bpf_fib_lookup(ctx, &fib, sizeof(fib), 0) != BPF_FIB_LKUP_RET_SUCCESS) { + hp_count(HP_STAT_PEER_PASS); + return XDP_PASS; + } + __u8 smac[6]; + __u8 dmac[6]; + + __builtin_memcpy(smac, fib.smac, 6); + __builtin_memcpy(dmac, fib.dmac, 6); + if (v->vlan_ifindex && fib.ifindex == v->vlan_ifindex) { + __u32 vid = v->vid; + __u32 lower = v->lower_ifindex; + + if (hp_xdp_adjust_head(ctx, -4)) + return XDP_PASS; + data = (void *) (long) ctx->data; + end = (void *) (long) ctx->data_end; + eth = data; + __be16 *tag = data + 12; + + if ((void *) (eth + 1) > end || (void *) (tag + 3) > end) + return XDP_ABORTED; + __builtin_memcpy(eth->h_dest, dmac, 6); + __builtin_memcpy(eth->h_source, smac, 6); + tag[0] = HP_HTONS(HP_ETH_P_8021Q); + tag[1] = HP_HTONS((__u16) (vid & 0x0fff)); + tag[2] = HP_HTONS(HP_ETH_P_IPV6); + hp_count(HP_STAT_PEER_VLAN_PUSH); + hp_count(HP_STAT_PEER_REDIRECT); + return bpf_redirect(lower, 0); + } + __builtin_memcpy(eth->h_source, smac, 6); + __builtin_memcpy(eth->h_dest, dmac, 6); + hp_count(HP_STAT_PEER_REDIRECT); + return bpf_redirect(fib.ifindex, 0); +} + +// hp_tc_pair_rx is the reverse tc receive on the tenant end of the internal +// veth pair. It decides the receiver's policy on the PublicNAT path and +// delivers to the TAP. +SEC("tc") +int hp_tc_pair_rx(struct __sk_buff *skb) +{ + __u32 ifindex = skb->ifindex; + struct hp_pair_rx_value *v = bpf_map_lookup_elem(&hp_pair_rx, &ifindex); + + if (!v) + return TC_ACT_UNSPEC; + if (bpf_skb_pull_data(skb, sizeof(struct usid_ethhdr) + sizeof(struct usid_ip6hdr))) + return TC_ACT_UNSPEC; + + struct usid_ip6hdr *ip6 = hp_skb_ip6(skb); + + if (!ip6 || !hp_eq16(ip6->daddr, v->guest)) + return TC_ACT_UNSPEC; + + __u32 tap = v->tap_ifindex; + + tp_capture(skb, TP_DIR_INGRESS, tap, 0); + if (tp_decide(tap, TP_DIR_INGRESS, TP_PATH_PUBLIC_NAT)) { + hp_count(HP_STAT_PAIR_RX_DENY); + return TC_ACT_SHOT; + } + skb->mark |= tp_ingress_mark(); + + struct bpf_redir_neigh nh = {}; + + nh.nh_family = USID_AF_INET6; + __builtin_memcpy(nh.ipv6_nh, v->guest, 16); + hp_count(HP_STAT_PAIR_RX); + return bpf_redirect_neigh(tap, &nh, sizeof(nh), 0); +} + +// hp_tc_link_guard runs at tc ingress of the host end of a pod-to-host routed +// link. The link joins the pod to the tenant VRF, so a source the link was +// not given would let the pod speak as any address in the VPC. +SEC("tc") +int hp_tc_link_guard(struct __sk_buff *skb) +{ + __u32 ifindex = skb->ifindex; + struct hp_guard_value *v = bpf_map_lookup_elem(&hp_guard, &ifindex); + + if (!v) + return TC_ACT_UNSPEC; + if (bpf_skb_pull_data(skb, sizeof(struct usid_ethhdr) + sizeof(struct usid_ip6hdr))) + return TC_ACT_SHOT; + + struct usid_ip6hdr *ip6 = hp_skb_ip6(skb); + + if (!ip6) + return TC_ACT_UNSPEC; + // Link-local sources carry Neighbor Discovery on the link itself. + if (ip6->saddr[0] == 0xfe && (ip6->saddr[1] & 0xc0) == 0x80) { + hp_count(HP_STAT_GUARD_PASS); + return TC_ACT_UNSPEC; + } + + const __u64 *s = (const __u64 *) ip6->saddr; + const __u64 *p = (const __u64 *) v->prefix; + // A /127: every bit but the last. + __u64 mask = HP_HTONS(1) == 1 ? ~1ULL : ~(1ULL << 56); + + if (s[0] == p[0] && (s[1] & mask) == (p[1] & mask)) { + hp_count(HP_STAT_GUARD_PASS); + return TC_ACT_UNSPEC; + } + hp_count(HP_STAT_GUARD_DROP); + return TC_ACT_SHOT; +} diff --git a/tests/handoffs/probe/load.go b/tests/handoffs/probe/load.go new file mode 100644 index 00000000..c02affa5 --- /dev/null +++ b/tests/handoffs/probe/load.go @@ -0,0 +1,179 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package probe + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/cilium/ebpf" +) + +// ErrDatapathMismatch means a uSID map galactic-cni pinned does not match the +// map this build of handoff.c expects, so the running datapath was built from +// a different revision than this checkout. +var ErrDatapathMismatch = errors.New("probe: the pinned uSID datapath does not match this checkout") + +// Programs are the prototype programs. Assigning only these loads only the +// uSID maps they use, never the rest of the datapath handoff.c includes. +type Programs struct { + XDPPublic *ebpf.Program `ebpf:"hp_xdp_public"` + TCTenantRx *ebpf.Program `ebpf:"hp_tc_tenant_rx"` + TCTapRx *ebpf.Program `ebpf:"hp_tc_tap_rx"` + XDPPairPeer *ebpf.Program `ebpf:"hp_xdp_pair_peer"` + TCPairRx *ebpf.Program `ebpf:"hp_tc_pair_rx"` + TCLinkGuard *ebpf.Program `ebpf:"hp_tc_link_guard"` +} + +// Close closes every program descriptor. +func (p *Programs) Close() error { + var errs []error + for _, prog := range []*ebpf.Program{p.XDPPublic, p.TCTenantRx, p.TCTapRx, p.XDPPairPeer, p.TCPairRx, + p.TCLinkGuard} { + if prog != nil { + errs = append(errs, prog.Close()) + } + } + return errors.Join(errs...) +} + +// Maps are the probe's own maps. +type Maps struct { + VIP *ebpf.Map `ebpf:"hp_vip"` + Shard *ebpf.Map `ebpf:"hp_shard"` + Tap *ebpf.Map `ebpf:"hp_tap"` + SNAT *ebpf.Map `ebpf:"hp_snat"` + PairRx *ebpf.Map `ebpf:"hp_pair_rx"` + Guard *ebpf.Map `ebpf:"hp_guard"` + Cfg *ebpf.Map `ebpf:"hp_cfg"` + Stats *ebpf.Map `ebpf:"hp_stats"` +} + +// Close closes every map descriptor. +func (m *Maps) Close() error { + var errs []error + for _, mp := range []*ebpf.Map{m.VIP, m.Shard, m.Tap, m.SNAT, m.PairRx, m.Guard, m.Cfg, m.Stats} { + if mp != nil { + errs = append(errs, mp.Close()) + } + } + return errors.Join(errs...) +} + +// Objects are the loaded prototype programs and their own maps. +type Objects struct { + Programs Programs + Maps Maps + pinned []*ebpf.Map +} + +// Close releases this process's descriptors. Pinned maps, links and tc +// filters stay. +func (o *Objects) Close() error { + errs := make([]error, 0, 2+len(o.pinned)) + errs = append(errs, o.Programs.Close(), o.Maps.Close()) + for _, m := range o.pinned { + errs = append(errs, m.Close()) + } + return errors.Join(errs...) +} + +// Load loads the prototype programs. Every uSID map they use is replaced with +// the map pinned under galacticPinDir, and a map with no pin there fails the +// load, so Load never creates or changes a product pin. The probe's own maps +// (hp_*) are pinned under mapDir, so later invocations read the same ones. +func Load(galacticPinDir, mapDir string) (*Objects, error) { + spec, err := LoadHandoff() + if err != nil { + return nil, fmt.Errorf("probe: load compiled spec: %w", err) + } + if err := os.MkdirAll(mapDir, 0o755); err != nil { + return nil, fmt.Errorf("probe: create map pin directory: %w", err) + } + + objs := &Objects{} + replacements := map[string]*ebpf.Map{} + for name, m := range spec.Maps { + switch { + case strings.HasPrefix(name, "hp_"): + m.Pinning = ebpf.PinByName + case strings.HasPrefix(name, "."): + // Global data of the included source, private to this load. + default: + pinned, err := ebpf.LoadPinnedMap(filepath.Join(galacticPinDir, name), nil) + if err != nil { + // Not pinned: drop it from the spec. A program that uses it + // then fails to load instead of getting a fresh map. + delete(spec.Maps, name) + continue + } + replacements[name] = pinned + objs.pinned = append(objs.pinned, pinned) + } + } + + opts := &ebpf.CollectionOptions{ + Maps: ebpf.MapOptions{PinPath: mapDir}, + MapReplacements: replacements, + } + if err := spec.LoadAndAssign(&objs.Programs, opts); err != nil { + _ = objs.Close() + if errors.Is(err, ebpf.ErrMapIncompatible) { + return nil, fmt.Errorf("%w: %w", ErrDatapathMismatch, err) + } + var ve *ebpf.VerifierError + if errors.As(err, &ve) { + return nil, fmt.Errorf("probe: verifier rejected a prototype program: %w\n%s", err, fmt.Sprintf("%+v", ve)) + } + return nil, fmt.Errorf("probe: load programs: %w", err) + } + if err := spec.LoadAndAssign(&objs.Maps, opts); err != nil { + _ = objs.Close() + return nil, fmt.Errorf("probe: load probe maps: %w", err) + } + return objs, nil +} + +// StatNames names hp_stats' slots, in handoff.c's enum hp_stat order. +var StatNames = []string{ + "xdp_vip", + "xdp_meta_set", + "xdp_meta_fail", + "xdp_shard", + "xdp_shard_vlan_pop", + "tc_meta_ok", + "tc_meta_missing", + "tc_tenant_deny", + "tc_tenant_redirect", + "tap_reply", + "tap_reply_deny", + "tap_nat", + "tap_nat_deny", + "peer_snat", + "peer_redirect", + "peer_pass", + "pair_rx", + "pair_rx_deny", + "guard_pass", + "guard_drop", + "max_seen_len", + "peer_vlan_push", +} + +// Stats reads hp_stats by name. +func Stats(m *ebpf.Map) (map[string]uint64, error) { + out := make(map[string]uint64, len(StatNames)) + for i, name := range StatNames { + var v uint64 + if err := m.Lookup(uint32(i), &v); err != nil { + return nil, fmt.Errorf("probe: read %s: %w", name, err) + } + out[name] = v + } + return out, nil +} diff --git a/tests/handoffs/probe/load_root_test.go b/tests/handoffs/probe/load_root_test.go new file mode 100644 index 00000000..27bf7185 --- /dev/null +++ b/tests/handoffs/probe/load_root_test.go @@ -0,0 +1,96 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package probe + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "go.datum.net/galactic/internal/plumbing/ebpf/attach" +) + +func requireRoot(t *testing.T) { + t.Helper() + if os.Geteuid() != 0 { + t.Skip("test requires root (CAP_BPF/CAP_SYS_ADMIN) to load BPF programs; re-run via sudo") + } +} + +// TestLoadAgainstPinnedDatapath loads every prototype program through the +// verifier, with the uSID maps a datapath load pinned, and checks that Load +// uses those pins rather than creating any. +func TestLoadAgainstPinnedDatapath(t *testing.T) { + requireRoot(t) + + galactic := filepath.Join("/sys/fs/bpf", fmt.Sprintf("galactic-handoff-test-%d", os.Getpid())) + mine := galactic + "-probe" + t.Cleanup(func() { + _ = os.RemoveAll(galactic) + _ = os.RemoveAll(mine) + }) + usid, err := attach.Load(galactic) + if err != nil { + t.Fatalf("load the uSID datapath: %v", err) + } + defer usid.Close() //nolint:errcheck // test cleanup + + before, err := os.ReadDir(galactic) + if err != nil { + t.Fatal(err) + } + objs, err := Load(galactic, mine) + if err != nil { + t.Fatalf("Load: %v", err) + } + defer objs.Close() //nolint:errcheck // test cleanup + for name, p := range map[string]any{ + "hp_xdp_public": objs.Programs.XDPPublic, "hp_tc_tenant_rx": objs.Programs.TCTenantRx, + "hp_tc_tap_rx": objs.Programs.TCTapRx, "hp_xdp_pair_peer": objs.Programs.XDPPairPeer, + "hp_tc_pair_rx": objs.Programs.TCPairRx, "hp_tc_link_guard": objs.Programs.TCLinkGuard, + } { + if p == nil { + t.Errorf("%s did not load", name) + } + } + after, err := os.ReadDir(galactic) + if err != nil { + t.Fatal(err) + } + if len(after) != len(before) { + t.Errorf("Load changed the datapath's pins: %d entries before, %d after", len(before), len(after)) + } + if _, err := os.Stat(filepath.Join(mine, "hp_stats")); err != nil { + t.Errorf("the probe's own maps are not pinned under its directory: %v", err) + } +} + +// TestLoadRefusesWithoutDatapath checks that Load fails, and creates no map, +// when the uSID maps its programs use are not pinned. +func TestLoadRefusesWithoutDatapath(t *testing.T) { + requireRoot(t) + + empty := filepath.Join("/sys/fs/bpf", fmt.Sprintf("galactic-handoff-empty-%d", os.Getpid())) + mine := empty + "-probe" + t.Cleanup(func() { + _ = os.RemoveAll(empty) + _ = os.RemoveAll(mine) + }) + if err := os.MkdirAll(empty, 0o755); err != nil { + t.Fatal(err) + } + if objs, err := Load(empty, mine); err == nil { + _ = objs.Close() + t.Fatal("Load succeeded with no uSID maps pinned") + } + entries, err := os.ReadDir(empty) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Errorf("Load created %d entries in the datapath's pin directory", len(entries)) + } +} diff --git a/tests/handoffs/probe/stats_test.go b/tests/handoffs/probe/stats_test.go new file mode 100644 index 00000000..2fe2965b --- /dev/null +++ b/tests/handoffs/probe/stats_test.go @@ -0,0 +1,36 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package probe + +import ( + "os" + "regexp" + "strings" + "testing" +) + +// TestStatNamesMatchEnum keeps StatNames in step with enum hp_stat in +// handoff.c, which counters are read by. +func TestStatNamesMatchEnum(t *testing.T) { + src, err := os.ReadFile("handoff.c") + if err != nil { + t.Fatal(err) + } + body := regexp.MustCompile(`(?s)enum hp_stat \{(.*?)\};`).FindSubmatch(src) + if body == nil { + t.Fatal("handoff.c has no enum hp_stat") + } + var names []string + for _, m := range regexp.MustCompile(`HP_STAT_([A-Z0-9_]+)`).FindAllSubmatch(body[1], -1) { + name := strings.ToLower(string(m[1])) + if name == "count" { + continue + } + names = append(names, name) + } + if strings.Join(names, ",") != strings.Join(StatNames, ",") { + t.Fatalf("StatNames = %v, enum hp_stat = %v", StatNames, names) + } +} From 8249845e1db79d7875ade339be5386032acd7deb Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Thu, 19 Nov 2026 03:23:14 -0500 Subject: [PATCH 3/6] test(containerlab): Add the Local handoff fixture and checks ns80-tap puts a TAP-backed guest in each of ns80's VPCs on dfw-worker, with the same address, and an Envoy stand-in. Each TAP guest takes the cluster's network as its primary and galactic-tap as a secondary attachment, so its sandbox starts; a vmm container bridges the host TAP to the guest's interface. publish-attachments.sh now also publishes an attachment a pod names as a secondary network. task handoff:twice deploys the fixture, proves each handoff in every cell (veth, bond and VLAN devices, native and generic XDP) twice with captures, measures MTU and offload behavior, removes the fixture, and compares with handoffs/expected-results.json. Each check covers both directions, a policy-denied control, isolation between the two VPCs and the absence of the probes on the node's physical members. --- deploy/containerlab/AGENTS.md | 1 + deploy/containerlab/Taskfile.yaml | 38 + deploy/containerlab/docs/baseline.md | 6 +- deploy/containerlab/docs/handoffs.md | 258 ++ .../handoffs/expected-results.json | 2403 +++++++++++++++++ .../tenants/ns80-tap/base/guests.yaml | 200 ++ .../tenants/ns80-tap/base/kustomization.yaml | 9 + .../tenants/ns80-tap/base/namespace.yaml | 4 + .../tenants/ns80-tap/dfw/kustomization.yaml | 3 + .../resources/tenants/ns80-tap/dfw/nad.yaml | 71 + .../tenants/ns80-tap/policy/bindings.yaml | 19 + .../ns80-tap/policy/kustomization.yaml | 8 + .../resources/tenants/policies/README.md | 15 +- .../policies/handoff/kustomization.yaml | 2 + .../tenants/policies/handoff/policies.yaml | 77 + deploy/containerlab/scripts/handoff-echo.py | 37 + .../containerlab/scripts/handoff-fixture.sh | 47 + deploy/containerlab/scripts/handoff-lib.sh | 354 +++ .../containerlab/scripts/handoff-measure.sh | 197 ++ .../containerlab/scripts/handoff-observe.sh | 404 +++ deploy/containerlab/scripts/handoff-pcap.py | 147 + deploy/containerlab/scripts/handoff-rig.sh | 232 ++ deploy/containerlab/scripts/handoff-twice.sh | 85 + .../scripts/publish-attachments.sh | 18 +- 24 files changed, 4620 insertions(+), 15 deletions(-) create mode 100644 deploy/containerlab/docs/handoffs.md create mode 100644 deploy/containerlab/handoffs/expected-results.json create mode 100644 deploy/containerlab/resources/tenants/ns80-tap/base/guests.yaml create mode 100644 deploy/containerlab/resources/tenants/ns80-tap/base/kustomization.yaml create mode 100644 deploy/containerlab/resources/tenants/ns80-tap/base/namespace.yaml create mode 100644 deploy/containerlab/resources/tenants/ns80-tap/dfw/kustomization.yaml create mode 100644 deploy/containerlab/resources/tenants/ns80-tap/dfw/nad.yaml create mode 100644 deploy/containerlab/resources/tenants/ns80-tap/policy/bindings.yaml create mode 100644 deploy/containerlab/resources/tenants/ns80-tap/policy/kustomization.yaml create mode 100644 deploy/containerlab/resources/tenants/policies/handoff/kustomization.yaml create mode 100644 deploy/containerlab/resources/tenants/policies/handoff/policies.yaml create mode 100755 deploy/containerlab/scripts/handoff-echo.py create mode 100755 deploy/containerlab/scripts/handoff-fixture.sh create mode 100644 deploy/containerlab/scripts/handoff-lib.sh create mode 100755 deploy/containerlab/scripts/handoff-measure.sh create mode 100755 deploy/containerlab/scripts/handoff-observe.sh create mode 100755 deploy/containerlab/scripts/handoff-pcap.py create mode 100755 deploy/containerlab/scripts/handoff-rig.sh create mode 100755 deploy/containerlab/scripts/handoff-twice.sh diff --git a/deploy/containerlab/AGENTS.md b/deploy/containerlab/AGENTS.md index e9abac5f..a81910b9 100644 --- a/deploy/containerlab/AGENTS.md +++ b/deploy/containerlab/AGENTS.md @@ -46,3 +46,4 @@ Full documentation — topology, addressing, tasks — is in [README.md](README. Verification commands are in [docs/verification.md](docs/verification.md). Deploying the five test VPCs and verifying cross-site/same-node connectivity is documented in [docs/tenants.md](docs/tenants.md). The dedicated-placement baseline (`ns80`, `task baseline:twice`) is documented in [docs/baseline.md](docs/baseline.md). +The Local handoff checks (`ns80-tap`, `task handoff:twice`) are documented in [docs/handoffs.md](docs/handoffs.md). diff --git a/deploy/containerlab/Taskfile.yaml b/deploy/containerlab/Taskfile.yaml index 709d970e..6406a10b 100644 --- a/deploy/containerlab/Taskfile.yaml +++ b/deploy/containerlab/Taskfile.yaml @@ -1399,6 +1399,44 @@ tasks: cmds: - LAB={{.LAB}} ./scripts/baseline-twice.sh {{.CLI_ARGS}} + # The Local handoff checks (docs/handoffs.md). Not part of deploy or + # verify: handoff:twice deploys ns80-tap, observes twice, measures and + # removes ns80-tap again. + "deploy:ns80-tap": + desc: Deploy the Local handoff fixture (TAP guests in both ns80 VPCs and an Envoy stand-in) on dfw + cmds: + - ./scripts/handoff-fixture.sh up + + "delete:ns80-tap": + desc: Remove the Local handoff fixture from dfw + cmds: + - ./scripts/handoff-fixture.sh down + + "handoff:probe": + desc: Build handoff-probe (tests/handoffs) for the Local handoff checks + dir: ../.. + cmds: + - task build:ebpf + - CGO_ENABLED=0 go build -o bin/handoff-probe ./tests/handoffs/cmd/handoff-probe + + "handoff:observe": + desc: Prove the Local handoffs once, with captures (needs ns80-tap; OUTDIR via task handoff:observe -- DIR) + deps: ["handoff:probe"] + cmds: + - LAB={{.LAB}} ./scripts/handoff-observe.sh {{.CLI_ARGS | default "../../bin/handoffs/observe"}} + + "handoff:measure": + desc: Measure the Local handoffs' MTU limits and offload behavior (needs ns80-tap) + deps: ["handoff:probe"] + cmds: + - LAB={{.LAB}} ./scripts/handoff-measure.sh {{.CLI_ARGS | default "../../bin/handoffs/measure"}} + + "handoff:twice": + desc: Deploy ns80-tap, observe twice, measure, remove ns80-tap, and compare with handoffs/expected-results.json + deps: ["handoff:probe"] + cmds: + - LAB={{.LAB}} ./scripts/handoff-twice.sh {{.CLI_ARGS}} + clean: desc: Destroy the lab and delete artifacts cmds: diff --git a/deploy/containerlab/docs/baseline.md b/deploy/containerlab/docs/baseline.md index 6f0f290e..4d1d2e99 100644 --- a/deploy/containerlab/docs/baseline.md +++ b/deploy/containerlab/docs/baseline.md @@ -6,8 +6,10 @@ on two physical workers. Services run on dedicated edge nodes, and workloads run on compute nodes. Later work on the Galactic Worker reuses this fixture and compares its results against the reference recorded here. -The fixture has no TAP-backed guest. The tenant TAP capture point belongs to -[#1444](https://github.com/datum-cloud/galactic/issues/1444). +The fixture has no TAP-backed guest. The tenant TAP guests and their capture +points are in the separate `ns80-tap` fixture of the +[Local handoff checks](handoffs.md), which is deployed only while those checks +run. ## Fixture diff --git a/deploy/containerlab/docs/handoffs.md b/deploy/containerlab/docs/handoffs.md new file mode 100644 index 00000000..995eaada --- /dev/null +++ b/deploy/containerlab/docs/handoffs.md @@ -0,0 +1,258 @@ +# Local kernel handoffs + +This page describes the scripted proof of the three Local handoffs the Worker +[qualification plan](../../../docs/worker/qualification.md#handoffs) lists +([#1444](https://github.com/datum-cloud/galactic/issues/1444)): + +| Handoff | Capability | +| --------------------------------------------------------------------------------------------------------------- | ----------------------------------- | +| Public receive `XDP_PASS` into the tenant-side tc program, delivered to a tenant TAP | `local.tap` | +| tc on one end of the NAT internal veth pair, XDP on its peer, and the reverse tc receive | `local.veth_pair` | +| A VRF-bound (`SO_BINDTODEVICE`) socket in an Envoy pod reaching tenant guests through a pod-to-host routed link | `local.routed_link`, `local.socket` | + +The checks run on `dfw-worker`, the compute node of the +[dedicated-placement baseline](baseline.md), against both of `ns80`'s VPCs. +Their results and what they changed in the Worker contract are in +[contract amendments](../../../docs/worker/contract.md#local-handoffs). + +## Running + +```bash +cd deploy/containerlab +task deploy:ns80 # if ns80 is not deployed +task handoff:twice # fixture up, observe twice, measure, fixture down +task verify:ns80 # the baseline fixture is unchanged +``` + +`task handoff:twice` builds `bin/handoff-probe` from this checkout, deploys the +`ns80-tap` fixture, runs `scripts/handoff-observe.sh` twice and +`scripts/handoff-measure.sh` once, and removes the fixture. It fails unless both +observation runs pass, agree, and match +[`handoffs/expected-results.json`](../handoffs/expected-results.json). Output +lands in `bin/handoffs//` at the repository root, or in the +directory given after `--`. + +Before it changes anything, each script checks that the lab runs, that `ns80` +is ready, that no earlier run left a rig, fixture or probe state behind, and +that the probe's programs load against the running datapath. When a check +fails, the script writes the reason to `precondition`, changes nothing and +exits 0. After the run it removes the rig, the probe and everything the probe +placed, and fails if `handoff-probe leftovers` finds any of it. + +`task handoff:observe -- DIR` and `task handoff:measure -- DIR` run one script +against a fixture deployed with `task deploy:ns80-tap`, and +`HANDOFF_CELLS=plain-native,...` limits observation to some cells. Remove the +fixture afterwards with `task delete:ns80-tap`. + +## Fixture + +`ns80-tap` (`resources/tenants/ns80-tap/`) runs on `dfw-worker` only: + +| Pod | Network | Role | +| --------------- | ----------------------------------------------------------------------------------- | ---------------------------------------------------------- | +| `tap-guest-a` | The cluster's primary network, plus `vpc-a-tap` (VPC 80) as a secondary attachment | TAP guest `fd20:80:ff03::1:0:0`, VIP `2001:db8:4144:a::80` | +| `tap-guest-b` | The cluster's primary network, plus `vpc-b-tap` (VPC 81) as a secondary attachment | TAP guest `fd20:80:ff03::1:0:0`, VIP `2001:db8:4144:a::81` | +| `envoy-standin` | The cluster's primary network; the run adds a VRF per VPC and a routed link to each | Envoy's sockets | + +Both TAP guests use the same address, as `ns80`'s veth guests do. Their +attachments are bound to the `handoff` policy set +(`resources/tenants/policies/README.md`). + +### The tenant TAP + +`galactic-tap` creates the host TAP (`G000000080082H`, `G000000081082H`) in the +VPC's VRF and configures nothing inside the pod: a TAP's reader is a hypervisor, +not the pod's network namespace. Its CNI result names only that host TAP. In +[#1432](https://github.com/datum-cloud/galactic/issues/1432) the TAP guest used +`galactic-tap` as its default network, and the runtime, which needs a pod +interface from the default network, failed every sandbox with "failed to find +network info for sandbox". + +Here the TAP is a secondary attachment +(`k8s.v1.cni.cncf.io/networks: vpc-a-tap`), so the sandbox starts on the +cluster's network and multus runs `galactic-tap` after it. The pod's `vmm` +container stands in for the hypervisor. With `hostPID`, it names the node's +network namespace (`ip netns attach host 1`), opens the host TAP through socat's +`netns=` option, and bridges it to `tap0` inside the pod, the guest's interface. +It configures `tap0` only after the bridge holds it; an address or route added +before is lost with the interface. The pod is ready once `tap0` has carrier, +which it has only while the bridge reads the host TAP. + +`publish-attachments.sh` publishes an attachment named in a pod's +`k8s.v1.cni.cncf.io/networks` list as it does a default network, so the TAP +attachments get their `hostInterface` and their policy bindings. + +The TAP capture points are the host TAP on the node and `tap0` in the pod. + +### Cilium and the TAPs + +The lab runs Cilium with kube-proxy replacement. `deploy-cni.sh` pins its +devices to `eth0` and the bonds, as deployed nodes pin them to their uplinks. +Before that pin Cilium auto-detected its devices and adopted each host TAP, +because the TAP carries a global address (the guest's gateway). Its tcx attach +deleted every legacy tc filter on the TAP, including galactic-cni's egress +chain and `tp_deliver`. Each run keeps the evidence that this does not happen +any more: `logs/tc-monitor.txt` (every tc change on the node), +`logs/cilium-status.txt` (Cilium's devices) and `logs/cilium-taps.log`. The +[contract](../../../docs/worker/contract.md#local-handoffs) states the rule. + +## Rig + +The checks never touch the node's shared hooks: the dispatcher, gateway and NAT +on the edge nodes, and the compute node's uplinks. They build a rig on +`dfw-worker` instead. A rig container (`handoff-rig`) runs the debug image in +the node's network and PID namespaces; it creates every interface, runs every +listener and capture, and takes them all with it when it is removed. +`handoff-probe` runs in the node itself, next to `galactic-cni`, whose pinned +maps it reads. + +| Part | Interfaces | +| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Receiving device | `hp-dut0`, a veth; or `hp-bond` (802.3ad, `mkbond.sh`'s parameters) over `hp-dut0` and `hp-dut1`; or `hp-dut0.120`, a VLAN on `hp-dut0`. `2001:db8:4144:1::1/64` | +| Client | The other ends, in the rig's `hp-client` namespace. `2001:db8:4144:1::2/64` | +| Internal veth pairs | `hpn80t` in VPC 80's VRF and `hpn80n` in the node's namespace; `hpn81t` and `hpn81n` for VPC 81 | +| Routed links | `hpr80h` in VPC 80's VRF to `hpr80p` in the Envoy pod's `vrf80`; `hpr81h`, `hpr81p` and `vrf81` for VPC 81. Both use `fd20:80:ff04::/127` | + +The client's ends stand in for the far side of a physical link, which delivers +complete checksums. They send with checksum offload off; see +[checksums](#checksum-offload) for what changes when it is on. + +The addresses are documentation addresses under `2001:db8:4144::/48`, which no +other part of the lab uses and the node's FRR does not announce. + +## Prototype programs + +`tests/handoffs/probe/handoff.c` holds the prototypes. It includes the uSID +datapath source, so its policy decisions run the product's `tp_capture` and +`tp_decide` against the maps `galactic-cni` pinned. `handoff-probe` loads it with +every uSID map replaced by that pin, and refuses to load if a map it needs is +not pinned or does not match, so a run never creates or alters a product pin. +It places the programs with the product loaders: + +| Program | Hook | Loader | Role | +| ------------------ | --------------------------------------------------- | -------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `hp_xdp_public` | XDP of the receiving device | `xdpdispatch` slot `SlotGatewayLB`, on a private pin directory | A VIP packet gets metadata naming its TAP and `XDP_PASS`. A shard packet is translated back and redirected into the pair's NAT end. | +| `hp_tc_tenant_rx` | tc ingress of the receiving device | `attach.Attach` | Decides the TAP's ingress policy on the PublicService path, marks the packet decided, and delivers to the TAP with `bpf_redirect_neigh`. | +| `hp_tc_tap_rx` | tc ingress of the TAP, after galactic-cni's filters | netlink, one priority after the last galactic-cni filter | The VIP's reply leaves through the receiving device; any other flow to the client enters the pair. Each decides the TAP's egress policy. | +| `hp_xdp_pair_peer` | XDP of the pair's NAT end | `natattach.Attach` | Stands in for the NAT shard: translates the guest's source to the shard address and forwards; toward a VLAN it pushes the tag and redirects to the lower device. | +| `hp_tc_pair_rx` | tc ingress of the pair's tenant end | `attach.Attach` | Decides the TAP's ingress policy on the PublicNAT path, marks it, and delivers to the TAP. | +| `hp_tc_link_guard` | tc ingress of a routed link's host end | `attach.Attach` | Drops any source outside the link's `/127`. | + +The probe also writes Local pass-through entries (`egressroutemap`, under its +own owner) for the client's prefix and the routed link in both VRFs, which is +what Worker programs for a Local destination; see +[contract amendments](../../../docs/worker/contract.md#local-handoffs). + +## Cells + +Each cell builds one receiving device, places the prototypes in one mode, and +runs the `local.tap` window (`h1/`) and the `local.veth_pair` window +(`h2/`). `local.routed_link` and `local.socket` use no XDP hook and run +once (`h3`). + +| Cell | Receiving device | Dispatcher XDP | Peer XDP | Peer forwards with | `local.tap` | `local.veth_pair` | +| --------------------------- | ---------------- | ----------------- | -------- | ---------------------------------------------- | ----------- | --------------------------------------- | +| `plain-native` | veth | native | native | `XDP_REDIRECT` | delivers | delivers | +| `plain-generic` | veth | generic | generic | `XDP_REDIRECT` | delivers | fails: the peer's XDP never runs | +| `plain-generic-native-peer` | veth | generic | native | `XDP_REDIRECT` | delivers | delivers | +| `bond-native` | bond members | native | native | `XDP_REDIRECT` | delivers | delivers | +| `bond-generic-native-peer` | bond members | generic | native | `XDP_REDIRECT` | delivers | delivers | +| `vlan-native` | VLAN device | native: no attach | | | unsupported | unsupported | +| `vlan-lower-native` | VLAN's lower | native | native | tag pushed, `XDP_REDIRECT` to the lower device | delivers | delivers | +| `vlan-generic-native-peer` | VLAN device | generic | native | tag pushed, `XDP_REDIRECT` to the lower device | delivers | delivers | +| `vlan-generic-redirect` | VLAN device | generic | native | `XDP_REDIRECT` to the VLAN device | delivers | fails: no XDP transmit on a VLAN device | + +The two failing cells and `vlan-native` record the specified mechanism not +delivering; the cells after each are the alternatives the contract adopts. + +## Checks + +Each `local.tap` window sends 5 UDP requests to each VIP and one TCP request, +and checks: + +- each VPC's TAP guest receives its own VIP's requests and none of the other's, + and the client gets every answer, from the VIP, with no encapsulation; +- TCP to port 9 of a VIP reaches the receiving device and no guest, and a reply + to the client's UDP port 9 leaves the guest and never reaches the client + (the `handoff` set denies both on the PublicService path); +- a third VIP, whose tc program leaves the decision to `tp_deliver`, reaches no + guest: `tp_deliver` decides on the Private path, which the `handoff` set + grants only to the VPC's own addresses; +- the XDP metadata reached tc. + +Each `local.veth_pair` window sends 5 UDP flows and one TCP flow from each TAP +guest to the client, and checks that the client sees each from its VPC's shard +address, never the guest's, that each guest gets only its own answers, and that +UDP port 9 out and TCP port 9 back are denied on the PublicNAT path. + +The `h3` window checks that a socket bound to `vrf80` reads VPC 80's veth guest +and one bound to `vrf81` reads VPC 81's, at the same address; that each VPC's +guest reaches the socket listening in its own VRF; that port 9 is denied by the +guest's ingress policy; that an unbound socket reaches no guest; and that a +spoofed source crosses the link and is dropped by the guard. + +Every window also captures the node's four physical bond members, which must +carry nothing of the probes, plain or encapsulated in SRv6 or IPv6-in-IPv6. + +| Capture point | Interfaces | +| ------------------------------------- | ------------------------------------------ | +| `client` | The client's ends of the receiving device | +| `dut` | The receiving device's members on the node | +| `tap80-host`, `tap81-host` | The host TAPs | +| `tap80-guest`, `tap81-guest` | `tap0` in each TAP guest | +| `pair80`, `pair81` | Both ends of each internal veth pair | +| `veth80-guest`, `veth81-guest` | `eth0` of `ns80`'s guests | +| `link80-host`, `link81-host`, `envoy` | Both ends of the routed links | +| `physical` | `eth1` to `eth4` on `dfw-worker` | + +`scripts/handoff-pcap.py` counts the packets carrying each check's nonce. +`results.json` holds stable values only; the nonce and versions go to +`volatile.tsv`. + +## Measurements + +`scripts/handoff-measure.sh` records, for native XDP and for generic XDP with +a native peer: + +- `mtu`: UDP to a VIP at 1200 and 1451 bytes at MTU 1500 (each line also + carries a newline), at 1452 and 4000 bytes, which the client fragments, and + the receiving device at MTU 9000 against a TAP at 1500; +- `bulk`: 1 MiB each way, through `local.tap` and `local.veth_pair`, with the + client's checksum offload, and so TSO, off and on, with the largest frame XDP + saw; +- `pod-veth`: NAT flows from `ns80`'s veth guest through an internal veth pair, + with the guest's checksum offload on and off. + +The interface features of every rig interface are kept in `logs/`. + +Results, the same in every run so far: + +| Measurement | Native | Generic, native peer | +| ----------------------------------------------------------------------- | ------------------- | -------------------- | +| 1201- and 1452-byte UDP payloads to a VIP (MTU 1500) | answered | answered | +| 1453- and 4001-byte UDP payloads, fragmented by the client | not answered | not answered | +| 1 MiB client to VIP, checksum offload off, then on | intact, intact | intact, intact | +| 1 MiB client to guest through the pair, checksum offload off, then on | intact, corrupted | intact, corrupted | +| Largest frame XDP saw, offload on | 1514 | 34358 (GSO) | +| NAT flows from a veth pod, its offload off, then on | 3 of 3, 0 of 3 | 3 of 3, 0 of 3 | +| Receiving device at MTU 9000 | XDP does not attach | attaches | +| Unfragmented 1453- to 8001-byte payloads at MTU 9000 into a TAP at 1500 | | delivered, answered | + +The last row shows the TAP's MTU is not enforced on a redirect: the lab's +bridge hands such frames to the guest, which a real guest's driver would drop. + +### Checksum offload + +An XDP program that rewrites an address updates the L4 checksum as a complete +one. A frame that a sender on the same host sent through a veth with checksum +offload on carries a partial checksum instead, and XDP cannot tell the two +apart. After the rewrite and an `XDP_REDIRECT`, the frame reaches its receiver +with a wrong checksum and is dropped. A physical NIC delivers complete +checksums, which is why the rig's client sends with offload off; +`handoff-measure.sh` records both. + +## Updating the expected results + +After a deliberate change to the fixture, the prototypes or an expectation, +rerun with `HANDOFF_WRITE_EXPECTED=1 task handoff:twice`. The file is replaced +only when both runs pass and agree. Commit it with the change that caused it. diff --git a/deploy/containerlab/handoffs/expected-results.json b/deploy/containerlab/handoffs/expected-results.json new file mode 100644 index 00000000..83f76bd5 --- /dev/null +++ b/deploy/containerlab/handoffs/expected-results.json @@ -0,0 +1,2403 @@ +[ + { + "scenario": "cell/bond-generic-native-peer", + "check": "XDP attach", + "expect": "ok", + "got": "ok", + "result": "pass" + }, + { + "scenario": "cell/bond-native", + "check": "XDP attach", + "expect": "ok", + "got": "ok", + "result": "pass" + }, + { + "scenario": "cell/plain-generic", + "check": "XDP attach", + "expect": "ok", + "got": "ok", + "result": "pass" + }, + { + "scenario": "cell/plain-generic-native-peer", + "check": "XDP attach", + "expect": "ok", + "got": "ok", + "result": "pass" + }, + { + "scenario": "cell/plain-native", + "check": "XDP attach", + "expect": "ok", + "got": "ok", + "result": "pass" + }, + { + "scenario": "cell/select", + "check": "xdpmode.Select on dfw-worker", + "expect": "native", + "got": "native", + "result": "pass" + }, + { + "scenario": "cell/vlan-generic-native-peer", + "check": "XDP attach", + "expect": "ok", + "got": "ok", + "result": "pass" + }, + { + "scenario": "cell/vlan-generic-redirect", + "check": "XDP attach", + "expect": "ok", + "got": "ok", + "result": "pass" + }, + { + "scenario": "cell/vlan-lower-native", + "check": "XDP attach", + "expect": "ok", + "got": "ok", + "result": "pass" + }, + { + "scenario": "cell/vlan-native", + "check": "XDP attach", + "expect": "unsupported", + "got": "unsupported", + "result": "pass" + }, + { + "scenario": "h1/bond-generic-native-peer", + "check": "XDP metadata reaches tc", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/bond-generic-native-peer", + "check": "client gets tap80's TCP answer", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/bond-generic-native-peer", + "check": "client gets tap80's UDP answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/bond-generic-native-peer", + "check": "client gets tap81's UDP answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/bond-generic-native-peer", + "check": "client receives tap80's answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/bond-generic-native-peer", + "check": "client receives tap81's answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/bond-generic-native-peer", + "check": "denied TCP 9: reaches no guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/bond-generic-native-peer", + "check": "denied TCP 9: reaches the receiving device", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/bond-generic-native-peer", + "check": "denied UDP 9 reply: leaves the guest", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/bond-generic-native-peer", + "check": "denied UDP 9 reply: reaches no client", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/bond-generic-native-peer", + "check": "physical members carry the probes", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/bond-generic-native-peer", + "check": "physical members carry them encapsulated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/bond-generic-native-peer", + "check": "policy left to tp_deliver: reaches the guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/bond-generic-native-peer", + "check": "policy left to tp_deliver: reaches the receiving device", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/bond-generic-native-peer", + "check": "tap tc denies the reply on the PublicService path", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/bond-generic-native-peer", + "check": "tap80 guest receives its VIP's requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/bond-generic-native-peer", + "check": "tap80 guest receives tap81's requests", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/bond-generic-native-peer", + "check": "tap80's UDP listener answers inside the pod", + "expect": "-", + "got": "yes", + "result": "observed" + }, + { + "scenario": "h1/bond-generic-native-peer", + "check": "tap80's wildcard UDP listener answers the client", + "expect": "-", + "got": "no", + "result": "observed" + }, + { + "scenario": "h1/bond-generic-native-peer", + "check": "tap81 guest receives its VIP's requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/bond-generic-native-peer", + "check": "tap81 guest receives tap80's requests", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/bond-generic-native-peer", + "check": "tc denies on the PublicService path", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/bond-native", + "check": "XDP metadata reaches tc", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/bond-native", + "check": "client gets tap80's TCP answer", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/bond-native", + "check": "client gets tap80's UDP answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/bond-native", + "check": "client gets tap81's UDP answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/bond-native", + "check": "client receives tap80's answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/bond-native", + "check": "client receives tap81's answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/bond-native", + "check": "denied TCP 9: reaches no guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/bond-native", + "check": "denied TCP 9: reaches the receiving device", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/bond-native", + "check": "denied UDP 9 reply: leaves the guest", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/bond-native", + "check": "denied UDP 9 reply: reaches no client", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/bond-native", + "check": "physical members carry the probes", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/bond-native", + "check": "physical members carry them encapsulated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/bond-native", + "check": "policy left to tp_deliver: reaches the guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/bond-native", + "check": "policy left to tp_deliver: reaches the receiving device", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/bond-native", + "check": "tap tc denies the reply on the PublicService path", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/bond-native", + "check": "tap80 guest receives its VIP's requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/bond-native", + "check": "tap80 guest receives tap81's requests", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/bond-native", + "check": "tap80's UDP listener answers inside the pod", + "expect": "-", + "got": "yes", + "result": "observed" + }, + { + "scenario": "h1/bond-native", + "check": "tap80's wildcard UDP listener answers the client", + "expect": "-", + "got": "no", + "result": "observed" + }, + { + "scenario": "h1/bond-native", + "check": "tap81 guest receives its VIP's requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/bond-native", + "check": "tap81 guest receives tap80's requests", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/bond-native", + "check": "tc denies on the PublicService path", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/plain-generic", + "check": "XDP metadata reaches tc", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/plain-generic", + "check": "client gets tap80's TCP answer", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/plain-generic", + "check": "client gets tap80's UDP answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/plain-generic", + "check": "client gets tap81's UDP answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/plain-generic", + "check": "client receives tap80's answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/plain-generic", + "check": "client receives tap81's answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/plain-generic", + "check": "denied TCP 9: reaches no guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/plain-generic", + "check": "denied TCP 9: reaches the receiving device", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/plain-generic", + "check": "denied UDP 9 reply: leaves the guest", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/plain-generic", + "check": "denied UDP 9 reply: reaches no client", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/plain-generic", + "check": "physical members carry the probes", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/plain-generic", + "check": "physical members carry them encapsulated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/plain-generic", + "check": "policy left to tp_deliver: reaches the guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/plain-generic", + "check": "policy left to tp_deliver: reaches the receiving device", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/plain-generic", + "check": "tap tc denies the reply on the PublicService path", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/plain-generic", + "check": "tap80 guest receives its VIP's requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/plain-generic", + "check": "tap80 guest receives tap81's requests", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/plain-generic", + "check": "tap80's UDP listener answers inside the pod", + "expect": "-", + "got": "yes", + "result": "observed" + }, + { + "scenario": "h1/plain-generic", + "check": "tap80's wildcard UDP listener answers the client", + "expect": "-", + "got": "no", + "result": "observed" + }, + { + "scenario": "h1/plain-generic", + "check": "tap81 guest receives its VIP's requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/plain-generic", + "check": "tap81 guest receives tap80's requests", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/plain-generic", + "check": "tc denies on the PublicService path", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/plain-generic-native-peer", + "check": "XDP metadata reaches tc", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/plain-generic-native-peer", + "check": "client gets tap80's TCP answer", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/plain-generic-native-peer", + "check": "client gets tap80's UDP answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/plain-generic-native-peer", + "check": "client gets tap81's UDP answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/plain-generic-native-peer", + "check": "client receives tap80's answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/plain-generic-native-peer", + "check": "client receives tap81's answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/plain-generic-native-peer", + "check": "denied TCP 9: reaches no guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/plain-generic-native-peer", + "check": "denied TCP 9: reaches the receiving device", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/plain-generic-native-peer", + "check": "denied UDP 9 reply: leaves the guest", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/plain-generic-native-peer", + "check": "denied UDP 9 reply: reaches no client", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/plain-generic-native-peer", + "check": "physical members carry the probes", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/plain-generic-native-peer", + "check": "physical members carry them encapsulated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/plain-generic-native-peer", + "check": "policy left to tp_deliver: reaches the guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/plain-generic-native-peer", + "check": "policy left to tp_deliver: reaches the receiving device", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/plain-generic-native-peer", + "check": "tap tc denies the reply on the PublicService path", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/plain-generic-native-peer", + "check": "tap80 guest receives its VIP's requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/plain-generic-native-peer", + "check": "tap80 guest receives tap81's requests", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/plain-generic-native-peer", + "check": "tap80's UDP listener answers inside the pod", + "expect": "-", + "got": "yes", + "result": "observed" + }, + { + "scenario": "h1/plain-generic-native-peer", + "check": "tap80's wildcard UDP listener answers the client", + "expect": "-", + "got": "no", + "result": "observed" + }, + { + "scenario": "h1/plain-generic-native-peer", + "check": "tap81 guest receives its VIP's requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/plain-generic-native-peer", + "check": "tap81 guest receives tap80's requests", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/plain-generic-native-peer", + "check": "tc denies on the PublicService path", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/plain-native", + "check": "XDP metadata reaches tc", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/plain-native", + "check": "client gets tap80's TCP answer", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/plain-native", + "check": "client gets tap80's UDP answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/plain-native", + "check": "client gets tap81's UDP answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/plain-native", + "check": "client receives tap80's answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/plain-native", + "check": "client receives tap81's answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/plain-native", + "check": "denied TCP 9: reaches no guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/plain-native", + "check": "denied TCP 9: reaches the receiving device", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/plain-native", + "check": "denied UDP 9 reply: leaves the guest", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/plain-native", + "check": "denied UDP 9 reply: reaches no client", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/plain-native", + "check": "physical members carry the probes", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/plain-native", + "check": "physical members carry them encapsulated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/plain-native", + "check": "policy left to tp_deliver: reaches the guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/plain-native", + "check": "policy left to tp_deliver: reaches the receiving device", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/plain-native", + "check": "tap tc denies the reply on the PublicService path", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/plain-native", + "check": "tap80 guest receives its VIP's requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/plain-native", + "check": "tap80 guest receives tap81's requests", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/plain-native", + "check": "tap80's UDP listener answers inside the pod", + "expect": "-", + "got": "yes", + "result": "observed" + }, + { + "scenario": "h1/plain-native", + "check": "tap80's wildcard UDP listener answers the client", + "expect": "-", + "got": "no", + "result": "observed" + }, + { + "scenario": "h1/plain-native", + "check": "tap81 guest receives its VIP's requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/plain-native", + "check": "tap81 guest receives tap80's requests", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/plain-native", + "check": "tc denies on the PublicService path", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-native-peer", + "check": "XDP metadata reaches tc", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-native-peer", + "check": "client gets tap80's TCP answer", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-native-peer", + "check": "client gets tap80's UDP answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-native-peer", + "check": "client gets tap81's UDP answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-native-peer", + "check": "client receives tap80's answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-native-peer", + "check": "client receives tap81's answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-native-peer", + "check": "denied TCP 9: reaches no guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-native-peer", + "check": "denied TCP 9: reaches the receiving device", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-native-peer", + "check": "denied UDP 9 reply: leaves the guest", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-native-peer", + "check": "denied UDP 9 reply: reaches no client", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-native-peer", + "check": "physical members carry the probes", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-native-peer", + "check": "physical members carry them encapsulated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-native-peer", + "check": "policy left to tp_deliver: reaches the guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-native-peer", + "check": "policy left to tp_deliver: reaches the receiving device", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-native-peer", + "check": "tap tc denies the reply on the PublicService path", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-native-peer", + "check": "tap80 guest receives its VIP's requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-native-peer", + "check": "tap80 guest receives tap81's requests", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-native-peer", + "check": "tap80's UDP listener answers inside the pod", + "expect": "-", + "got": "yes", + "result": "observed" + }, + { + "scenario": "h1/vlan-generic-native-peer", + "check": "tap80's wildcard UDP listener answers the client", + "expect": "-", + "got": "no", + "result": "observed" + }, + { + "scenario": "h1/vlan-generic-native-peer", + "check": "tap81 guest receives its VIP's requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-native-peer", + "check": "tap81 guest receives tap80's requests", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-native-peer", + "check": "tc denies on the PublicService path", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-redirect", + "check": "XDP metadata reaches tc", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-redirect", + "check": "client gets tap80's TCP answer", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-redirect", + "check": "client gets tap80's UDP answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-redirect", + "check": "client gets tap81's UDP answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-redirect", + "check": "client receives tap80's answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-redirect", + "check": "client receives tap81's answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-redirect", + "check": "denied TCP 9: reaches no guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-redirect", + "check": "denied TCP 9: reaches the receiving device", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-redirect", + "check": "denied UDP 9 reply: leaves the guest", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-redirect", + "check": "denied UDP 9 reply: reaches no client", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-redirect", + "check": "physical members carry the probes", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-redirect", + "check": "physical members carry them encapsulated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-redirect", + "check": "policy left to tp_deliver: reaches the guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-redirect", + "check": "policy left to tp_deliver: reaches the receiving device", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-redirect", + "check": "tap tc denies the reply on the PublicService path", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-redirect", + "check": "tap80 guest receives its VIP's requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-redirect", + "check": "tap80 guest receives tap81's requests", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-redirect", + "check": "tap80's UDP listener answers inside the pod", + "expect": "-", + "got": "yes", + "result": "observed" + }, + { + "scenario": "h1/vlan-generic-redirect", + "check": "tap80's wildcard UDP listener answers the client", + "expect": "-", + "got": "no", + "result": "observed" + }, + { + "scenario": "h1/vlan-generic-redirect", + "check": "tap81 guest receives its VIP's requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-redirect", + "check": "tap81 guest receives tap80's requests", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/vlan-generic-redirect", + "check": "tc denies on the PublicService path", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/vlan-lower-native", + "check": "XDP metadata reaches tc", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/vlan-lower-native", + "check": "client gets tap80's TCP answer", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/vlan-lower-native", + "check": "client gets tap80's UDP answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/vlan-lower-native", + "check": "client gets tap81's UDP answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/vlan-lower-native", + "check": "client receives tap80's answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/vlan-lower-native", + "check": "client receives tap81's answers from its VIP", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/vlan-lower-native", + "check": "denied TCP 9: reaches no guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/vlan-lower-native", + "check": "denied TCP 9: reaches the receiving device", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/vlan-lower-native", + "check": "denied UDP 9 reply: leaves the guest", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/vlan-lower-native", + "check": "denied UDP 9 reply: reaches no client", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/vlan-lower-native", + "check": "physical members carry the probes", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/vlan-lower-native", + "check": "physical members carry them encapsulated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/vlan-lower-native", + "check": "policy left to tp_deliver: reaches the guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/vlan-lower-native", + "check": "policy left to tp_deliver: reaches the receiving device", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/vlan-lower-native", + "check": "tap tc denies the reply on the PublicService path", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h1/vlan-lower-native", + "check": "tap80 guest receives its VIP's requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/vlan-lower-native", + "check": "tap80 guest receives tap81's requests", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/vlan-lower-native", + "check": "tap80's UDP listener answers inside the pod", + "expect": "-", + "got": "yes", + "result": "observed" + }, + { + "scenario": "h1/vlan-lower-native", + "check": "tap80's wildcard UDP listener answers the client", + "expect": "-", + "got": "no", + "result": "observed" + }, + { + "scenario": "h1/vlan-lower-native", + "check": "tap81 guest receives its VIP's requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h1/vlan-lower-native", + "check": "tap81 guest receives tap80's requests", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h1/vlan-lower-native", + "check": "tc denies on the PublicService path", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/bond-generic-native-peer", + "check": "client receives tap80's flow from its shard address", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h2/bond-generic-native-peer", + "check": "client receives tap80's flow untranslated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/bond-generic-native-peer", + "check": "client receives tap81's flow from its shard address", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h2/bond-generic-native-peer", + "check": "client receives tap81's flow untranslated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/bond-generic-native-peer", + "check": "denied TCP 9 return: leaves the client", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/bond-generic-native-peer", + "check": "denied TCP 9 return: reaches no guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/bond-generic-native-peer", + "check": "denied TCP 9 return: reaches the pair's tenant end", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/bond-generic-native-peer", + "check": "denied UDP 9: leaves the guest", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/bond-generic-native-peer", + "check": "denied UDP 9: reaches no client", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/bond-generic-native-peer", + "check": "pair peer XDP translates", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/bond-generic-native-peer", + "check": "physical members carry the probes", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/bond-generic-native-peer", + "check": "physical members carry them encapsulated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/bond-generic-native-peer", + "check": "tap80 gets the client's TCP answer", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/bond-generic-native-peer", + "check": "tap80 gets the client's UDP answers", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h2/bond-generic-native-peer", + "check": "tap80 guest receives tap81's answers", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/bond-generic-native-peer", + "check": "tap81 gets the client's UDP answers", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h2/bond-generic-native-peer", + "check": "tap81 guest receives tap80's answers", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/bond-native", + "check": "client receives tap80's flow from its shard address", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h2/bond-native", + "check": "client receives tap80's flow untranslated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/bond-native", + "check": "client receives tap81's flow from its shard address", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h2/bond-native", + "check": "client receives tap81's flow untranslated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/bond-native", + "check": "denied TCP 9 return: leaves the client", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/bond-native", + "check": "denied TCP 9 return: reaches no guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/bond-native", + "check": "denied TCP 9 return: reaches the pair's tenant end", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/bond-native", + "check": "denied UDP 9: leaves the guest", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/bond-native", + "check": "denied UDP 9: reaches no client", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/bond-native", + "check": "pair peer XDP translates", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/bond-native", + "check": "physical members carry the probes", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/bond-native", + "check": "physical members carry them encapsulated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/bond-native", + "check": "tap80 gets the client's TCP answer", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/bond-native", + "check": "tap80 gets the client's UDP answers", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h2/bond-native", + "check": "tap80 guest receives tap81's answers", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/bond-native", + "check": "tap81 gets the client's UDP answers", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h2/bond-native", + "check": "tap81 guest receives tap80's answers", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-generic", + "check": "client receives tap80's flow from its shard address", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-generic", + "check": "client receives tap80's flow untranslated", + "expect": "-", + "got": "5", + "result": "observed" + }, + { + "scenario": "h2/plain-generic", + "check": "client receives tap81's flow from its shard address", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-generic", + "check": "client receives tap81's flow untranslated", + "expect": "-", + "got": "5", + "result": "observed" + }, + { + "scenario": "h2/plain-generic", + "check": "denied TCP 9 return: leaves the client", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/plain-generic", + "check": "denied TCP 9 return: reaches no guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-generic", + "check": "denied TCP 9 return: reaches the pair's tenant end", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/plain-generic", + "check": "denied UDP 9: leaves the guest", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/plain-generic", + "check": "denied UDP 9: reaches no client", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-generic", + "check": "pair peer XDP translates", + "expect": "no", + "got": "no", + "result": "pass" + }, + { + "scenario": "h2/plain-generic", + "check": "physical members carry the probes", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-generic", + "check": "physical members carry them encapsulated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-generic", + "check": "tap80 gets the client's TCP answer", + "expect": "no", + "got": "no", + "result": "pass" + }, + { + "scenario": "h2/plain-generic", + "check": "tap80 gets the client's UDP answers", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-generic", + "check": "tap80 guest receives tap81's answers", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-generic", + "check": "tap81 gets the client's UDP answers", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-generic", + "check": "tap81 guest receives tap80's answers", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-generic-native-peer", + "check": "client receives tap80's flow from its shard address", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h2/plain-generic-native-peer", + "check": "client receives tap80's flow untranslated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-generic-native-peer", + "check": "client receives tap81's flow from its shard address", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h2/plain-generic-native-peer", + "check": "client receives tap81's flow untranslated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-generic-native-peer", + "check": "denied TCP 9 return: leaves the client", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/plain-generic-native-peer", + "check": "denied TCP 9 return: reaches no guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-generic-native-peer", + "check": "denied TCP 9 return: reaches the pair's tenant end", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/plain-generic-native-peer", + "check": "denied UDP 9: leaves the guest", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/plain-generic-native-peer", + "check": "denied UDP 9: reaches no client", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-generic-native-peer", + "check": "pair peer XDP translates", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/plain-generic-native-peer", + "check": "physical members carry the probes", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-generic-native-peer", + "check": "physical members carry them encapsulated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-generic-native-peer", + "check": "tap80 gets the client's TCP answer", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/plain-generic-native-peer", + "check": "tap80 gets the client's UDP answers", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h2/plain-generic-native-peer", + "check": "tap80 guest receives tap81's answers", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-generic-native-peer", + "check": "tap81 gets the client's UDP answers", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h2/plain-generic-native-peer", + "check": "tap81 guest receives tap80's answers", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-native", + "check": "client receives tap80's flow from its shard address", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h2/plain-native", + "check": "client receives tap80's flow untranslated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-native", + "check": "client receives tap81's flow from its shard address", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h2/plain-native", + "check": "client receives tap81's flow untranslated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-native", + "check": "denied TCP 9 return: leaves the client", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/plain-native", + "check": "denied TCP 9 return: reaches no guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-native", + "check": "denied TCP 9 return: reaches the pair's tenant end", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/plain-native", + "check": "denied UDP 9: leaves the guest", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/plain-native", + "check": "denied UDP 9: reaches no client", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-native", + "check": "pair peer XDP translates", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/plain-native", + "check": "physical members carry the probes", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-native", + "check": "physical members carry them encapsulated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-native", + "check": "tap80 gets the client's TCP answer", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/plain-native", + "check": "tap80 gets the client's UDP answers", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h2/plain-native", + "check": "tap80 guest receives tap81's answers", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/plain-native", + "check": "tap81 gets the client's UDP answers", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h2/plain-native", + "check": "tap81 guest receives tap80's answers", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-native-peer", + "check": "client receives tap80's flow from its shard address", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-native-peer", + "check": "client receives tap80's flow untranslated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-native-peer", + "check": "client receives tap81's flow from its shard address", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-native-peer", + "check": "client receives tap81's flow untranslated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-native-peer", + "check": "denied TCP 9 return: leaves the client", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-native-peer", + "check": "denied TCP 9 return: reaches no guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-native-peer", + "check": "denied TCP 9 return: reaches the pair's tenant end", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-native-peer", + "check": "denied UDP 9: leaves the guest", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-native-peer", + "check": "denied UDP 9: reaches no client", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-native-peer", + "check": "pair peer XDP translates", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-native-peer", + "check": "physical members carry the probes", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-native-peer", + "check": "physical members carry them encapsulated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-native-peer", + "check": "tap80 gets the client's TCP answer", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-native-peer", + "check": "tap80 gets the client's UDP answers", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-native-peer", + "check": "tap80 guest receives tap81's answers", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-native-peer", + "check": "tap81 gets the client's UDP answers", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-native-peer", + "check": "tap81 guest receives tap80's answers", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-redirect", + "check": "client receives tap80's flow from its shard address", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-redirect", + "check": "client receives tap80's flow untranslated", + "expect": "-", + "got": "0", + "result": "observed" + }, + { + "scenario": "h2/vlan-generic-redirect", + "check": "client receives tap81's flow from its shard address", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-redirect", + "check": "client receives tap81's flow untranslated", + "expect": "-", + "got": "0", + "result": "observed" + }, + { + "scenario": "h2/vlan-generic-redirect", + "check": "denied TCP 9 return: leaves the client", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-redirect", + "check": "denied TCP 9 return: reaches no guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-redirect", + "check": "denied TCP 9 return: reaches the pair's tenant end", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-redirect", + "check": "denied UDP 9: leaves the guest", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-redirect", + "check": "denied UDP 9: reaches no client", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-redirect", + "check": "pair peer XDP translates", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-redirect", + "check": "physical members carry the probes", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-redirect", + "check": "physical members carry them encapsulated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-redirect", + "check": "tap80 gets the client's TCP answer", + "expect": "no", + "got": "no", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-redirect", + "check": "tap80 gets the client's UDP answers", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-redirect", + "check": "tap80 guest receives tap81's answers", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-redirect", + "check": "tap81 gets the client's UDP answers", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-generic-redirect", + "check": "tap81 guest receives tap80's answers", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-lower-native", + "check": "client receives tap80's flow from its shard address", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h2/vlan-lower-native", + "check": "client receives tap80's flow untranslated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-lower-native", + "check": "client receives tap81's flow from its shard address", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h2/vlan-lower-native", + "check": "client receives tap81's flow untranslated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-lower-native", + "check": "denied TCP 9 return: leaves the client", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/vlan-lower-native", + "check": "denied TCP 9 return: reaches no guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-lower-native", + "check": "denied TCP 9 return: reaches the pair's tenant end", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/vlan-lower-native", + "check": "denied UDP 9: leaves the guest", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/vlan-lower-native", + "check": "denied UDP 9: reaches no client", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-lower-native", + "check": "pair peer XDP translates", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/vlan-lower-native", + "check": "physical members carry the probes", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-lower-native", + "check": "physical members carry them encapsulated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-lower-native", + "check": "tap80 gets the client's TCP answer", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h2/vlan-lower-native", + "check": "tap80 gets the client's UDP answers", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h2/vlan-lower-native", + "check": "tap80 guest receives tap81's answers", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h2/vlan-lower-native", + "check": "tap81 gets the client's UDP answers", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h2/vlan-lower-native", + "check": "tap81 guest receives tap80's answers", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h3", + "check": "denied TCP 9: crosses the routed link", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h3", + "check": "denied TCP 9: reaches no guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h3", + "check": "link guard drops the spoofed source", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h3", + "check": "physical members carry the probes", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h3", + "check": "physical members carry them encapsulated", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h3", + "check": "socket bound to vrf80 reads vpc80's guest", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h3", + "check": "socket bound to vrf80 reads vpc81's guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h3", + "check": "socket bound to vrf81 reads vpc80's guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h3", + "check": "socket bound to vrf81 reads vpc81's guest", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h3", + "check": "spoofed source: crosses the routed link", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "h3", + "check": "spoofed source: reaches no guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h3", + "check": "unbound socket: gets no answer", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h3", + "check": "unbound socket: reaches no guest", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h3", + "check": "vpc80's guest is answered by the vrf80 socket", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h3", + "check": "vpc80's guest receives vrf80's requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h3", + "check": "vpc80's guest receives vrf81's requests", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h3", + "check": "vpc81's guest is answered by the vrf81 socket", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h3", + "check": "vpc81's guest receives vrf80's requests", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "h3", + "check": "vpc81's guest receives vrf81's requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h3", + "check": "vrf80 socket reads the guests' lines", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "h3", + "check": "vrf81 socket reads the guests' lines", + "expect": "5", + "got": "5", + "result": "pass" + } +] diff --git a/deploy/containerlab/resources/tenants/ns80-tap/base/guests.yaml b/deploy/containerlab/resources/tenants/ns80-tap/base/guests.yaml new file mode 100644 index 00000000..3c8a6980 --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80-tap/base/guests.yaml @@ -0,0 +1,200 @@ +# A TAP-backed guest in VPC 80. The pod's primary network is the cluster's, +# so the runtime has a pod interface and the sandbox starts. vpc-a-tap is a +# secondary attachment: galactic-tap creates the host tap G000000080082H in the VPC's +# VRF and configures nothing in the pod. The vmm container plays the +# hypervisor: it opens that tap and bridges it to tap0 in the pod, the +# guest's interface, with socat. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tap-guest-a + labels: + app: vpc-a-tap +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: vpc-a-tap + template: + metadata: + labels: + app: vpc-a-tap + annotations: + k8s.v1.cni.cncf.io/networks: vpc-a-tap + spec: + nodeSelector: + galactic.datumapis.com/node: compute + hostPID: true + terminationGracePeriodSeconds: 1 + containers: + - name: vmm + image: ghcr.io/datum-cloud/galactic-debug@sha256:b64e338f845d06f82f2575ae6bb6855da330af7551a8816b7ade2602a85722f2 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + env: + - name: TAP + value: G000000080082H + - name: GUEST + value: fd20:80:ff03::1:0:0/96 + - name: GATEWAY + value: fd20:80:ff03::1 + - name: VIP + value: "2001:db8:4144:a::80" + - name: PUBLIC + value: 2001:db8:4144::/48 + command: ["sh", "-c"] + args: + - | + set -eu + # Name the node's network namespace for socat's netns= option. hostPID makes + # PID 1 the node's init. + [ -e /run/netns/host ] || ip netns attach host 1 + i=0 + until ip netns exec host ip link show "${TAP}" >/dev/null 2>&1; do + i=$((i + 1)) + if [ "${i}" -ge 120 ]; then + echo "vmm: host tap ${TAP} did not appear" >&2 + exit 1 + fi + sleep 1 + done + socat -d TUN,tun-type=tap,tun-name="${TAP}",iff-no-pi,netns=host \ + TUN,tun-type=tap,tun-name=tap0,iff-no-pi,iff-up & + bridge=$! + i=0 + until ip link show tap0 >/dev/null 2>&1; do + i=$((i + 1)) + if [ "${i}" -ge 100 ]; then + echo "vmm: tap0 did not appear" >&2 + exit 1 + fi + sleep 0.1 + done + # Routes only after the bridge holds tap0, or the kernel drops them with it. + ip -6 addr replace "${GUEST}" dev tap0 nodad + ip -6 addr replace "${VIP}/128" dev lo + ip -6 route replace fd20:80::/32 via "${GATEWAY}" dev tap0 onlink + ip -6 route replace "${PUBLIC}" via "${GATEWAY}" dev tap0 onlink + touch /tmp/ready + wait "${bridge}" + readinessProbe: + exec: + command: ["sh", "-c", "test -f /tmp/ready && ip link show tap0 | grep -q LOWER_UP"] + periodSeconds: 2 +--- +# A TAP-backed guest in VPC 81. The pod's primary network is the cluster's, +# so the runtime has a pod interface and the sandbox starts. vpc-b-tap is a +# secondary attachment: galactic-tap creates the host tap G000000081082H in the VPC's +# VRF and configures nothing in the pod. The vmm container plays the +# hypervisor: it opens that tap and bridges it to tap0 in the pod, the +# guest's interface, with socat. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tap-guest-b + labels: + app: vpc-b-tap +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: vpc-b-tap + template: + metadata: + labels: + app: vpc-b-tap + annotations: + k8s.v1.cni.cncf.io/networks: vpc-b-tap + spec: + nodeSelector: + galactic.datumapis.com/node: compute + hostPID: true + terminationGracePeriodSeconds: 1 + containers: + - name: vmm + image: ghcr.io/datum-cloud/galactic-debug@sha256:b64e338f845d06f82f2575ae6bb6855da330af7551a8816b7ade2602a85722f2 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + env: + - name: TAP + value: G000000081082H + - name: GUEST + value: fd20:80:ff03::1:0:0/96 + - name: GATEWAY + value: fd20:80:ff03::1 + - name: VIP + value: "2001:db8:4144:a::81" + - name: PUBLIC + value: 2001:db8:4144::/48 + command: ["sh", "-c"] + args: + - | + set -eu + # Name the node's network namespace for socat's netns= option. hostPID makes + # PID 1 the node's init. + [ -e /run/netns/host ] || ip netns attach host 1 + i=0 + until ip netns exec host ip link show "${TAP}" >/dev/null 2>&1; do + i=$((i + 1)) + if [ "${i}" -ge 120 ]; then + echo "vmm: host tap ${TAP} did not appear" >&2 + exit 1 + fi + sleep 1 + done + socat -d TUN,tun-type=tap,tun-name="${TAP}",iff-no-pi,netns=host \ + TUN,tun-type=tap,tun-name=tap0,iff-no-pi,iff-up & + bridge=$! + i=0 + until ip link show tap0 >/dev/null 2>&1; do + i=$((i + 1)) + if [ "${i}" -ge 100 ]; then + echo "vmm: tap0 did not appear" >&2 + exit 1 + fi + sleep 0.1 + done + # Routes only after the bridge holds tap0, or the kernel drops them with it. + ip -6 addr replace "${GUEST}" dev tap0 nodad + ip -6 addr replace "${VIP}/128" dev lo + ip -6 route replace fd20:80::/32 via "${GATEWAY}" dev tap0 onlink + ip -6 route replace "${PUBLIC}" via "${GATEWAY}" dev tap0 onlink + touch /tmp/ready + wait "${bridge}" + readinessProbe: + exec: + command: ["sh", "-c", "test -f /tmp/ready && ip link show tap0 | grep -q LOWER_UP"] + periodSeconds: 2 +--- +# Stands in for an Envoy pod. The handoff run gives it a VRF per VPC and a +# routed link from each to the host's VRF, and opens VRF-bound sockets in it. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: envoy-standin + labels: + app: envoy-standin +spec: + replicas: 1 + selector: + matchLabels: + app: envoy-standin + template: + metadata: + labels: + app: envoy-standin + spec: + nodeSelector: + galactic.datumapis.com/node: compute + terminationGracePeriodSeconds: 1 + containers: + - name: envoy + image: ghcr.io/datum-cloud/galactic-debug@sha256:b64e338f845d06f82f2575ae6bb6855da330af7551a8816b7ade2602a85722f2 + imagePullPolicy: IfNotPresent + command: ["sleep", "infinity"] diff --git a/deploy/containerlab/resources/tenants/ns80-tap/base/kustomization.yaml b/deploy/containerlab/resources/tenants/ns80-tap/base/kustomization.yaml new file mode 100644 index 00000000..daafa356 --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80-tap/base/kustomization.yaml @@ -0,0 +1,9 @@ +# ns80-tap: the Local handoff fixture (docs/handoffs.md). Two TAP-backed +# guests on dfw's compute worker, one in each of ns80's VPCs, with the same +# address, and a pod standing in for Envoy. It is deployed only for a handoff +# run (task deploy:ns80-tap) and removed after it, so the ns80 baseline +# (docs/baseline.md) never sees it. +namespace: ns80-tap +resources: + - namespace.yaml + - guests.yaml diff --git a/deploy/containerlab/resources/tenants/ns80-tap/base/namespace.yaml b/deploy/containerlab/resources/tenants/ns80-tap/base/namespace.yaml new file mode 100644 index 00000000..7a9ab83a --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80-tap/base/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ns80-tap diff --git a/deploy/containerlab/resources/tenants/ns80-tap/dfw/kustomization.yaml b/deploy/containerlab/resources/tenants/ns80-tap/dfw/kustomization.yaml new file mode 100644 index 00000000..ee81d25b --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80-tap/dfw/kustomization.yaml @@ -0,0 +1,3 @@ +resources: + - ../base + - nad.yaml diff --git a/deploy/containerlab/resources/tenants/ns80-tap/dfw/nad.yaml b/deploy/containerlab/resources/tenants/ns80-tap/dfw/nad.yaml new file mode 100644 index 00000000..805fbe0a --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80-tap/dfw/nad.yaml @@ -0,0 +1,71 @@ +# dfw: one TAP attachment in each of ns80's VPCs, with the same address, +# fd20:80:ff03::1:0:0, as secondary attachments of the TAP guests. +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: vpc-a-tap + namespace: ns80-tap +spec: + config: |- + { + "cniVersion": "1.0.0", + "name": "vpc-a-tap", + "plugins": [ + { + "type": "galactic-tap", + "vpc": "80", + "vpcattachment": "82", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "addresses": [ + { + "address": "fd20:80:ff03:0:0:1::/96", + "gateway": "fd20:80:ff03::1" + } + ] + } + }, + { + "type": "galactic-bgp", + "vpc": "80", + "vpcattachment": "82", + "namespace": "galactic-system" + } + ] + } +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: vpc-b-tap + namespace: ns80-tap +spec: + config: |- + { + "cniVersion": "1.0.0", + "name": "vpc-b-tap", + "plugins": [ + { + "type": "galactic-tap", + "vpc": "81", + "vpcattachment": "82", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "addresses": [ + { + "address": "fd20:80:ff03:0:0:1::/96", + "gateway": "fd20:80:ff03::1" + } + ] + } + }, + { + "type": "galactic-bgp", + "vpc": "81", + "vpcattachment": "82", + "namespace": "galactic-system" + } + ] + } diff --git a/deploy/containerlab/resources/tenants/ns80-tap/policy/bindings.yaml b/deploy/containerlab/resources/tenants/ns80-tap/policy/bindings.yaml new file mode 100644 index 00000000..987dc6dd --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80-tap/policy/bindings.yaml @@ -0,0 +1,19 @@ +apiVersion: galactic.datumapis.com/v1alpha1 +kind: TrafficPolicyBinding +metadata: + name: vpc-a-tap +spec: + ingressPolicyRef: + name: handoff-ingress + egressPolicyRef: + name: handoff-egress +--- +apiVersion: galactic.datumapis.com/v1alpha1 +kind: TrafficPolicyBinding +metadata: + name: vpc-b-tap +spec: + ingressPolicyRef: + name: handoff-ingress + egressPolicyRef: + name: handoff-egress diff --git a/deploy/containerlab/resources/tenants/ns80-tap/policy/kustomization.yaml b/deploy/containerlab/resources/tenants/ns80-tap/policy/kustomization.yaml new file mode 100644 index 00000000..59aa0c0b --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80-tap/policy/kustomization.yaml @@ -0,0 +1,8 @@ +# ns80-tap's traffic policy: the handoff set +# (../../policies/README.md), bound to both TAP attachments in both +# directions. A binding is named after its VPCAttachment, which +# scripts/publish-attachments.sh creates. +namespace: ns80-tap +resources: + - ../../policies/handoff + - bindings.yaml diff --git a/deploy/containerlab/resources/tenants/policies/README.md b/deploy/containerlab/resources/tenants/policies/README.md index ece9a1c0..52a67554 100644 --- a/deploy/containerlab/resources/tenants/policies/README.md +++ b/deploy/containerlab/resources/tenants/policies/README.md @@ -5,13 +5,16 @@ in `resources/galactic-cni/shared/`), so a tenant interface forwards only what its bindings grant. These are the grants, one set per kind of tenant, each an ingress and an egress policy granting only the paths that tenant's checks use: -| Set | Tenants | Egress grants | Ingress grants | -| ---------------- | ---------------------- | ------------------------------------------------------ | ------------------------------------------------------ | -| `private-nat` | ns10, ns20, ns30, ns71 | Private, PublicNAT | Private (except TCP 9), PublicNAT | -| `private` | ns40 | Private | Private (except TCP 9) | -| `public-service` | ns60 | PublicService, Private TCP from port 80 to `fe80::/10` | PublicService, Private TCP to port 80 from `fe80::/10` | +| Set | Tenants | Egress grants | Ingress grants | +| ---------------- | ---------------------- | ------------------------------------------------------ | ---------------------------------------------------------------------------------------------- | +| `private-nat` | ns10, ns20, ns30, ns71 | Private, PublicNAT | Private (except TCP 9), PublicNAT | +| `private` | ns40 | Private | Private (except TCP 9) | +| `public-service` | ns60 | PublicService, Private TCP from port 80 to `fe80::/10` | PublicService, Private TCP to port 80 from `fe80::/10` | +| `handoff` | ns80-tap | PublicService and PublicNAT (except UDP 9), Private | PublicService and PublicNAT (except TCP 9), Private from the VPC (`fd20:80::/32`, `fe80::/10`) | The TCP 9 deny on Private ingress, and every path a set leaves out, are what -`verify:traffic-policy` uses to prove a denial is enforced and counted. Each +`verify:traffic-policy` uses to prove a denial is enforced and counted. The +`handoff` set's port 9 denials and its Private ingress limit are the +policy-denied controls of the Local handoff checks (`docs/handoffs.md`). Each tenant's `/policy/` applies one set into its namespace and binds every attachment the tenant has to it, in both directions. diff --git a/deploy/containerlab/resources/tenants/policies/handoff/kustomization.yaml b/deploy/containerlab/resources/tenants/policies/handoff/kustomization.yaml new file mode 100644 index 00000000..1bf9e66a --- /dev/null +++ b/deploy/containerlab/resources/tenants/policies/handoff/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - policies.yaml diff --git a/deploy/containerlab/resources/tenants/policies/handoff/policies.yaml b/deploy/containerlab/resources/tenants/policies/handoff/policies.yaml new file mode 100644 index 00000000..3a17c6fc --- /dev/null +++ b/deploy/containerlab/resources/tenants/policies/handoff/policies.yaml @@ -0,0 +1,77 @@ +# The TAP guests of the Local handoff fixture (ns80-tap, docs/handoffs.md). +# They stand in for a gateway backend and a NAT client at once, so each +# direction grants PublicService and PublicNAT, and denies one port on each +# as the policy-denied control. Private ingress comes only from the VPC +# itself, so a public client delivered without a PublicService decision is +# denied. Private egress is open: usid_egress decides a Local pass-through +# destination on the Private path before the handoff's own decision runs. +apiVersion: galactic.datumapis.com/v1alpha1 +kind: TrafficPolicy +metadata: + name: handoff-egress +spec: + rules: + - name: deny-service-discard + action: Deny + match: + path: PublicService + protocol: UDP + destinationPorts: + - start: 9 + end: 9 + - name: allow-service + action: Allow + match: + path: PublicService + - name: deny-nat-discard + action: Deny + match: + path: PublicNAT + protocol: UDP + destinationPorts: + - start: 9 + end: 9 + - name: allow-nat + action: Allow + match: + path: PublicNAT + - name: allow-private + action: Allow + match: + path: Private +--- +apiVersion: galactic.datumapis.com/v1alpha1 +kind: TrafficPolicy +metadata: + name: handoff-ingress +spec: + rules: + - name: deny-service-discard + action: Deny + match: + path: PublicService + protocol: TCP + destinationPorts: + - start: 9 + end: 9 + - name: allow-service + action: Allow + match: + path: PublicService + - name: deny-nat-discard + action: Deny + match: + path: PublicNAT + protocol: TCP + destinationPorts: + - start: 9 + end: 9 + - name: allow-nat-returns + action: Allow + match: + path: PublicNAT + - name: allow-vpc + action: Allow + match: + path: Private + sourceCIDRs: ["fd20:80::/32", "fe80::/10"] diff --git a/deploy/containerlab/scripts/handoff-echo.py b/deploy/containerlab/scripts/handoff-echo.py new file mode 100755 index 00000000..4067a4e6 --- /dev/null +++ b/deploy/containerlab/scripts/handoff-echo.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +# Copyright 2026 Datum Cloud, Inc. +# +# SPDX-License-Identifier: AGPL-3.0-or-later +"""A UDP echo server for the Local handoff checks. + +Usage: handoff-echo.py ADDRESS PORT REPLY + +Answers each datagram's first line, cut to 200 bytes so a large request +gets a small answer, with " REPLY", from ADDRESS, so a listener bound +to a VIP answers from the VIP. An empty datagram, which socat +sends at the end of its input, gets no answer. +""" + +import socket +import sys + + +def main(): + addr, port, reply = sys.argv[1], int(sys.argv[2]), sys.argv[3].encode() + s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind((addr, port)) + while True: + data, peer = s.recvfrom(65535) + if not data: + continue + try: + s.sendto(data.split(b"\n", 1)[0][:200] + b" " + reply + b"\n", peer) + except OSError: + # The rig changes routes between cells; a late datagram whose + # answer has no route must not end the server. + continue + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/deploy/containerlab/scripts/handoff-fixture.sh b/deploy/containerlab/scripts/handoff-fixture.sh new file mode 100755 index 00000000..da8e3ea6 --- /dev/null +++ b/deploy/containerlab/scripts/handoff-fixture.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# handoff-fixture.sh: deploy or remove the Local handoff fixture, ns80-tap +# (resources/tenants/ns80-tap, docs/handoffs.md), on dfw. +# +# Usage: handoff-fixture.sh up|down +# +# up applies its traffic policy, its pods and its attachments' status, then +# waits until both TAP guests have bridged their TAP and the Envoy stand-in +# runs. down deletes the namespace and waits until galactic-tap has removed +# both host TAPs, so nothing of it is left on dfw-worker. +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +source "${SCRIPT_DIR}/lib.sh" + +NS=ns80-tap +CP=$(control_plane dfw) +NODE=dfw-worker +TAPS=(G000000080082H G000000081082H) + +case "${1:-}" in + up) + "${SCRIPT_DIR}/deploy-tenant-policy.sh" "${NS}" dfw + "${SCRIPT_DIR}/deploy-ns.sh" "${NS}" dfw + "${SCRIPT_DIR}/publish-attachments.sh" "${NS}" dfw + docker exec "${CP}" kubectl -n "${NS}" wait --for=condition=Ready pod \ + -l 'app in (vpc-a-tap,vpc-b-tap,envoy-standin)' --timeout=240s + ;; + down) + docker exec "${CP}" kubectl delete namespace "${NS}" --ignore-not-found --wait=true --timeout=240s + for tap in "${TAPS[@]}"; do + for i in $(seq 1 60); do + docker exec "${NODE}" ip link show "${tap}" >/dev/null 2>&1 || break + if [ "${i}" -eq 60 ]; then + echo "handoff-fixture: ${tap} is still on ${NODE} after the namespace went" >&2 + exit 1 + fi + sleep 1 + done + done + echo "ns80-tap removed" + ;; + *) + echo "usage: $(basename "$0") up|down" >&2 + exit 2 + ;; +esac diff --git a/deploy/containerlab/scripts/handoff-lib.sh b/deploy/containerlab/scripts/handoff-lib.sh new file mode 100644 index 00000000..489ac486 --- /dev/null +++ b/deploy/containerlab/scripts/handoff-lib.sh @@ -0,0 +1,354 @@ +#!/bin/bash +# handoff-lib.sh: shared setup for the Local handoff checks +# (docs/handoffs.md). Sourced by handoff-observe.sh, handoff-measure.sh and +# handoff-twice.sh, after lib.sh. Needs OUT set before rig_start. +# +# Everything a check touches on the compute node is made by the rig container +# (RIG), which shares the node's network and PID namespaces, or by +# handoff-probe, which runs in the node itself. Removing the rig container +# stops every listener and capture it started. + +# shellcheck disable=SC2034 # read by the scripts that source this file +NODE=dfw-worker +CP=$(control_plane dfw) +RIG=handoff-rig +PROBE=/usr/local/bin/handoff-probe +# Not under /tmp: a Kind node's /tmp is a tmpfs, which docker cp cannot reach. +NODE_CFG=/var/lib/handoff-probe +GALACTIC_PIN=/sys/fs/bpf/galactic +FIXTURE_NS=ns80-tap +DEBUG_IMAGE=ghcr.io/datum-cloud/galactic-debug@sha256:b64e338f845d06f82f2575ae6bb6855da330af7551a8816b7ade2602a85722f2 + +VRF_A=G000000080V +VRF_B=G000000081V +TAP_A=G000000080082H +TAP_B=G000000081082H +VETH_A=G000000080080H +VETH_B=G000000081081H +CLIENT=2001:db8:4144:1::2 +CLIENT_NET=2001:db8:4144:1::/64 +VIP_A=2001:db8:4144:a::80 +VIP_B=2001:db8:4144:a::81 +# VIP_A_UNDECIDED reaches the same TAP as VIP_A, but its tc program leaves +# the policy decision to tp_deliver. +VIP_A_UNDECIDED=2001:db8:4144:a::82 +SHARD_A=2001:db8:4144:5::80 +SHARD_B=2001:db8:4144:5::81 +TAP_GUEST=fd20:80:ff03::1:0:0 +VETH_GUEST=fd20:80:ff01::1:0:0 +LINK_NET=fd20:80:ff04::/127 +LINK_POD=fd20:80:ff04::1 +# The compute node's physical members: bond0 and bond1, to dfw-worker2 and +# dfw-worker3. A Local handoff must put nothing of a probe on them. +PHYSICAL_IFS=(eth1 eth3 eth2 eth4) + +SCRIPT_DIR=${SCRIPT_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)} +PCAP_COUNT=("python3" "-I" "${SCRIPT_DIR}/handoff-pcap.py") + +rig() { docker exec -i "${RIG}" "$@"; } +rig_bg() { docker exec -d "${RIG}" "$@"; } +client() { rig ip netns exec hp-client "$@"; } +probe() { docker exec -i "${NODE}" "${PROBE}" "$@"; } +kc() { docker exec -i "${CP}" kubectl "$@"; } + +# pod_pid NAMESPACE APP prints the PID of the ready pod's sandbox on NODE. +pod_pid() { + local id + id=$(docker exec "${NODE}" crictl pods --namespace "$1" --label "app=$2" --state ready -q | head -1) + [ -n "${id}" ] || return 1 + docker exec "${NODE}" crictl inspectp "${id}" | jq -r '.info.pid' +} + +# in_pod PID CMD... runs CMD with the rig's tools in the pod's network +# namespace. +in_pod() { + local pid="$1" + shift + rig nsenter -t "${pid}" -n "$@" +} + +# listen WHERE NAME ADDRESS REPLY starts a socat listener that answers each +# request line with " REPLY". WHERE is client or pid:N. +listen() { + local where="$1" addr="$2" reply="$3" run=() log + case "${where}" in + client) run=(ip netns exec hp-client) ;; + pid:*) run=(nsenter -t "${where#pid:}" -n) ;; + esac + # socat logs to a file per listener, so a listener that fails to start or + # dies is on record. + log=$(printf '%s-%s' "${where#pid:}" "${addr%%,*}" | tr -c 'A-Za-z0-9.-' '_') + rig_bg "${run[@]}" socat -d -lf "/out/logs/listen-${log}.log" "${addr},fork,reuseaddr" \ + "SYSTEM:read -r l; echo \"\$l ${reply}\"" +} + +# listen_udp WHERE ADDRESS PORT REPLY starts a UDP echo server +# (handoff-echo.py) that answers each datagram with " REPLY" from +# ADDRESS ("::" for any). +listen_udp() { + local where="$1" addr="$2" port="$3" reply="$4" run=() + case "${where}" in + client) run=(ip netns exec hp-client) ;; + pid:*) run=(nsenter -t "${where#pid:}" -n) ;; + esac + rig_bg "${run[@]}" python3 -I /hp/handoff-echo.py "${addr}" "${port}" "${reply}" +} + +# ask WHERE TIMEOUT ADDRESS LINE sends LINE and prints the answer, if any. +ask() { + local where="$1" t="$2" addr="$3" line="$4" run=() + case "${where}" in + client) run=(ip netns exec hp-client) ;; + pid:*) run=(nsenter -t "${where#pid:}" -n) ;; + esac + printf '%s\n' "${line}" | docker exec -i "${RIG}" timeout "$((t + 1))" "${run[@]}" socat -T"${t}" - "${addr}" 2>/dev/null || true +} + +# rig_start runs the rig container in NODE's network and PID namespaces, +# with OUT mounted at /out and the scripts at /hp. +rig_start() { + docker rm -f "${RIG}" >/dev/null 2>&1 || true + docker run -d --rm --name "${RIG}" --privileged \ + --network "container:${NODE}" --pid "container:${NODE}" \ + -v "${OUT}:/out" -v "${SCRIPT_DIR}:/hp:ro" \ + --entrypoint sleep "${DEBUG_IMAGE}" infinity >/dev/null + rig mkdir -p /out/.cap /out/pcap /out/logs +} + +rig_stop() { + docker rm -f "${RIG}" >/dev/null 2>&1 || true +} + +# probe_install copies the probe binary built from this checkout into NODE. +probe_install() { + docker cp "$1" "${NODE}:${PROBE}" >/dev/null + docker exec "${NODE}" mkdir -p "${NODE_CFG}" +} + +probe_uninstall() { + docker exec "${NODE}" rm -rf "${PROBE}" "${NODE_CFG}" 2>/dev/null || true +} + +# probe_config NAME writes stdin to NODE as a probe configuration and prints +# its path there. A copy stays in OUT/configs. +probe_config() { + mkdir -p "${OUT}/configs" + cat >"${OUT}/configs/$1.json" + docker exec -i "${NODE}" sh -c "mkdir -p '${NODE_CFG}' && cat >'${NODE_CFG}/$1.json'" <"${OUT}/configs/$1.json" + echo "${NODE_CFG}/$1.json" +} + +# Captures ------------------------------------------------------------------ + +# cap WINDOW POINT WHERE IFACE... captures every IPv6 packet on each IFACE into +# OUT/pcap/WINDOW/POINT.IFACE.pcap. WHERE is node, client or pid:N. +cap() { + local window="$1" point="$2" where="$3" iface run=() name + shift 3 + case "${where}" in + node) run=() ;; + client) run=(ip netns exec hp-client) ;; + pid:*) run=(nsenter -t "${where#pid:}" -n) ;; + esac + rig mkdir -p "/out/pcap/${window}" + for iface in "$@"; do + name="${window}/${point}.${iface}" + CAP_LOGS+=("${OUT}/pcap/${name}.log") + rig_bg sh -c "echo \$\$ >'/out/.cap/${point}.${iface}.pid'; exec timeout 300 ${run[*]} \ + tcpdump -U -n -i '${iface}' -w '/out/pcap/${name}.pcap' ip6 2>'/out/pcap/${name}.log'" + done +} + +# CAP_LOGS lists the logs of the captures started since the last cap_wait. +declare -ga CAP_LOGS=() + +# cap_wait waits until every capture started since the last call is +# listening. Each is started asynchronously, so its log may not exist yet. +cap_wait() { + local log i + for log in "${CAP_LOGS[@]}"; do + for i in $(seq 1 50); do + if grep -q 'listening on' "${log}" 2>/dev/null; then + break + fi + if [ "${i}" -eq 50 ]; then + echo "capture never started: ${log}" >&2 + cat "${log}" >&2 || true + return 1 + fi + sleep 0.2 + done + done + CAP_LOGS=() +} + +# cap_stop stops every running capture and waits for them to flush. +cap_stop() { + sleep 1 + rig sh -c 'for f in /out/.cap/*.pid; do [ -e "$f" ] || continue; kill -INT "$(cat "$f")" 2>/dev/null; done + for i in $(seq 1 50); do + alive=0 + for f in /out/.cap/*.pid; do [ -e "$f" ] || continue; kill -0 "$(cat "$f")" 2>/dev/null && alive=1; done + [ "$alive" = 0 ] && break + sleep 0.2 + done + rm -f /out/.cap/*.pid' || true +} + +# count WINDOW POINT NONCE [handoff-pcap.py options] counts POINT's packets +# that carry NONCE, summed over its interfaces. An empty NONCE matches every +# packet, for checks that match on addresses and ports instead. +count() { + local window="$1" point="$2" nonce="$3" + shift 3 + "${PCAP_COUNT[@]}" --nonce "${nonce}" "$@" "${OUT}/pcap/${window}/${point}".*.pcap +} + +# answers PATTERN counts the lines of stdin equal to PATTERN's lines. +answered() { grep -cxF -- "$1" || true; } + +# counter NAME reads one probe counter from the JSON on stdin. +counter() { jq -r --arg k "$1" '.[$k] // 0'; } + +# Runs ---------------------------------------------------------------------- + +# node_monitors stops the run's tc monitor on NODE and keeps its output, +# with Cilium's view of its devices and its log lines about the TAPs. +node_monitors() { + local pod + docker exec "${NODE}" pkill -f "tc -ts monitor" 2>/dev/null || true + docker exec "${NODE}" cat "${NODE_CFG}/tc-monitor.txt" >"${OUT}/logs/tc-monitor.txt" 2>/dev/null || true + pod=$(kc -n kube-system get pods -l k8s-app=cilium --field-selector "spec.nodeName=${NODE}" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + [ -n "${pod}" ] || return 0 + kc -n kube-system exec "${pod}" -c cilium-agent -- cilium-dbg status --verbose \ + >"${OUT}/logs/cilium-status.txt" 2>&1 || true + kc -n kube-system logs "${pod}" -c cilium-agent --since=2h 2>/dev/null | + grep -iE "${TAP_A}|${TAP_B}|devices" >"${OUT}/logs/cilium-taps.log" || true +} + +# cni_log keeps galactic-cni's log lines on NODE about the fixture's TAPs and +# stale TAP reclaims, for the run's record. +cni_log() { + local pod + pod=$(kc -n galactic-system get pods --field-selector "spec.nodeName=${NODE}" \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null | grep -m1 '^galactic-cni' || true) + [ -n "${pod}" ] || return 0 + kc -n galactic-system logs "${pod}" --since=2h 2>/dev/null | + grep -E "${TAP_A}|${TAP_B}|stale|reclaim" >"${OUT}/logs/cni-taps.log" || true +} + +# precondition REASON records why a run stopped before changing anything. +precondition() { + echo "PRECONDITION: $*" | tee "${OUT}/precondition" + exit 0 +} + +# handoff_preconditions checks, before any change, that the lab, ns80, the +# fixture and the probe are ready and that no earlier run left anything +# behind. It fills PID with the pods' sandbox PIDs. It installs the probe on +# NODE only to ask it, and removes it again when it stops. +handoff_preconditions() { + local c k + for c in "${NODE}" "${CP}"; do + if [ "$(docker inspect -f '{{.State.Running}}' "${c}" 2>/dev/null || true)" != true ]; then + precondition "lab container ${c} is not running" + fi + done + [ -x "${HANDOFF_PROBE}" ] || precondition "no probe at ${HANDOFF_PROBE} (task handoff:probe)" + declare -gA PID + PID[veth80]=$(pod_pid ns80 vpc-a || true) + PID[veth81]=$(pod_pid ns80 vpc-b || true) + PID[tap80]=$(pod_pid "${FIXTURE_NS}" vpc-a-tap || true) + PID[tap81]=$(pod_pid "${FIXTURE_NS}" vpc-b-tap || true) + PID[envoy]=$(pod_pid "${FIXTURE_NS}" envoy-standin || true) + for k in veth80 veth81; do + [ -n "${PID[${k}]}" ] || precondition "ns80's ${k} guest is not ready on ${NODE} (task deploy:ns80)" + done + for k in tap80 tap81 envoy; do + [ -n "${PID[${k}]}" ] || precondition "the ns80-tap fixture's ${k} pod is not ready on ${NODE} (task deploy:ns80-tap)" + done + if docker inspect "${RIG}" >/dev/null 2>&1; then + precondition "a rig container ${RIG} already exists; an earlier run did not clean up" + fi + probe_install "${HANDOFF_PROBE}" + if ! probe leftovers "${VRF_A}" "${VRF_B}" >"${OUT}/logs/leftovers-before.txt" 2>&1; then + probe_uninstall + precondition "an earlier run left probe state behind: $(tr '\n' ';' <"${OUT}/logs/leftovers-before.txt")" + fi + local rc=0 + probe check "${GALACTIC_PIN}" >"${OUT}/logs/check.txt" 2>&1 || rc=$? + if [ "${rc}" -ne 0 ]; then + probe_uninstall + [ "${rc}" -eq 3 ] && precondition "the running uSID datapath differs from this checkout: $(tail -1 "${OUT}/logs/check.txt")" + precondition "the probe cannot load: $(tail -1 "${OUT}/logs/check.txt")" + fi +} + +# handoff_cleanup undoes everything a run placed on NODE, checks that +# nothing is left, and finishes the run. HANDOFF_RESTORE, when set, is run +# first, for state a script changed outside the rig. +handoff_cleanup() { + local cfg + set +e + [ -n "${HANDOFF_RESTORE:-}" ] && "${HANDOFF_RESTORE}" + cap_stop + cni_log + node_monitors + for cfg in "${OUT}"/configs/*.json; do + [ -e "${cfg}" ] && probe teardown "${NODE_CFG}/$(basename "${cfg}")" >>"${OUT}/logs/teardown.txt" 2>&1 + done + rig /hp/handoff-rig.sh down >>"${OUT}/logs/teardown.txt" 2>&1 + rig_stop + if probe leftovers "${VRF_A}" "${VRF_B}" >"${OUT}/logs/leftovers-after.txt" 2>&1; then + echo "cleanup: nothing left on ${NODE}" + else + echo "cleanup: LEFTOVERS on ${NODE}:" >&2 + cat "${OUT}/logs/leftovers-after.txt" >&2 + FAILED=1 + fi + probe_uninstall + handoff_finish +} +# handoff_finish writes results.json and exits with the run's outcome. +handoff_finish() { + jq -R -s 'split("\n") | map(select(length > 0) | split("\t") + | {scenario: .[0], check: .[1], expect: .[2], got: .[3], result: .[4]}) + | sort_by(.scenario, .check)' "${OUT}/results.tsv" >"${OUT}/results.json" + echo "=== $(jq length "${OUT}/results.json") checks, $(jq '[.[] | select(.result == "fail")] | length' \ + "${OUT}/results.json") failed; results in ${OUT}/results.json ===" + exit "${FAILED}" +} + +# json_list A,B prints ["A","B"]. +json_list() { jq -cn --arg s "$1" '$s | split(",")'; } + +# cell_config NAME MODE PEER-MODE REDIRECT XDP TC VLAN DUT [PUSH] writes one +# rig cell's probe configuration to NODE and prints its path there. PUSH=1 +# makes the pair peer push the VLAN tag and redirect to the VLAN's lower +# device when its egress is the rig's VLAN device. +cell_config() { + local name="$1" mode="$2" peer="$3" redirect="$4" xdp="$5" tc="$6" vlan="$7" dut="$8" push="${9:-0}" + jq -n --arg mode "${mode}" --arg peer "${peer}" --argjson redirect "${redirect}" \ + --argjson xdp "$(json_list "${xdp}")" --argjson tc "$(json_list "${tc}")" --argjson vlan "${vlan}" \ + --arg dut "${dut}" --arg pin "/sys/fs/bpf/galactic-handoff-proof" --arg gpin "${GALACTIC_PIN}" \ + --arg tapA "${TAP_A}" --arg tapB "${TAP_B}" --arg guest "${TAP_GUEST}" \ + --arg vipA "${VIP_A}" --arg vipB "${VIP_B}" --arg vipU "${VIP_A_UNDECIDED}" \ + --arg shardA "${SHARD_A}" --arg shardB "${SHARD_B}" --arg client "${CLIENT_NET}" \ + --arg vrfA "${VRF_A}" --arg vrfB "${VRF_B}" --argjson push "${push}" '{ + pinDir: $pin, galacticPinDir: $gpin, mode: $mode, peerMode: $peer, + xdp: $xdp, tc: $tc, vlanID: $vlan, + vips: [ + {vip: $vipA, tap: $tapA, guest: $guest, decide: true}, + {vip: $vipB, tap: $tapB, guest: $guest, decide: true}, + {vip: $vipU, tap: $tapA, guest: $guest, decide: false}], + taps: [ + {tap: $tapA, dut: $dut, pair: "hpn80t", peer: "hpn80n", vip: $vipA, clientPrefix: $client}, + {tap: $tapB, dut: $dut, pair: "hpn81t", peer: "hpn81n", vip: $vipB, clientPrefix: $client}], + pairs: [ + {pair: "hpn80t", peer: "hpn80n", tap: $tapA, guest: $guest, shard: $shardA, redirect: $redirect}, + {pair: "hpn81t", peer: "hpn81n", tap: $tapB, guest: $guest, shard: $shardB, redirect: $redirect}], + passThrough: [{vrf: $vrfA, prefix: $client}, {vrf: $vrfB, prefix: $client}]} + | if $push == 1 then .pairs[] += {vlan: {device: "hp-dut0.120", lower: "hp-dut0", vid: 120}} else . end' | + probe_config "${name}" +} diff --git a/deploy/containerlab/scripts/handoff-measure.sh b/deploy/containerlab/scripts/handoff-measure.sh new file mode 100755 index 00000000..4a8c88dd --- /dev/null +++ b/deploy/containerlab/scripts/handoff-measure.sh @@ -0,0 +1,197 @@ +#!/bin/bash +# handoff-measure.sh: measure the Local handoffs' MTU limits and their +# behavior under GRO, GSO and checksum offload (docs/handoffs.md), on the +# same fixture and rig as handoff-observe.sh. +# +# Usage: handoff-measure.sh OUTDIR +# +# For native XDP and for generic XDP with a native veth peer: +# +# mtu UDP to a VIP at 1200 and 1451 bytes (with its newline, the +# largest unfragmented payload at MTU 1500), and at 1452 and +# 4000 bytes, which the sender fragments +# bulk 1 MiB over TCP each way, through local.tap and through +# local.veth_pair, with the client's checksum offload (and so +# TSO) off, then on +# pod-veth NAT flows from ns80's veth guest through an internal veth +# pair, with the guest's own checksum offload on, then off +# +# Then the receiving device at MTU 9000: whether each mode attaches, and +# what becomes of 4000- and 8000-byte packets for a TAP at MTU 1500. +# +# Expectations are set only where the result is a requirement; the rest are +# recorded observations, compared between runs like any other row. +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +source "${SCRIPT_DIR}/lib.sh" + +LAB=${LAB:-gvpc} +OUT=${1:?usage: handoff-measure.sh OUTDIR} +mkdir -p "${OUT}/logs" +OUT=$(cd "${OUT}" && pwd) +: >"${OUT}/results.tsv" +: >"${OUT}/volatile.tsv" +rm -f "${OUT}/precondition" +source "${SCRIPT_DIR}/baseline-lib.sh" +source "${SCRIPT_DIR}/handoff-lib.sh" + +REPO=$(cd "${SCRIPT_DIR}/../../.." && pwd) +HANDOFF_PROBE=${HANDOFF_PROBE:-${REPO}/bin/handoff-probe} +SHARD_V=2001:db8:4144:5::180 +VETH_TX="" + +handoff_preconditions + +# restore_guest puts the veth guest's checksum offload back as it was. +restore_guest() { + if [ -n "${VETH_TX}" ]; then + in_pod "${PID[veth80]}" ethtool -K eth0 tx "${VETH_TX}" >/dev/null 2>&1 || true + fi +} +HANDOFF_RESTORE=restore_guest +trap handoff_cleanup EXIT + +rig_start +rig /hp/handoff-rig.sh client-up +VETH_TX=$(in_pod "${PID[veth80]}" ethtool -k eth0 | awk '/^tx-checksumming:/ {print $2}') +volatile veth-guest-tx "${VETH_TX}" +rig sh -c 'head -c 1048576 /dev/urandom >/out/blob' +BLOB_SHA=$(sha256sum <"${OUT}/blob" | cut -c1-16) + +listen_udp "pid:${PID[tap80]}" "${VIP_A}" 7 tap80 +rig_bg nsenter -t "${PID[tap80]}" -n socat TCP6-LISTEN:8081,fork,reuseaddr "SYSTEM:sha256sum | cut -c1-16" +listen_udp client :: 7 client +rig_bg ip netns exec hp-client socat TCP6-LISTEN:8082,fork,reuseaddr "SYSTEM:cat /out/blob" +sleep 1 + +# udp_to_vip BYTES: does a BYTES-long line to VIP_A get tap80's answer? +udp_to_vip() { + local line + line="${NONCE:-m}-$(head -c "$1" /dev/zero | tr '\0' x)" + line=${line:0:$1} + seen "$(ask client 3 "UDP6:[${VIP_A}]:7" "${line}" | grep -c ' tap80$' || true)" +} + +bulk_h1() { + docker exec "${RIG}" sh -c "ip netns exec hp-client timeout 30 socat -T10 - TCP6:[${VIP_A}]:8081 /dev/null || true +} + +bulk_h2() { + docker exec "${RIG}" sh -c "timeout 30 nsenter -t ${PID[tap80]} -n socat -T10 -u TCP6:[${CLIENT}]:8082 - | sha256sum | cut -c1-16" 2>/dev/null || true +} + +veth_flows() { + local i n=0 + for i in 1 2 3; do + n=$((n + $(ask "pid:${PID[veth80]}" 2 "UDP6:[${CLIENT}]:7" "veth-${i}" | grep -c ' client$' || true))) + done + echo "${n}" +} + +# A cell's configuration plus the veth guest's pair (hpv80t, hpv80n). +measure_config() { + local name="$1" + shift + local json + cell_config "${name}" "$@" >/dev/null + json=$(jq --arg veth "${VETH_A}" --arg guest "${VETH_GUEST}" --arg shard "${SHARD_V}" --arg client "${CLIENT_NET}" \ + --arg dut "$(jq -r '.taps[0].dut' "${OUT}/configs/${name}.json")" ' + .taps += [{tap: $veth, dut: $dut, pair: "hpv80t", peer: "hpv80n", vip: "2001:db8:4144:a::ff", + clientPrefix: $client}] + | .pairs += [{pair: "hpv80t", peer: "hpv80n", tap: $veth, guest: $guest, shard: $shard, redirect: true}]' \ + "${OUT}/configs/${name}.json") + # probe_config rewrites the file jq just read, so it gets the output only + # once jq is done. + printf '%s\n' "${json}" | probe_config "${name}" +} + +features() { + local f="${OUT}/logs/features-$1.txt" i + for i in hp-dut0 "${TAP_A}" hpn80t hpn80n hpv80t; do + echo "## ${i}" >>"${f}" + rig ethtool -k "${i}" >>"${f}" 2>&1 || true + done + echo "## client hp-cl0" >>"${f}" + client ethtool -k hp-cl0 >>"${f}" 2>&1 || true +} + +NONCE="hm$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')" +volatile nonce "${NONCE}" + +for c in "native native" "generic native"; do + read -r mode peer <<<"${c}" + name="${mode}" + [ "${mode}" = generic ] && name="generic-native-peer" + for tx in off on; do + m="${name}-client-tx-${tx}" + echo "=== ${m} ===" + dut=$(rig env CLIENT_TX="${tx}" /hp/handoff-rig.sh dut-up plain | tail -1) + rig /hp/handoff-rig.sh pairs-up on >/dev/null + rig /hp/handoff-rig.sh vpair-up + cfg=$(measure_config "${m}" "${mode}" "${peer}" true hp-dut0 hp-dut0 0 "${dut}") + probe setup "${cfg}" >"${OUT}/logs/setup-${m}.txt" 2>&1 + features "${m}" + ask client 2 "UDP6:[${VIP_A}]:7" warm >/dev/null + ask "pid:${PID[tap80]}" 2 "UDP6:[${CLIENT}]:7" warm >/dev/null + ask "pid:${PID[veth80]}" 2 "UDP6:[${CLIENT}]:7" warm >/dev/null + if [ "${tx}" = off ]; then + # A line of N bytes is an N+1-byte payload: 1451 is the largest that + # travels unfragmented at MTU 1500. The larger ones are fragmented by + # the client. + for size in 1200 1451; do + record "mtu/${name}" "UDP ${size}+1 bytes to a VIP is answered" yes "$(udp_to_vip "${size}")" + done + for size in 1452 4000; do + record "mtu/${name}" "UDP ${size}+1 bytes, fragmented, to a VIP is answered" - "$(udp_to_vip "${size}")" + done + fi + want=- + [ "${tx}" = off ] && want=yes + got=$(bulk_h1) + record "bulk/${m}" "1 MiB client to VIP arrives intact" "${want}" "$([ "${got}" = "${BLOB_SHA}" ] && echo yes || echo no)" + got=$(bulk_h2) + record "bulk/${m}" "1 MiB client to guest through the pair arrives intact" "${want}" \ + "$([ "${got}" = "${BLOB_SHA}" ] && echo yes || echo no)" + if [ "${tx}" = off ]; then + for vtx in on off; do + in_pod "${PID[veth80]}" ethtool -K eth0 tx "${vtx}" >/dev/null + want=- + [ "${vtx}" = off ] && want=3 + record "pod-veth/${name}" "veth guest with checksum offload ${vtx}: NAT flows answered" "${want}" "$(veth_flows)" + done + restore_guest + fi + probe counters "${cfg}" >"${OUT}/logs/counters-${m}.json" + record "bulk/${m}" "largest frame XDP saw" - "$(counter max_seen_len <"${OUT}/logs/counters-${m}.json")" + probe teardown "${cfg}" >>"${OUT}/logs/teardown.txt" 2>&1 + rig /hp/handoff-rig.sh vpair-down + rig /hp/handoff-rig.sh pairs-down + rig /hp/handoff-rig.sh dut-down + done +done + +# MTU 9000 on the receiving device; the TAP stays at 1500. +for mode in native generic; do + m="jumbo-${mode}" + echo "=== ${m} ===" + dut=$(rig /hp/handoff-rig.sh dut-up plain 9000 | tail -1) + rig /hp/handoff-rig.sh pairs-up on >/dev/null + cfg=$(cell_config "${m}" "${mode}" native true hp-dut0 hp-dut0 0 "${dut}") + if probe setup "${cfg}" >"${OUT}/logs/setup-${m}.txt" 2>&1; then + record "mtu/${m}" "XDP attaches at MTU 9000" - yes + record "mtu/${m}" "UDP 1451+1 bytes at MTU 9000 to a TAP at 1500 is answered" yes "$(udp_to_vip 1451)" + for size in 1452 4000 8000; do + record "mtu/${m}" "unfragmented UDP ${size}+1 bytes at MTU 9000 to a TAP at 1500 is answered" - \ + "$(udp_to_vip "${size}")" + done + probe counters "${cfg}" >"${OUT}/logs/counters-${m}.json" + record "mtu/${m}" "largest frame XDP saw" - "$(counter max_seen_len <"${OUT}/logs/counters-${m}.json")" + else + record "mtu/${m}" "XDP attaches at MTU 9000" - no + volatile "setup-${m}" "$(tail -1 "${OUT}/logs/setup-${m}.txt")" + fi + probe teardown "${cfg}" >>"${OUT}/logs/teardown.txt" 2>&1 || true + rig /hp/handoff-rig.sh pairs-down + rig /hp/handoff-rig.sh dut-down +done diff --git a/deploy/containerlab/scripts/handoff-observe.sh b/deploy/containerlab/scripts/handoff-observe.sh new file mode 100755 index 00000000..6489b1e6 --- /dev/null +++ b/deploy/containerlab/scripts/handoff-observe.sh @@ -0,0 +1,404 @@ +#!/bin/bash +# handoff-observe.sh: prove the Local kernel handoffs (docs/handoffs.md) on +# the compute node of the #1432 fixture, with captures, and write the +# results. +# +# Usage: handoff-observe.sh OUTDIR +# +# Needs ns80 (task deploy:ns80) and the ns80-tap fixture (task +# deploy:ns80-tap) deployed, and the probe built from this checkout +# (HANDOFF_PROBE, default bin/handoff-probe at the repository root). When a +# precondition does not hold, it says which in OUTDIR/precondition, changes +# nothing and exits 0. +# +# Each cell builds a receiving device (plain veth, 802.3ad bond, VLAN) on the +# node, places the prototypes in an XDP mode, and runs two capture windows: +# +# h1/ local.tap: client to two VIPs, one per VPC, delivered to the +# TAP guests with the same address; the replies; a denied port +# each way; a VIP whose tc program leaves policy to tp_deliver +# h2/ local.veth_pair: each TAP guest's flow out through the +# internal veth pair and the shard address, and back; a denied +# port each way +# +# Then one window, h3, for local.routed_link and local.socket: VRF-bound +# sockets in a pod reach each VPC's veth guest through a routed link and +# read the answer, and the guests reach sockets in the pod; a denied port; +# an unbound socket; a spoofed source. +# +# Every window also captures the node's physical bond members, which must +# carry nothing of the probes, encapsulated or not. +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +source "${SCRIPT_DIR}/lib.sh" + +LAB=${LAB:-gvpc} +OUT=${1:?usage: handoff-observe.sh OUTDIR} +mkdir -p "${OUT}/logs" +OUT=$(cd "${OUT}" && pwd) +: >"${OUT}/results.tsv" +: >"${OUT}/volatile.tsv" +rm -f "${OUT}/precondition" +source "${SCRIPT_DIR}/baseline-lib.sh" +source "${SCRIPT_DIR}/handoff-lib.sh" + +REPO=$(cd "${SCRIPT_DIR}/../../.." && pwd) +HANDOFF_PROBE=${HANDOFF_PROBE:-${REPO}/bin/handoff-probe} +N=5 + +handoff_preconditions +trap handoff_cleanup EXIT + +NONCE="hp$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')" +volatile nonce "${NONCE}" +volatile kernel "$(docker exec "${NODE}" uname -r)" +volatile cni-image "$(kc -n galactic-system get ds -o jsonpath='{range .items[*]}{.spec.template.spec.containers[0].image}{"\n"}{end}' | grep -m1 cni || true)" +volatile checkout "$(git -C "${REPO}" rev-parse HEAD)" +cp "${OUT}/logs/check.txt" "${OUT}/logs/datapath-check.txt" + +rig_start +rig /hp/handoff-rig.sh client-up + +# Listeners. Each answers a line with the line and who it is. +for v in 80 81; do + vip=${VIP_A} + [ "${v}" = 81 ] && vip=${VIP_B} + listen_udp "pid:${PID[tap${v}]}" "${vip}" 7 "tap${v}" + listen "pid:${PID[tap${v}]}" "TCP6-LISTEN:8080" "tap${v}" + listen "pid:${PID[tap${v}]}" "TCP6-LISTEN:9" "tap${v}-open9" + listen "pid:${PID[veth${v}]}" "TCP6-LISTEN:8080" "vpc${v}" + listen "pid:${PID[veth${v}]}" "TCP6-LISTEN:9" "vpc${v}-open9" +done +listen_udp "pid:${PID[tap80]}" :: 17 "tap80-wild" +listen_udp client :: 7 client +listen client "TCP6-LISTEN:8080" client +sleep 1 +for k in tap80 tap81 veth80 veth81; do + in_pod "${PID[${k}]}" ss -lntup >"${OUT}/logs/sockets-${k}.txt" 2>&1 || true +done +in_pod "${PID[tap80]}" nstat -az Udp6InDatagrams Udp6NoPorts Udp6InErrors Udp6RcvbufErrors Udp6InCsumErrors \ + >"${OUT}/logs/udp6-tap80.txt" 2>&1 || true +# tc changes on the node for the whole run, collected by handoff_cleanup. +docker exec -d "${NODE}" sh -c "exec tc -ts monitor >'${NODE_CFG}/tc-monitor.txt' 2>&1" +# Link changes on the node for the whole run. +rig_bg sh -c 'exec ip -ts monitor link >/out/logs/link-monitor.txt 2>&1' +docker exec "${NODE}" crictl ps --name galactic-cni -o json 2>/dev/null | + jq -r '.containers[] | "galactic-cni \(.id[0:12]) started \(.createdAt)"' >"${OUT}/logs/cni.txt" || true + +# Cells --------------------------------------------------------------------- +# name kind mode peerMode redirect xdp-ifaces tc-ifaces vlan setup h2 push +# setup: ok, or unsupported when the attach must fail. h2: pass, or how the +# handoff as specified fails to deliver, which the run records: +# peer-skipped generic XDP on the peer never runs on a packet tc +# redirected (the kernel's skb_is_redirected check) +# egress-unsupported the peer translates, but its XDP_REDIRECT to a VLAN +# device has no transmit path +# push: 1 where the peer pushes the VLAN tag itself and redirects to the +# VLAN's lower device, the alternative the contract adopts. +CELLS=( + "plain-native plain native native true hp-dut0 hp-dut0 0 ok pass 0" + "plain-generic plain generic generic true hp-dut0 hp-dut0 0 ok peer-skipped 0" + "plain-generic-native-peer plain generic native true hp-dut0 hp-dut0 0 ok pass 0" + "bond-native bond native native true hp-dut0,hp-dut1 hp-dut0,hp-dut1 0 ok pass 0" + "bond-generic-native-peer bond generic native true hp-dut0,hp-dut1 hp-dut0,hp-dut1 0 ok pass 0" + "vlan-native vlan native native true hp-dut0.120 hp-dut0.120 0 unsupported - 0" + "vlan-lower-native vlan native native true hp-dut0 hp-dut0.120 120 ok pass 1" + "vlan-generic-native-peer vlan generic native true hp-dut0.120 hp-dut0.120 0 ok pass 1" + "vlan-generic-redirect vlan generic native true hp-dut0.120 hp-dut0.120 0 ok egress-unsupported 0" +) +if [ -n "${HANDOFF_CELLS:-}" ]; then + # A comma-separated subset, for a quicker run while working on a cell. + mapfile -t CELLS < <(for c in "${CELLS[@]}"; do + case ",${HANDOFF_CELLS}," in *",${c%% *},"*) echo "${c}" ;; esac + done) +fi + +# trace CELL STEP keeps both TAPs' tc ingress filters after a step of a +# cell, so a run shows which step, if any, changes galactic-cni's chain. +trace() { + local t + for t in "${TAP_A}" "${TAP_B}"; do + { + echo "## $(date -u +%T.%N) $1 $2 ${t}" + docker exec "${NODE}" tc qdisc show dev "${t}" 2>&1 + docker exec "${NODE}" tc filter show dev "${t}" ingress 2>&1 + docker exec "${NODE}" tc filter show dev "${t}" egress 2>&1 + rig bpftool net show dev "${t}" 2>&1 + } >>"${OUT}/logs/trace.txt" + done +} + +# capture_cell WINDOW DUT-MEMBERS... arms every capture point of a cell. +capture_cell() { + local w="$1" + shift + cap "${w}" client client "${@/hp-dut/hp-cl}" + cap "${w}" dut node "$@" + cap "${w}" tap80-host node "${TAP_A}" + cap "${w}" tap81-host node "${TAP_B}" + cap "${w}" tap80-guest "pid:${PID[tap80]}" tap0 + cap "${w}" tap81-guest "pid:${PID[tap81]}" tap0 + cap "${w}" pair80 node hpn80t hpn80n + cap "${w}" pair81 node hpn81t hpn81n + cap "${w}" physical node "${PHYSICAL_IFS[@]}" + cap_wait +} + +# no_trace WINDOW TAG records that the physical members carried nothing +# with TAG, and nothing encapsulated with it. +no_trace() { + record "$1" "physical members carry the probes" 0 "$(count "$1" physical "$2")" + record "$1" "physical members carry them encapsulated" 0 "$(count "$1" physical "$2" --srv6)" +} + +run_cell() { + local name kind mode peer redirect xdp tc vlan setup h2 push dut cfg w tag v vip shard answers err members + read -r name kind mode peer redirect xdp tc vlan setup h2 push <<<"$1" + echo "=== cell ${name} ===" + # The prototypes run after galactic-cni's own filters on the TAPs; a cell + # without them would prove nothing. Record and skip it. + local t chain=yes + for t in "${TAP_A}" "${TAP_B}"; do + docker exec "${NODE}" tc filter show dev "${t}" ingress >"${OUT}/logs/tap-filters-${name}-${t}.txt" 2>&1 || true + grep -q galactic_usid_egress "${OUT}/logs/tap-filters-${name}-${t}.txt" || chain=no + done + if [ "${chain}" = no ]; then + record "cell/${name}" "galactic-cni's egress chain is on both TAPs" yes no + return 0 + fi + trace "${name}" start + dut=$(rig /hp/handoff-rig.sh dut-up "${kind}" | tail -1) + trace "${name}" dut-up + rig /hp/handoff-rig.sh pairs-up on >/dev/null + trace "${name}" pairs-up + cfg=$(cell_config "${name}" "${mode}" "${peer}" "${redirect}" "${xdp}" "${tc}" "${vlan}" "${dut}" "${push}") + if ! probe setup "${cfg}" >"${OUT}/logs/setup-${name}.txt" 2>&1; then + err=$(tail -1 "${OUT}/logs/setup-${name}.txt") + volatile "setup-${name}" "${err}" + if grep -q 'operation not supported' "${OUT}/logs/setup-${name}.txt"; then + record "cell/${name}" "XDP attach" "${setup}" unsupported + else + record "cell/${name}" "XDP attach" "${setup}" "error" + fi + probe teardown "${cfg}" >>"${OUT}/logs/teardown.txt" 2>&1 || true + rig /hp/handoff-rig.sh pairs-down + rig /hp/handoff-rig.sh dut-down + return 0 + fi + record "cell/${name}" "XDP attach" "${setup}" ok + trace "${name}" setup + read -r -a members <<<"${xdp//,/ }" + [ "${kind}" = vlan ] && members=(hp-dut0) + + # Warm the neighbors on every path, outside the capture windows. + for v in 80 81; do + vip=${VIP_A} + [ "${v}" = 81 ] && vip=${VIP_B} + ask client 2 "UDP6:[${vip}]:7" warm >/dev/null + ask "pid:${PID[tap${v}]}" 2 "UDP6:[${CLIENT}]:7" warm >/dev/null + done + + # local.tap --------------------------------------------------------------- + w="h1/${name}" + capture_cell "${w}" "${members[@]}" + for v in 80 81; do + vip=${VIP_A} + [ "${v}" = 81 ] && vip=${VIP_B} + tag="${NONCE}-${name}-h1-${v}" + answers=$(for i in $(seq 1 "${N}"); do ask client 2 "UDP6:[${vip}]:7" "${tag}-${i}"; done) + record "${w}" "client gets tap${v}'s UDP answers from its VIP" "${N}" \ + "$(grep -c " tap${v}\$" <<<"${answers}" || true)" + echo "${answers}" >>"${OUT}/logs/answers.txt" + done + # UDP diagnostics, recorded as observations: the TAP guest's own UDP + # listener asked from inside the pod, and a wildcard listener on port 17. + record "${w}" "tap80's UDP listener answers inside the pod" - \ + "$(seen "$(ask "pid:${PID[tap80]}" 2 "UDP6:[${VIP_A}]:7" selftest | grep -c ' tap80$' || true)")" + record "${w}" "tap80's wildcard UDP listener answers the client" - \ + "$(seen "$(ask client 2 "UDP6:[${VIP_A}]:17" "${NONCE}-${name}-h1-wild" | grep -c ' tap80-wild$' || true)")" + in_pod "${PID[tap80]}" nstat -az Udp6InDatagrams Udp6NoPorts Udp6InErrors Udp6RcvbufErrors Udp6InCsumErrors \ + >>"${OUT}/logs/udp6-tap80.txt" 2>&1 || true + in_pod "${PID[tap80]}" ss -uanp >>"${OUT}/logs/udp6-tap80.txt" 2>&1 || true + tag="${NONCE}-${name}-h1-tcp" + record "${w}" "client gets tap80's TCP answer" yes \ + "$(seen "$(ask client 3 "TCP6:[${VIP_A}]:8080" "${tag}" | grep -c ' tap80$' || true)")" + ask client 2 "TCP6:[${VIP_A}]:9" "${NONCE}-${name}-h1-deny-in" >/dev/null + printf '%s\n' "${NONCE}-${name}-h1-deny-out" | docker exec -i "${RIG}" timeout 3 nsenter -t "${PID[tap80]}" -n \ + socat -u - "UDP6-SENDTO:[${CLIENT}]:9,bind=[${VIP_A}]" || true + ask client 2 "UDP6:[${VIP_A_UNDECIDED}]:7" "${NONCE}-${name}-h1-undecided" >/dev/null + probe counters "${cfg}" >"${OUT}/logs/counters-h1-${name}.json" + cap_stop + for v in 80 81; do + vip=${VIP_A} + [ "${v}" = 81 ] && vip=${VIP_B} + tag="${NONCE}-${name}-h1-${v}" + record "${w}" "tap${v} guest receives its VIP's requests" "${N}" \ + "$(count "${w}" "tap${v}-guest" "${tag}" --dst "${vip}")" + o=81 + [ "${v}" = 81 ] && o=80 + record "${w}" "tap${o} guest receives tap${v}'s requests" 0 "$(count "${w}" "tap${o}-guest" "${tag}")" + record "${w}" "client receives tap${v}'s answers from its VIP" "${N}" \ + "$(count "${w}" client "${tag}" --src "${vip}" --dst "${CLIENT}")" + done + # A denied TCP connection never carries the nonce: count its SYNs. + record "${w}" "denied TCP 9: reaches the receiving device" yes \ + "$(seen "$(count "${w}" dut "" --dst "${VIP_A}" --proto tcp --dport 9)")" + record "${w}" "denied TCP 9: reaches no guest" 0 \ + "$(($(count "${w}" tap80-guest "" --proto tcp --dport 9) + $(count "${w}" tap81-guest "" --proto tcp --dport 9)))" + record "${w}" "denied UDP 9 reply: leaves the guest" yes \ + "$(seen "$(count "${w}" tap80-host "${NONCE}-${name}-h1-deny-out")")" + record "${w}" "denied UDP 9 reply: reaches no client" 0 "$(count "${w}" client "${NONCE}-${name}-h1-deny-out")" + record "${w}" "policy left to tp_deliver: reaches the receiving device" yes \ + "$(seen "$(count "${w}" dut "${NONCE}-${name}-h1-undecided")")" + record "${w}" "policy left to tp_deliver: reaches the guest" 0 \ + "$(count "${w}" tap80-guest "${NONCE}-${name}-h1-undecided")" + record "${w}" "XDP metadata reaches tc" yes \ + "$(seen "$(counter tc_meta_ok <"${OUT}/logs/counters-h1-${name}.json")")" + record "${w}" "tc denies on the PublicService path" yes \ + "$(seen "$(counter tc_tenant_deny <"${OUT}/logs/counters-h1-${name}.json")")" + record "${w}" "tap tc denies the reply on the PublicService path" yes \ + "$(seen "$(counter tap_reply_deny <"${OUT}/logs/counters-h1-${name}.json")")" + no_trace "${w}" "${NONCE}-${name}-h1" + + trace "${name}" h1 + # local.veth_pair --------------------------------------------------------- + w="h2/${name}" + probe renew "${cfg}" >/dev/null + capture_cell "${w}" "${members[@]}" + local want=${N} wantyes=yes translates=yes leak=0 + if [ "${h2}" != pass ]; then + want=0 + wantyes=no + leak=- + fi + [ "${h2}" = peer-skipped ] && translates=no + for v in 80 81; do + tag="${NONCE}-${name}-h2-${v}" + answers=$(for i in $(seq 1 "${N}"); do ask "pid:${PID[tap${v}]}" 2 "UDP6:[${CLIENT}]:7" "${tag}-${i}"; done) + record "${w}" "tap${v} gets the client's UDP answers" "${want}" "$(grep -c ' client$' <<<"${answers}" || true)" + done + record "${w}" "tap80 gets the client's TCP answer" "${wantyes}" \ + "$(seen "$(ask "pid:${PID[tap80]}" 3 "TCP6:[${CLIENT}]:8080" "${NONCE}-${name}-h2-tcp" | grep -c ' client$' || true)")" + printf '%s\n' "${NONCE}-${name}-h2-deny-out" | docker exec -i "${RIG}" timeout 3 nsenter -t "${PID[tap80]}" -n \ + socat -u - "UDP6-SENDTO:[${CLIENT}]:9" || true + ask client 2 "TCP6:[${SHARD_A}]:9" "${NONCE}-${name}-h2-deny-in" >/dev/null + probe counters "${cfg}" >"${OUT}/logs/counters-h2-${name}.json" + cap_stop + for v in 80 81; do + shard=${SHARD_A} + [ "${v}" = 81 ] && shard=${SHARD_B} + tag="${NONCE}-${name}-h2-${v}" + record "${w}" "client receives tap${v}'s flow from its shard address" "${want}" \ + "$(count "${w}" client "${tag}" --src "${shard}" --dst "${CLIENT}")" + record "${w}" "client receives tap${v}'s flow untranslated" "${leak}" \ + "$(count "${w}" client "${tag}" --src "${TAP_GUEST}")" + o=81 + [ "${v}" = 81 ] && o=80 + record "${w}" "tap${o} guest receives tap${v}'s answers" 0 "$(count "${w}" "tap${o}-guest" "${tag}" --dst "${TAP_GUEST}")" + done + record "${w}" "pair peer XDP translates" "${translates}" \ + "$(seen "$(counter peer_snat <"${OUT}/logs/counters-h2-${name}.json")")" + record "${w}" "denied UDP 9: leaves the guest" yes "$(seen "$(count "${w}" tap80-host "${NONCE}-${name}-h2-deny-out")")" + record "${w}" "denied UDP 9: reaches no client" 0 "$(count "${w}" client "${NONCE}-${name}-h2-deny-out")" + # The dispatcher redirects the return before tcpdump on the receiving + # device sees it: count it where it is sent and where it is delivered. + record "${w}" "denied TCP 9 return: leaves the client" yes \ + "$(seen "$(count "${w}" client "" --dst "${SHARD_A}" --proto tcp --dport 9)")" + record "${w}" "denied TCP 9 return: reaches the pair's tenant end" yes \ + "$(seen "$(count "${w}" pair80 "" --dst "${TAP_GUEST}" --proto tcp --dport 9)")" + record "${w}" "denied TCP 9 return: reaches no guest" 0 "$(count "${w}" tap80-guest "" --proto tcp --dport 9)" + no_trace "${w}" "${NONCE}-${name}-h2" + + trace "${name}" h2 + probe teardown "${cfg}" >>"${OUT}/logs/teardown.txt" 2>&1 + trace "${name}" teardown + rig /hp/handoff-rig.sh pairs-down + trace "${name}" pairs-down + rig /hp/handoff-rig.sh dut-down + trace "${name}" dut-down +} + +# What xdpmode.Select decides for the lab's own interfaces. +rig /hp/handoff-rig.sh dut-up plain >/dev/null +record cell/select "xdpmode.Select on ${NODE}" native "$(probe mode hp-dut0 | jq -r .mode)" +probe mode hp-dut0 >"${OUT}/logs/select.json" +rig /hp/handoff-rig.sh dut-down + +for c in "${CELLS[@]}"; do + run_cell "${c}" +done + +# local.routed_link and local.socket ---------------------------------------- +echo "=== routed link and socket ===" +rig /hp/handoff-rig.sh links-up "${PID[envoy]}" +cfg=$(jq -n --arg pin /sys/fs/bpf/galactic-handoff-proof --arg gpin "${GALACTIC_PIN}" --arg link "${LINK_NET}" \ + --arg vrfA "${VRF_A}" --arg vrfB "${VRF_B}" '{pinDir: $pin, galacticPinDir: $gpin, mode: "native", + guards: [{iface: "hpr80h", prefix: $link}, {iface: "hpr81h", prefix: $link}], + passThrough: [{vrf: $vrfA, prefix: $link}, {vrf: $vrfB, prefix: $link}]}' | probe_config routed-link) +probe setup "${cfg}" >"${OUT}/logs/setup-routed-link.txt" 2>&1 +E="pid:${PID[envoy]}" +for v in 80 81; do + rig_bg nsenter -t "${PID[envoy]}" -n socat "TCP6-LISTEN:8081,so-bindtodevice=vrf${v},fork,reuseaddr" \ + "SYSTEM:read -r l; echo \"\$l\" >>/out/logs/envoy${v}.reads; echo \"\$l envoy${v}\"" +done +sleep 1 +for v in 80 81; do + ask "${E}" 2 "TCP6:[${VETH_GUEST}]:8080,so-bindtodevice=vrf${v}" warm >/dev/null +done +# The listeners run as root in the rig, so the rig empties their logs. +rig sh -c ': >/out/logs/envoy80.reads; : >/out/logs/envoy81.reads' +w=h3 +cap "${w}" veth80-guest "pid:${PID[veth80]}" eth0 +cap "${w}" veth81-guest "pid:${PID[veth81]}" eth0 +cap "${w}" link80-host node hpr80h +cap "${w}" link81-host node hpr81h +cap "${w}" envoy "${E}" hpr80p hpr81p +cap "${w}" physical node "${PHYSICAL_IFS[@]}" +cap_wait +for v in 80 81; do + tag="${NONCE}-h3-${v}" + answers=$(for i in $(seq 1 "${N}"); do + ask "${E}" 3 "TCP6:[${VETH_GUEST}]:8080,so-bindtodevice=vrf${v}" "${tag}-${i}" + done) + record "${w}" "socket bound to vrf${v} reads vpc${v}'s guest" "${N}" "$(grep -c " vpc${v}\$" <<<"${answers}" || true)" + o=81 + [ "${v}" = 81 ] && o=80 + record "${w}" "socket bound to vrf${v} reads vpc${o}'s guest" 0 "$(grep -c " vpc${o}\$" <<<"${answers}" || true)" + tag="${NONCE}-h3-rev-${v}" + answers=$(for i in $(seq 1 "${N}"); do + ask "pid:${PID[veth${v}]}" 3 "TCP6:[${LINK_POD}]:8081" "${tag}-${i}" + done) + record "${w}" "vpc${v}'s guest is answered by the vrf${v} socket" "${N}" \ + "$(grep -c " envoy${v}\$" <<<"${answers}" || true)" +done +ask "${E}" 3 "TCP6:[${VETH_GUEST}]:9,so-bindtodevice=vrf80" "${NONCE}-h3-deny" >/dev/null +ask "${E}" 2 "TCP6:[${VETH_GUEST}]:8080" "${NONCE}-h3-unbound" >"${OUT}/logs/unbound.txt" +in_pod "${PID[envoy]}" ip -6 addr add fd20:80:ff04::5/128 dev hpr80p nodad +printf '%s\n' "${NONCE}-h3-spoof" | docker exec -i "${RIG}" timeout 3 nsenter -t "${PID[envoy]}" -n \ + socat -u - "UDP6-SENDTO:[${VETH_GUEST}]:8080,bind=[fd20:80:ff04::5],so-bindtodevice=vrf80" || true +in_pod "${PID[envoy]}" ip -6 addr del fd20:80:ff04::5/128 dev hpr80p +probe counters "${cfg}" >"${OUT}/logs/counters-h3.json" +cap_stop +for v in 80 81; do + o=81 + [ "${v}" = 81 ] && o=80 + record "${w}" "vrf${v} socket reads the guests' lines" "${N}" \ + "$(grep -c "^${NONCE}-h3-rev-${v}-" "${OUT}/logs/envoy${v}.reads" || true)" + record "${w}" "vpc${v}'s guest receives vrf${v}'s requests" "${N}" \ + "$(count "${w}" "veth${v}-guest" "${NONCE}-h3-${v}" --dst "${VETH_GUEST}")" + record "${w}" "vpc${o}'s guest receives vrf${v}'s requests" 0 \ + "$(count "${w}" "veth${o}-guest" "${NONCE}-h3-${v}")" +done +record "${w}" "denied TCP 9: crosses the routed link" yes \ + "$(seen "$(count "${w}" link80-host "" --dst "${VETH_GUEST}" --proto tcp --dport 9)")" +record "${w}" "denied TCP 9: reaches no guest" 0 \ + "$(($(count "${w}" veth80-guest "" --proto tcp --dport 9) + $(count "${w}" veth81-guest "" --proto tcp --dport 9)))" +record "${w}" "unbound socket: gets no answer" 0 "$(grep -c . "${OUT}/logs/unbound.txt" || true)" +record "${w}" "unbound socket: reaches no guest" 0 \ + "$(($(count "${w}" veth80-guest "${NONCE}-h3-unbound") + $(count "${w}" veth81-guest "${NONCE}-h3-unbound")))" +record "${w}" "spoofed source: crosses the routed link" yes "$(seen "$(count "${w}" link80-host "${NONCE}-h3-spoof")")" +record "${w}" "spoofed source: reaches no guest" 0 "$(count "${w}" veth80-guest "${NONCE}-h3-spoof")" +record "${w}" "link guard drops the spoofed source" yes "$(seen "$(counter guard_drop <"${OUT}/logs/counters-h3.json")")" +no_trace "${w}" "${NONCE}-h3" +# cleanup (the EXIT trap) removes the rig and the probe, and finishes. diff --git a/deploy/containerlab/scripts/handoff-pcap.py b/deploy/containerlab/scripts/handoff-pcap.py new file mode 100755 index 00000000..4d577773 --- /dev/null +++ b/deploy/containerlab/scripts/handoff-pcap.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +# Copyright 2026 Datum Cloud, Inc. +# +# SPDX-License-Identifier: AGPL-3.0-or-later +"""Count the packets in pcap files that carry a probe nonce. + +Usage: handoff-pcap.py [--nonce TEXT] [--src ADDR] [--dst ADDR] + [--proto tcp|udp] [--dport PORT] [--srv6] FILE... + +A packet counts when its frame contains the nonce bytes anywhere, including +inside an encapsulation, and its outer IPv6 source and destination match +--src and --dst when given. --proto and --dport match the transport header +that directly follows the outer IPv6 header, for a packet with no payload to +carry a nonce, such as a TCP SYN to a denied port. --srv6 counts only packets that carry an IPv6 +Routing Header of type 4 (an SRv6 Segment Routing Header) or that are IPv6 +in IPv6, the forms the uSID datapath encapsulates in. The sum over every +file is printed. A missing or empty file counts as zero, and a truncated +one counts up to its last whole record. +""" + +import argparse +import ipaddress +import struct +import sys + +ETH_P_IPV6 = 0x86DD +VLAN_TYPES = (0x8100, 0x88A8) +LINKTYPE_ETHERNET = 1 +LINKTYPE_LINUX_SLL = 113 +LINKTYPE_LINUX_SLL2 = 276 +LINKTYPE_RAW = 101 +EXT_HEADERS = (0, 43, 44, 51, 60) + + +def records(path): + try: + with open(path, "rb") as f: + data = f.read() + except FileNotFoundError: + return + if len(data) < 24: + return + magic = data[:4] + if magic in (b"\xd4\xc3\xb2\xa1", b"\x4d\x3c\xb2\xa1"): + order = "<" + elif magic in (b"\xa1\xb2\xc3\xd4", b"\xa1\xb2\x3c\x4d"): + order = ">" + else: + raise ValueError(f"{path}: not a pcap file") + linktype = struct.unpack(order + "I", data[20:24])[0] & 0x0FFFFFFF + off = 24 + while off + 16 <= len(data): + incl = struct.unpack(order + "I", data[off + 8 : off + 12])[0] + off += 16 + if off + incl > len(data): + return + yield linktype, data[off : off + incl] + off += incl + + +def ipv6_offset(linktype, frame): + """Offset of the outer IPv6 header, or None.""" + if linktype == LINKTYPE_ETHERNET: + off, ethertype = 14, None + if len(frame) < 14: + return None + ethertype = struct.unpack(">H", frame[12:14])[0] + while ethertype in VLAN_TYPES and len(frame) >= off + 4: + ethertype = struct.unpack(">H", frame[off + 2 : off + 4])[0] + off += 4 + elif linktype == LINKTYPE_LINUX_SLL: + off = 16 + ethertype = struct.unpack(">H", frame[14:16])[0] if len(frame) >= 16 else None + elif linktype == LINKTYPE_LINUX_SLL2: + off = 20 + ethertype = struct.unpack(">H", frame[0:2])[0] if len(frame) >= 20 else None + elif linktype == LINKTYPE_RAW: + off, ethertype = 0, ETH_P_IPV6 + else: + return None + if ethertype != ETH_P_IPV6 or len(frame) < off + 40 or frame[off] >> 4 != 6: + return None + return off + + +def encapsulated(frame, off): + """True for an SRv6 Routing Header (type 4) or IPv6-in-IPv6 chain.""" + nexthdr = frame[off + 6] + pos = off + 40 + for _ in range(8): + if nexthdr == 41: + return True + if nexthdr == 43 and len(frame) >= pos + 4 and frame[pos + 2] == 4: + return True + if nexthdr not in EXT_HEADERS or len(frame) < pos + 2: + return False + length = 8 if nexthdr == 44 else (frame[pos + 1] + 1) * 8 + nexthdr = frame[pos] + pos += length + return False + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--nonce", default="") + ap.add_argument("--proto", choices=("tcp", "udp")) + ap.add_argument("--dport", type=int) + ap.add_argument("--src") + ap.add_argument("--dst") + ap.add_argument("--srv6", action="store_true") + ap.add_argument("files", nargs="*") + args = ap.parse_args() + nonce = args.nonce.encode() + src = ipaddress.IPv6Address(args.src).packed if args.src else None + dst = ipaddress.IPv6Address(args.dst).packed if args.dst else None + + total = 0 + for path in args.files: + for linktype, frame in records(path): + if nonce not in frame: + continue + off = ipv6_offset(linktype, frame) + if off is None: + if src or dst or args.srv6 or args.proto or args.dport is not None: + continue + total += 1 + continue + if src and frame[off + 8 : off + 24] != src: + continue + if dst and frame[off + 24 : off + 40] != dst: + continue + if args.srv6 and not encapsulated(frame, off): + continue + if args.proto or args.dport is not None: + nexthdr = frame[off + 6] + if args.proto and nexthdr != {"tcp": 6, "udp": 17}[args.proto]: + continue + if nexthdr not in (6, 17) or len(frame) < off + 44: + continue + if args.dport is not None and struct.unpack(">H", frame[off + 42 : off + 44])[0] != args.dport: + continue + total += 1 + print(total) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/deploy/containerlab/scripts/handoff-rig.sh b/deploy/containerlab/scripts/handoff-rig.sh new file mode 100755 index 00000000..9d52039f --- /dev/null +++ b/deploy/containerlab/scripts/handoff-rig.sh @@ -0,0 +1,232 @@ +#!/bin/bash +# handoff-rig.sh: build and remove the scratch rig the Local handoff checks +# run on (docs/handoffs.md). It runs inside the rig container, which shares +# the compute node's network and PID namespaces, so every interface it makes +# is on the node, and the client namespace is the rig's own. +# +# Usage, inside the rig: +# +# handoff-rig.sh dut-up plain|bond|vlan [MTU] receiving device and client +# handoff-rig.sh dut-down +# handoff-rig.sh pairs-up GRO internal veth pairs, GRO on|off on the tenant ends +# handoff-rig.sh pairs-down +# handoff-rig.sh vpair-up | vpair-down a pair for VPC 80's veth guest +# handoff-rig.sh links-up PID routed links into the pod whose process is PID +# handoff-rig.sh links-down +# handoff-rig.sh client-up the client namespace alone +# handoff-rig.sh down everything above +# +# The receiving device is always reached as hp-dut (its L3 interface name is +# printed by dut-up), the client namespace is hp-client and the client's end +# of the link carries 2001:db8:4144:1::2. +set -euo pipefail + +CLIENT_NS=hp-client +NODE_ADDR=2001:db8:4144:1::1 +CLIENT_ADDR=2001:db8:4144:1::2 +VLAN_ID=120 +VRF_A=G000000080V +VRF_B=G000000081V +LINK_HOST=fd20:80:ff04:: +LINK_POD=fd20:80:ff04::1 + +client() { ip netns exec "${CLIENT_NS}" "$@"; } + +# lacp_bond NETNS BOND MEMBER... builds the lab's bond (mkbond.sh's +# parameters) in NETNS, or on the node when NETNS is empty. +lacp_bond() { + local ns="$1" bond="$2" m + shift 2 + local run=() + [ -n "${ns}" ] && run=(ip netns exec "${ns}") + "${run[@]}" ip link add "${bond}" type bond mode 802.3ad miimon 100 lacp_rate fast xmit_hash_policy layer3+4 + for m in "$@"; do + "${run[@]}" ip link set "${m}" down + "${run[@]}" ip link set "${m}" master "${bond}" + "${run[@]}" ip link set "${m}" up + done + "${run[@]}" ip link set "${bond}" up +} + +dut_up() { + local kind="$1" mtu="${2:-1500}" l3 cl3 + ip netns add "${CLIENT_NS}" 2>/dev/null || true + client ip link set lo up + case "${kind}" in + plain) + ip link add hp-dut0 mtu "${mtu}" type veth peer name hp-cl0 mtu "${mtu}" netns "${CLIENT_NS}" + l3=hp-dut0 + cl3=hp-cl0 + ip link set hp-dut0 up + client ip link set hp-cl0 up + ;; + bond) + ip link add hp-dut0 mtu "${mtu}" type veth peer name hp-cl0 mtu "${mtu}" netns "${CLIENT_NS}" + ip link add hp-dut1 mtu "${mtu}" type veth peer name hp-cl1 mtu "${mtu}" netns "${CLIENT_NS}" + lacp_bond "" hp-bond hp-dut0 hp-dut1 + lacp_bond "${CLIENT_NS}" hp-cbond hp-cl0 hp-cl1 + ip link set hp-bond mtu "${mtu}" + client ip link set hp-cbond mtu "${mtu}" + l3=hp-bond + cl3=hp-cbond + ;; + vlan) + ip link add hp-dut0 mtu "${mtu}" type veth peer name hp-cl0 mtu "${mtu}" netns "${CLIENT_NS}" + ip link set hp-dut0 up + client ip link set hp-cl0 up + ip link add link hp-dut0 name hp-dut0.${VLAN_ID} type vlan id "${VLAN_ID}" + client ip link add link hp-cl0 name hp-cl0.${VLAN_ID} type vlan id "${VLAN_ID}" + ip link set hp-dut0.${VLAN_ID} up + client ip link set hp-cl0.${VLAN_ID} up + l3=hp-dut0.${VLAN_ID} + cl3=hp-cl0.${VLAN_ID} + ;; + *) + echo "dut-up: unknown kind ${kind}" >&2 + return 2 + ;; + esac + # A native XDP_REDIRECT into a veth needs NAPI on its peer; GRO turns it + # on. The client's ends stand in for the far side of a physical link, + # which needs nothing of the kind, so they get it unconditionally. + # They also stand in for a physical link in what they send: a NIC delivers + # complete checksums, while a veth from a local sender carries offloaded, + # partial ones (docs/handoffs.md). CLIENT_TX=on keeps the offload, for the + # measurement of that case. + for m in hp-cl0 hp-cl1; do + if client ip link show "${m}" >/dev/null 2>&1; then + client ethtool -K "${m}" gro on >/dev/null + client ethtool -K "${m}" tx "${CLIENT_TX:-off}" >/dev/null + fi + done + for m in hp-cbond hp-cl0.${VLAN_ID}; do + if client ip link show "${m}" >/dev/null 2>&1; then + client ethtool -K "${m}" tx "${CLIENT_TX:-off}" >/dev/null 2>&1 || true + fi + done + ip -6 addr add "${NODE_ADDR}/64" dev "${l3}" nodad + client ip -6 addr add "${CLIENT_ADDR}/64" dev "${cl3}" nodad + client ip -6 route add 2001:db8:4144::/48 via "${NODE_ADDR}" dev "${cl3}" + if [ "${kind}" = bond ]; then + wait_lacp hp-bond hp-dut0 hp-dut1 + fi + # Resolve both neighbors now, so the first probe is not the one that + # waits for Neighbor Discovery. + client ping -6 -c 1 -W 2 "${NODE_ADDR}" >/dev/null 2>&1 || true + ping -6 -c 1 -W 2 "${CLIENT_ADDR}" >/dev/null 2>&1 || true + echo "${l3}" +} + +# wait_lacp BOND MEMBER... waits until every member is distributing. +wait_lacp() { + local bond="$1" i n + shift + for i in $(seq 1 60); do + # Members whose own (actor) LACP state is collecting and distributing. + n=$(awk '/details actor lacp pdu:/ { actor = 1; next } + /details partner lacp pdu:/ { actor = 0 } + actor && /port state:/ { if (int($3 / 16) % 4 == 3) n++ } + END { print n + 0 }' "/proc/net/bonding/${bond}" 2>/dev/null || echo 0) + if [ "${n}" -ge "$#" ]; then + return 0 + fi + sleep 0.5 + done + echo "bond ${bond} did not aggregate" >&2 + cat "/proc/net/bonding/${bond}" >&2 || true + return 1 +} + +# dut_down removes the receiving device and the client's links, and keeps +# the client namespace, so listeners started in it survive between cells. +dut_down() { + local l + for l in hp-bond hp-dut0 hp-dut1; do + ip link del "${l}" 2>/dev/null || true + done + client ip link del hp-cbond 2>/dev/null || true +} + +pairs_up() { + local gro="$1" v vrf + for v in 80 81; do + vrf=${VRF_A} + [ "${v}" = 81 ] && vrf=${VRF_B} + ip link add "hpn${v}t" type veth peer name "hpn${v}n" + ip link set "hpn${v}t" master "${vrf}" + ip link set "hpn${v}t" up + ip link set "hpn${v}n" up + ethtool -K "hpn${v}t" gro "${gro}" >/dev/null + done +} + +pairs_down() { + ip link del hpn80t 2>/dev/null || true + ip link del hpn81t 2>/dev/null || true +} + +# vpair_up: one more internal veth pair in VPC 80, for flows from ns80's veth +# guest, whose sends carry offloaded checksums. +vpair_up() { + ip link add hpv80t type veth peer name hpv80n + ip link set hpv80t master "${VRF_A}" + ip link set hpv80t up + ip link set hpv80n up + ethtool -K hpv80t gro on >/dev/null +} + +# links_up PID: a routed link from each tenant VRF into the pod, ending in a +# VRF of the pod's own (vrf80, vrf81), as the galactic-vrf sidecar gives +# Envoy. Both VPCs use the same link addresses. +links_up() { + local pid="$1" v vrf table + for v in 80 81; do + vrf=${VRF_A} + [ "${v}" = 81 ] && vrf=${VRF_B} + table=$((1000 + v)) + ip link add "hpr${v}h" type veth peer name "hpr${v}p" netns "${pid}" + ip link set "hpr${v}h" master "${vrf}" + ip -6 addr add "${LINK_HOST}/127" dev "hpr${v}h" nodad + ip link set "hpr${v}h" up + nsenter -t "${pid}" -n sh -ec " + ip link show vrf${v} >/dev/null 2>&1 || ip link add vrf${v} type vrf table ${table} + ip link set vrf${v} up + ip link set hpr${v}p master vrf${v} + ip -6 addr add ${LINK_POD}/127 dev hpr${v}p nodad + ip link set hpr${v}p up + ip -6 route replace fd20:80::/32 via ${LINK_HOST} dev hpr${v}p vrf vrf${v} + ip -6 route replace unreachable default vrf vrf${v} metric 4278198272 + " + done +} + +links_down() { + ip link del hpr80h 2>/dev/null || true + ip link del hpr81h 2>/dev/null || true +} + +case "${1:-}" in + client-up) + ip netns add "${CLIENT_NS}" 2>/dev/null || true + client ip link set lo up + ;; + dut-up) dut_up "${2:?kind}" "${3:-1500}" ;; + dut-down) dut_down ;; + pairs-up) pairs_up "${2:?gro on|off}" ;; + pairs-down) pairs_down ;; + vpair-up) vpair_up ;; + vpair-down) ip link del hpv80t 2>/dev/null || true ;; + links-up) links_up "${2:?pid}" ;; + links-down) links_down ;; + down) + links_down + pairs_down + ip link del hpv80t 2>/dev/null || true + dut_down + ip netns del "${CLIENT_NS}" 2>/dev/null || true + ;; + *) + sed -n '2,20p' "$0" >&2 + exit 2 + ;; +esac diff --git a/deploy/containerlab/scripts/handoff-twice.sh b/deploy/containerlab/scripts/handoff-twice.sh new file mode 100755 index 00000000..e6f25230 --- /dev/null +++ b/deploy/containerlab/scripts/handoff-twice.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# handoff-twice.sh: deploy the Local handoff fixture, run handoff-observe.sh +# twice and handoff-measure.sh once, remove the fixture, and require both +# observation runs to pass with identical results.json files that also match +# the committed handoffs/expected-results.json. +# +# Usage: handoff-twice.sh [OUTDIR] +# OUTDIR defaults to bin/handoffs/ at the repository root. +# +# Set HANDOFF_WRITE_EXPECTED=1 to replace the expected results with run 1's +# once both runs pass and agree. Set HANDOFF_MEASURE=0 to skip the +# measurements. +# +# When a precondition does not hold it changes nothing, says which, and +# exits 0: ns80 must be deployed, and no earlier fixture or rig may remain. +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +source "${SCRIPT_DIR}/lib.sh" +LAB_DIR=$(cd "${SCRIPT_DIR}/.." && pwd) +REPO=$(cd "${LAB_DIR}/../.." && pwd) +EXPECTED="${LAB_DIR}/handoffs/expected-results.json" +OUT=${1:-${REPO}/bin/handoffs/$(date -u +%Y%m%dT%H%M%SZ)} +mkdir -p "${OUT}" +OUT=$(cd "${OUT}" && pwd) +CP=$(control_plane dfw) + +precondition() { + echo "PRECONDITION: $*" | tee "${OUT}/precondition" + exit 0 +} + +if ! docker exec "${CP}" kubectl get namespace ns80 >/dev/null 2>&1; then + precondition "ns80 is not deployed (task deploy:ns80)" +fi +if docker exec "${CP}" kubectl get namespace ns80-tap >/dev/null 2>&1; then + precondition "ns80-tap is already deployed; an earlier run did not remove it (task delete:ns80-tap)" +fi +if docker inspect handoff-rig >/dev/null 2>&1; then + precondition "a rig container handoff-rig already exists; an earlier run did not clean up" +fi + +rc=0 +fixture_down() { + "${SCRIPT_DIR}/handoff-fixture.sh" down || rc=1 +} +trap 'status=$?; fixture_down; [ "${status}" -ne 0 ] && rc=${status}; exit "${rc}"' EXIT +"${SCRIPT_DIR}/handoff-fixture.sh" up 2>&1 | tee "${OUT}/fixture-up.log" + +for run in 1 2; do + echo "##### run ${run} #####" + "${SCRIPT_DIR}/handoff-observe.sh" "${OUT}/run-${run}" 2>&1 | tee "${OUT}/run-${run}.log" || rc=1 + if [ -f "${OUT}/run-${run}/precondition" ]; then + cp "${OUT}/run-${run}/precondition" "${OUT}/precondition" + echo "stopped at a precondition: $(cat "${OUT}/precondition")" + exit 0 + fi +done +if [ "${HANDOFF_MEASURE:-1}" = 1 ]; then + echo "##### measurements #####" + "${SCRIPT_DIR}/handoff-measure.sh" "${OUT}/measure" 2>&1 | tee "${OUT}/measure.log" || rc=1 +fi + +if diff -u "${OUT}/run-1/results.json" "${OUT}/run-2/results.json" >"${OUT}/runs.diff"; then + echo "runs 1 and 2: identical results ($(sha256sum <"${OUT}/run-1/results.json" | cut -c1-16))" +else + echo "FAIL: runs 1 and 2 differ:" >&2 + cat "${OUT}/runs.diff" >&2 + rc=1 +fi +if [ "${rc}" -eq 0 ] && [ "${HANDOFF_WRITE_EXPECTED:-0}" = 1 ]; then + mkdir -p "$(dirname "${EXPECTED}")" + cp "${OUT}/run-1/results.json" "${EXPECTED}" + echo "wrote ${EXPECTED}" +elif [ ! -f "${EXPECTED}" ]; then + echo "FAIL: no ${EXPECTED}; rerun with HANDOFF_WRITE_EXPECTED=1 to create it" >&2 + rc=1 +elif diff -u "${EXPECTED}" "${OUT}/run-1/results.json" >"${OUT}/expected.diff"; then + echo "run 1 matches handoffs/expected-results.json" +else + echo "FAIL: run 1 differs from handoffs/expected-results.json:" >&2 + cat "${OUT}/expected.diff" >&2 + rc=1 +fi +echo "evidence in ${OUT}" diff --git a/deploy/containerlab/scripts/publish-attachments.sh b/deploy/containerlab/scripts/publish-attachments.sh index 39567259..599d94dc 100755 --- a/deploy/containerlab/scripts/publish-attachments.sh +++ b/deploy/containerlab/scripts/publish-attachments.sh @@ -9,8 +9,9 @@ # reports once scheduled: # # - vpc, vpcAttachment: the NAD's identifiers; -# - node, podName: where the pod using the NAD (as its -# v1.multus-cni.io/default-network) runs; +# - node, podName: where the pod using the NAD runs, as its +# v1.multus-cni.io/default-network or as a secondary network in its +# k8s.v1.cni.cncf.io/networks list (a TAP guest, ns80-tap); # - hostInterface: the host side of its veth, GH # (internal/plumbing/intf), which galactic-router's traffic policy # reconciler binds the attachment's policies to. @@ -60,16 +61,19 @@ spec: EOF fi - # The pod naming this NAD as its default network, by bare name or - # namespace/name: one that is running and not being deleted when there - # is one, so a rollout's old pod is never the one published. + # The pod naming this NAD as its default network or a secondary one, by + # bare name or namespace/name: one that is running and not being deleted + # when there is one, so a rollout's old pod is never the one published. pod="" for i in $(seq 1 60); do pod=$(docker exec "${cp}" kubectl get pods -n "${NS}" -o json | jq -r --arg nad "${nad}" --arg qualified "${NS}/${nad}" ' [.items[] - | select(.metadata.annotations["v1.multus-cni.io/default-network"] as $n - | $n == $nad or $n == $qualified) + | select((.metadata.annotations["v1.multus-cni.io/default-network"] as $n + | $n == $nad or $n == $qualified) + or ((.metadata.annotations["k8s.v1.cni.cncf.io/networks"] // "") + | split(",") | map(gsub("^\\s+|\\s+$"; "") | split("@")[0]) + | any(. == $nad or . == $qualified))) | select(.spec.nodeName != null and .metadata.deletionTimestamp == null)] | (map(select(.status.phase == "Running")) + .)[0].metadata.name // empty') [ -n "${pod}" ] && break From 20215e0353efb21fb40a8196bc7e92512b4dd185 Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Thu, 19 Nov 2026 03:23:14 -0500 Subject: [PATCH 4/6] docs(worker): Amend the contract with the proven Local handoffs The Local handoffs were proven in the lab, and several mechanisms the qualification plan named do not work as specified. The contract now states what the Local module programs and what each handoff requires: native XDP on the veth peer whatever the host's mode, since generic XDP skips tc-redirected packets; a VLAN egress through a pushed tag and the lower device; policy decided on each handoff's own path and marked; Local pass-through entries; tc order after galactic-cni's filters; no other tc owner on a tenant TAP; and the MTU, fragment and checksum limits. The qualification plan lists the supported combinations and marks the unsupported ones HookUnsupported. --- docs/worker/README.md | 1 + docs/worker/contract.md | 82 ++++++++++++++++++++++++++++++++++++ docs/worker/qualification.md | 40 +++++++++++++++++- 3 files changed, 121 insertions(+), 2 deletions(-) diff --git a/docs/worker/README.md b/docs/worker/README.md index 19b95c04..a816439c 100644 --- a/docs/worker/README.md +++ b/docs/worker/README.md @@ -61,3 +61,4 @@ PR. "Confirmed" means the product owner approved the decision on the date given. | D24 | galactic-router hands Worker its tenant routes through `Routes.Sync` on the Worker socket, behind the `fabric.routes` capability, applying D8 | Accepted | [Contract](contract.md#tenant-route-handoff) | | D25 | Each destination reports `forward_ready` and `return_ready`; `ready` requires both | Accepted | [Contract](contract.md#state-model) | | D26 | A destination's addresses come from its VPCAttachment's spec; one with none reports `DestinationAddressUnknown` | Accepted | [Contract](contract.md#trusted-sources) | +| D27 | Local handoffs as #1444 proved them: native XDP on the veth peer, VLAN and checksum limits, per-path policy, Local pass-through | Proposed | [Contract](contract.md#local-handoffs) | diff --git a/docs/worker/contract.md b/docs/worker/contract.md index b58863ea..449720ee 100644 --- a/docs/worker/contract.md +++ b/docs/worker/contract.md @@ -210,6 +210,88 @@ A claim is cross-VPC when any destination belongs to a VPC other than the claim's attachment VPC. Worker refuses it. Two VPCs may use the same addresses, so Worker compares VPC identity, never addresses. +## Local handoffs + +[#1444](https://github.com/datum-cloud/galactic/issues/1444) proved each Local +handoff on the [qualified platform](qualification.md), with prototypes placed by +today's loaders (D27). Each row below is the mechanism the Local module +([#1435](https://github.com/datum-cloud/galactic/issues/1435)) programs, and +each requirement is part of that handoff's `Ready`. The scripts, captures and +results are in +[the lab's handoff checks](../../deploy/containerlab/docs/handoffs.md). + +| Capability | Mechanism | +| ----------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `local.tap` | A dispatcher slot on the receiving interface returns `XDP_PASS` with metadata naming the TAP. A tc ingress program on the same interface decides the TAP's ingress policy, marks the packet decided and delivers with `bpf_redirect_neigh`. The reply leaves from the TAP's tc ingress. | +| `local.veth_pair` | A tc program on the tenant TAP or veth redirects into the pair's tenant end. XDP on the NAT end runs the shard. The return is redirected into the NAT end and received by tc on the tenant end, which decides policy and delivers. | +| `local.routed_link`, `local.socket` | A veth from a VRF in the pod to the tenant VRF on the host, with a `/127` on each end. A socket bound to the pod's VRF (`SO_BINDTODEVICE`) sends and receives through it with no redirect; the kernel routes both ways. | + +These amend the mechanisms the qualification plan first named: + +- **The NAT end of a veth pair runs native XDP, whatever mode the host + selects.** Generic XDP skips every packet a tc program redirected (the + kernel's `skb_is_redirected` check in the generic receive path), so on a host + that selects generic mode, as one with a `bnxt_en` NIC does, XDP on the peer + never sees the tenant's traffic. A veth supports native XDP on every kernel the + platform allows, so the peer is attached native while the uplinks follow + `xdpmode.Select`. +- **A native `XDP_REDIRECT` from the peer to a VLAN device is unsupported**: a + VLAN device has no XDP transmit. Where the egress is a VLAN, the peer pushes + the 802.1Q tag itself and redirects to the VLAN's lower device. Returning + `XDP_PASS` so the kernel forwards is not an alternative: the return comes + back through the dispatcher's `XDP_REDIRECT`, past netfilter, so conntrack + sees the flow's later packets as invalid and the node drops them, which + stops TCP after the handshake. +- **Native XDP does not attach to a VLAN device.** A `local.tap` receiving + interface that is a VLAN takes its native dispatcher on the VLAN's lower + device and its tc program on the VLAN device; the metadata survives the + kernel's untag. With VLAN receive offload, as on veth and most NICs, XDP on + the lower device sees the frame untagged and cannot tell VLANs apart, so the + slot program selects by destination, never by VLAN. +- **Each handoff decides the receiver's policy on its own path and marks the + packet.** `tp_deliver` decides every unmarked packet on the Private path. A + public client delivered to a TAP is PublicService and a NAT return is + PublicNAT, so the handoff's tc program calls the datapath's policy decision + with that path and sets the decided mark, as `usid_ingress` does for gateway + deliveries. Left to `tp_deliver`, a public client is denied by any policy that + grants PublicService and not Private. +- **A Local destination needs a pass-through entry in `egress_route_table`.** + Without one, `usid_egress` sends a tenant's packet for a Local peer (a public + client, the shard address's client, the routed link) to the VPC's `::/0` NAT + route, encapsulated. Worker writes the pass-through under its own owner. The + egress decision `usid_egress` then makes is on the Private path, so until + [#1435](https://github.com/datum-cloud/galactic/issues/1435) moves the Local + branch into `usid_egress`, a tenant's Private egress grant also governs its + Local flows, and the handoff's own program decides again on the right path. +- **No other tc owner may claim a tenant TAP.** A node's other datapath, + Cilium in particular, must never attach to a tenant interface. Deployed + configurations meet this by pinning Cilium's `devices` to the node's uplink + bonds and primary Ethernet interfaces, and the lab now does the same + (`deploy/containerlab/scripts/deploy-cni.sh`). When the rule is broken, as in + the lab before that pin, Cilium with kube-proxy replacement auto-detects any + interface with a global address. A host TAP has one, the guest's gateway, so + Cilium adopts it, registers that tenant address as a node address and + attaches its programs with tcx, which deletes every legacy tc filter on the + TAP: the egress chain, `tp_deliver` and any Local program. The TAP then + forwards without its traffic policy; the #1444 runs recorded a public client + reaching a guest whose policy denies it. Worker checks the TAP's chain as + observed state and reports `HandoffInterfaceMissing` until it is present. + Veths are not affected: Cilium excludes them. +- **A handoff program on a tenant interface runs after galactic-cni's own.** + Its filters take the lowest tc priorities on the interface's ingress, so a + separate Local filter sits at the next priority and sees only what + `usid_egress` passed on. #1435 either keeps that order or moves the Local + branch into `usid_egress`. + +Each handoff places requirements on its interfaces: + +| Handoff | Requirement | +| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `local.tap` | The TAP has carrier only while its reader (the hypervisor) holds it; Worker reports `HandoffInterfaceMissing` until then. `bpf_redirect_neigh` does not check the TAP's MTU: the lab delivered 8049-byte packets into a TAP at MTU 1500, and its guest took them. A real guest drops frames above its own MTU, with no ICMPv6 Packet Too Big to the sender, so the TAP's MTU is at least the receiving path's. IPv6 fragments to a TAP guest were not delivered in either mode; #1435 decides whether Local delivery reassembles or refuses them. | +| `local.veth_pair` | The tenant end of the pair needs GRO (or an XDP program) for a native `XDP_REDIRECT` into the pair to be received. XDP rewrites checksums as complete, so a frame with an offloaded, partial checksum is corrupted; see the checksum row in the [qualification plan](qualification.md#handoffs). | +| `local.routed_link` | The host end is a tenant interface the pod controls. It drops any source outside its `/127`, and the routed link's prefix has a pass-through entry in the tenant VRF. | +| `local.socket` | The socket is bound to the pod's VRF. An unbound socket uses the pod's main table and reaches no tenant. | + ## Errors and retry | gRPC code | Reasons | Client action | diff --git a/docs/worker/qualification.md b/docs/worker/qualification.md index c823edb7..a6579e6f 100644 --- a/docs/worker/qualification.md +++ b/docs/worker/qualification.md @@ -2,7 +2,7 @@ Status: proposed. The NIC and the numeric limits (D16, D18) were confirmed on 2026-11-18. [#1444](https://github.com/datum-cloud/galactic/issues/1444) -proves the handoffs on this platform, and +proved the handoffs on this platform and amended the mechanisms that failed (D27), and [#1442](https://github.com/datum-cloud/galactic/issues/1442) runs the comparison and reports against the limits. @@ -19,7 +19,12 @@ comparison and reports against the limits. A combination of mode and device type that fails is recorded here as unsupported, and Worker reports it as `HookUnsupported` in its capabilities. -## Handoffs #1444 proves +The `bnxt_en` hardware itself is outstanding: no lab host has one. #1444 forced +generic mode on the lab's veth, bond and VLAN devices, which is the mode such a +host selects, and [#1442](https://github.com/datum-cloud/galactic/issues/1442) +carries the run on the NIC. + +## Handoffs | Handoff | Capability | Consumer | | ------------------------------------------------------------------------------------------------------- | ----------------------------------- | --------------------------------------------------------------------------- | @@ -29,6 +34,37 @@ unsupported, and Worker reports it as `HookUnsupported` in its capabilities. Each proof covers both directions, a packet that policy denies, and the absence of local encapsulation on the physical wire. +[#1444](https://github.com/datum-cloud/galactic/issues/1444) proved them in the +lab ([handoff checks](../../deploy/containerlab/docs/handoffs.md)); the +mechanisms it adopted, and their requirements, are in the +[contract](contract.md#local-handoffs). + +### Supported combinations + +| Handoff | Interface | Native | Generic | +| ----------------------------------- | -------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------- | +| `local.tap` | Plain receiving interface | Supported | Supported | +| `local.tap` | Bond member | Supported | Supported | +| `local.tap` | VLAN device | `HookUnsupported`: native XDP does not attach. Use its lower device. | Supported | +| `local.tap` | VLAN's lower device, tc on the VLAN | Supported | Not needed | +| `local.veth_pair` | NAT end of the pair | Supported | `HookUnsupported`: never sees tc-redirected packets. Attach it native. | +| `local.veth_pair` | Peer's `XDP_REDIRECT` to a plain or bond egress | Supported | Supported | +| `local.veth_pair` | Peer's `XDP_REDIRECT` to a VLAN egress | `HookUnsupported`: no XDP transmit. Push the tag and redirect to the lower device. | `HookUnsupported`: as native | +| `local.tap`, `local.veth_pair` | A veth receiving interface above about 3500 bytes of MTU | `HookUnsupported`: the dispatcher's programs are not multi-buffer, so native XDP does not attach (MTU 9000 in the lab) | Supported | +| `local.routed_link`, `local.socket` | Any | No XDP hook | No XDP hook | + +Measured in the lab ([handoff measurements](../../deploy/containerlab/docs/handoffs.md#measurements)): +a 1500-byte packet (the largest unfragmented at MTU 1500) reaches a TAP guest in +both modes, an IPv6 fragment does not, and 1 MiB crosses each handoff intact +when the sender's checksum offload is off. Generic XDP sees GSO frames of up to +34358 bytes when the sender uses TSO; native XDP on a veth sees at most 1514, +because the veth turns GSO off on its peer. + +Checksums: XDP that rewrites an address treats the L4 checksum as complete. A +frame that a sender on the same host sent through a veth with checksum offload +carries a partial checksum, and the rewrite corrupts it. Receiving NICs deliver +complete checksums; a pod on the node does not unless its offload is off. See +the [measurements](../../deploy/containerlab/docs/handoffs.md#measurements). ## Workload From c07cbd9c41261a46690f65446ec1294c76bfae17 Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Thu, 19 Nov 2026 03:34:14 -0500 Subject: [PATCH 5/6] style(containerlab): Reflow the Cilium devices comment Trim the comment on the pinned Cilium devices to the reason for the pin and point at the lab doc for the evidence, wrapped evenly. --- deploy/containerlab/scripts/deploy-cni.sh | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/deploy/containerlab/scripts/deploy-cni.sh b/deploy/containerlab/scripts/deploy-cni.sh index 02872234..a9a9e701 100755 --- a/deploy/containerlab/scripts/deploy-cni.sh +++ b/deploy/containerlab/scripts/deploy-cni.sh @@ -60,16 +60,11 @@ for site in dfw sjc iad; do # node silently kept serving the old config instead of applying the fix # above. `set -e` inside the inner script plus install-or-upgrade makes # reruns actually idempotent. - # devices pins the interfaces Cilium attaches its host programs to, as - # production deployments pin them to their uplink bonds and primary - # Ethernet interfaces. Left to - # auto-detection, Cilium also adopts every interface with a global address: - # a galactic-tap host TAP (it carries the guest's gateway), galactic-vip0. - # Its tcx attach then deletes every legacy tc filter there, including - # galactic-cni's egress chain and tp_deliver (docs/handoffs.md). Here the - # pin is eth0 and the bonds: eth+ would also match the bonds' members, - # eth1 to eth4, whose legacy tc filters (usid_ingress) Cilium would delete - # the same way. + # devices pins Cilium to eth0 and the bonds, as production deployments pin + # it to their uplinks. Auto-detected, it also adopts any interface with a + # global address, such as a galactic-tap host TAP, and its tcx attach then + # deletes galactic's legacy tc filters there (docs/handoffs.md). eth+ would + # match the bond members too, whose usid_ingress filters it would delete. # k8sServiceHost/k8sServicePort point the agent at the API server directly. # Without them it uses the kubernetes Service, which Cilium itself # load-balances, so an API server restart can leave the site unable to From 7130772499bb2b29e0c5e8da5e06be4ed388e5d8 Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Thu, 19 Nov 2026 03:55:50 -0500 Subject: [PATCH 6/6] ci: Generate the handoff probe in the required CI job scripts/required-ci.sh generates each eBPF package by name before go build ./..., so the new tests/handoffs/probe package failed to build there without its bpf2go output. Generate it as build:ebpf does, and run its unit tests with the rest. --- scripts/required-ci.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/required-ci.sh b/scripts/required-ci.sh index 0628779c..f70fade7 100755 --- a/scripts/required-ci.sh +++ b/scripts/required-ci.sh @@ -8,6 +8,7 @@ export BPF2GO_CC=clang-18 for pkg in prog edgeprog natprog xdpdispatch xmitdrops; do go generate "./internal/plumbing/ebpf/$pkg/..." done +go generate ./tests/handoffs/probe/... go build ./... @@ -15,4 +16,4 @@ assets=$(go run sigs.k8s.io/controller-runtime/tools/setup-envtest@32759fab31191 test -x "$assets/kube-apiserver" export KUBEBUILDER_ASSETS=$assets -go test -timeout 20m ./api/... ./cmd/... ./internal/... ./tests/edgeconformance/edgeprobe/... +go test -timeout 20m ./api/... ./cmd/... ./internal/... ./tests/edgeconformance/edgeprobe/... ./tests/handoffs/...