From 8c16831d7e51e93c502e038edc823f50b1ec6346 Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Wed, 18 Nov 2026 16:09:45 -0500 Subject: [PATCH 1/2] test(containerlab): add dedicated-placement baseline fixture Add ns80, two VPCs (80 and 81) whose guests take identical fixed addresses on dfw's and sjc's compute workers, with gateway and NAT left on the dedicated edge nodes. deploy:scenarios deploys it and verify:scenarios checks placement, the overlap and cross-site ping. Add task baseline:observe, baseline:twice and baseline:versions. The observations cover cross-node delivery in both VPCs, NAT66 and NAT64 egress from both overlapping guests at once, gateway ingress over IPv6 and IPv4, and a negative isolation control. Each scenario captures at the public side, the guests' veth pairs and the remote underlay. results.json keeps only stable fields, so two runs compare equal; ports, nonces and translated tuples go to volatile.tsv. baseline/pins.yaml pins the host tools, the Kind node, FRR, the remote-host base and the guest image digest, and the run fails on any mismatch. baseline/reference-results.json is the result of two identical passing runs. Recorded limits: inbound packets bypass the host veth through bpf_redirect_peer, the edge nodes forward in XDP out of tcpdump's view, and the lab's gateway checks intermittently lose a TCP flow at the ns60 responder, so those are observations or per-run values. The TAP capture point moved to datum-cloud/galactic#1444. Fixes datum-cloud/galactic#1432 --- deploy/containerlab/AGENTS.md | 1 + deploy/containerlab/README.md | 4 +- deploy/containerlab/Taskfile.yaml | 35 + deploy/containerlab/baseline/pins.yaml | 14 + .../baseline/reference-results.json | 758 ++++++++++++++++++ deploy/containerlab/docs/baseline.md | 149 ++++ deploy/containerlab/docs/tenants.md | 3 + .../resources/tenants/ns80/base/guests.yaml | 56 ++ .../tenants/ns80/base/kustomization.yaml | 8 + .../tenants/ns80/base/namespace.yaml | 4 + .../tenants/ns80/dfw/kustomization.yaml | 3 + .../resources/tenants/ns80/dfw/nad.yaml | 71 ++ .../tenants/ns80/policy/base/bindings.yaml | 20 + .../ns80/policy/base/kustomization.yaml | 7 + .../tenants/ns80/policy/kustomization.yaml | 3 + .../tenants/ns80/sjc/kustomization.yaml | 3 + .../resources/tenants/ns80/sjc/nad.yaml | 71 ++ deploy/containerlab/scripts/baseline-lib.sh | 131 +++ .../containerlab/scripts/baseline-observe.sh | 325 ++++++++ deploy/containerlab/scripts/baseline-twice.sh | 52 ++ .../containerlab/scripts/baseline-versions.sh | 87 ++ deploy/containerlab/scripts/verify-ns80.sh | 49 ++ 22 files changed, 1853 insertions(+), 1 deletion(-) create mode 100644 deploy/containerlab/baseline/pins.yaml create mode 100644 deploy/containerlab/baseline/reference-results.json create mode 100644 deploy/containerlab/docs/baseline.md create mode 100644 deploy/containerlab/resources/tenants/ns80/base/guests.yaml create mode 100644 deploy/containerlab/resources/tenants/ns80/base/kustomization.yaml create mode 100644 deploy/containerlab/resources/tenants/ns80/base/namespace.yaml create mode 100644 deploy/containerlab/resources/tenants/ns80/dfw/kustomization.yaml create mode 100644 deploy/containerlab/resources/tenants/ns80/dfw/nad.yaml create mode 100644 deploy/containerlab/resources/tenants/ns80/policy/base/bindings.yaml create mode 100644 deploy/containerlab/resources/tenants/ns80/policy/base/kustomization.yaml create mode 100644 deploy/containerlab/resources/tenants/ns80/policy/kustomization.yaml create mode 100644 deploy/containerlab/resources/tenants/ns80/sjc/kustomization.yaml create mode 100644 deploy/containerlab/resources/tenants/ns80/sjc/nad.yaml create mode 100755 deploy/containerlab/scripts/baseline-lib.sh create mode 100755 deploy/containerlab/scripts/baseline-observe.sh create mode 100755 deploy/containerlab/scripts/baseline-twice.sh create mode 100755 deploy/containerlab/scripts/baseline-versions.sh create mode 100755 deploy/containerlab/scripts/verify-ns80.sh diff --git a/deploy/containerlab/AGENTS.md b/deploy/containerlab/AGENTS.md index 187f4955b..e9abac5fd 100644 --- a/deploy/containerlab/AGENTS.md +++ b/deploy/containerlab/AGENTS.md @@ -45,3 +45,4 @@ Full documentation — topology, addressing, tasks — is in [README.md](README.md). Verification commands are in [docs/verification.md](docs/verification.md). Deploying the five test VPCs and verifying cross-site/same-node connectivity is documented in [docs/tenants.md](docs/tenants.md). +The dedicated-placement baseline (`ns80`, `task baseline:twice`) is documented in [docs/baseline.md](docs/baseline.md). diff --git a/deploy/containerlab/README.md b/deploy/containerlab/README.md index 24cca5eeb..edab89a9d 100644 --- a/deploy/containerlab/README.md +++ b/deploy/containerlab/README.md @@ -599,7 +599,9 @@ PublicNAT and PublicService paths and the built-in default. See See [docs/verification.md](docs/verification.md) for transit fabric, FRR, and galactic-router health checks, and [docs/tenants.md](docs/tenants.md) for deploying and verifying -the `ns10`/`ns20`/`ns30`/`ns40` test VPCs. +the `ns10`/`ns20`/`ns30`/`ns40` test VPCs. [docs/baseline.md](docs/baseline.md) +covers the dedicated-placement baseline: the `ns80` fixture, its scripted packet +observations and pinned versions (`task baseline:twice`). `task verify` (and its constituent `task verify:scenarios`) also pings every VPC's pods end-to-end via `task verify:ns10`/`ns20`/`ns30`/`ns40` — diff --git a/deploy/containerlab/Taskfile.yaml b/deploy/containerlab/Taskfile.yaml index d597c37e3..709d970ec 100644 --- a/deploy/containerlab/Taskfile.yaml +++ b/deploy/containerlab/Taskfile.yaml @@ -434,6 +434,7 @@ tasks: - task: "deploy:ns40" - task: "deploy:ns60" - task: "deploy:ns71" + - task: "deploy:ns80" "deploy:ns10": desc: Deploy ns10 test VPC (IPv6-only, fd20 ULA) @@ -477,6 +478,15 @@ tasks: - ./scripts/deploy-ns.sh ns71 sjc - ./scripts/publish-attachments.sh ns71 sjc + # The dedicated-placement baseline fixture (docs/baseline.md): two VPCs + # with identical guest addresses on dfw's and sjc's compute workers. + "deploy:ns80": + desc: Deploy ns80 test VPCs (overlapping guests on two workers; baseline fixture) + cmds: + - ./scripts/deploy-tenant-policy.sh ns80 dfw sjc + - ./scripts/deploy-ns.sh ns80 dfw sjc + - ./scripts/publish-attachments.sh ns80 dfw sjc + verify: desc: Run all verification checks cmds: @@ -1296,6 +1306,7 @@ tasks: - task: "verify:ns20" - task: "verify:ns30" - task: "verify:ns40" + - task: "verify:ns80" # Not idempotent while it runs: it edits ns60's dfw policy and unbinds # ns30/private-b's ingress for a moment, and restores both on exit. @@ -1364,6 +1375,30 @@ tasks: cmds: - ./scripts/verify-ns40.sh + "verify:ns80": + desc: Verify ns80 placement, overlapping addresses and cross-site ping in both VPCs + cmds: + - ./scripts/verify-ns80.sh + + # The dedicated-placement baseline (docs/baseline.md). Not part of verify: + # each observation takes a few minutes and runs the gateway ingress checks + # again under capture. Evidence lands in bin/baseline/ at the repository + # root. + "baseline:versions": + desc: Check baseline/pins.yaml against the host, the Dockerfiles and the running lab, and record versions + cmds: + - LAB={{.LAB}} ./scripts/baseline-versions.sh {{.CLI_ARGS | default "../../bin/baseline/versions.yaml"}} + + "baseline:observe": + desc: Observe today's remote paths on ns80 once, with captures (OUTDIR via task baseline:observe -- DIR) + cmds: + - LAB={{.LAB}} ./scripts/baseline-observe.sh {{.CLI_ARGS | default "../../bin/baseline/observe"}} + + "baseline:twice": + desc: Record versions, observe twice, and require identical results matching baseline/reference-results.json + cmds: + - LAB={{.LAB}} ./scripts/baseline-twice.sh {{.CLI_ARGS}} + clean: desc: Destroy the lab and delete artifacts cmds: diff --git a/deploy/containerlab/baseline/pins.yaml b/deploy/containerlab/baseline/pins.yaml new file mode 100644 index 000000000..c14493a95 --- /dev/null +++ b/deploy/containerlab/baseline/pins.yaml @@ -0,0 +1,14 @@ +# Versions the dedicated-placement baseline (docs/baseline.md) was recorded +# with. scripts/baseline-versions.sh fails when the host tools, the lab's +# Dockerfiles, the fixture or the running lab disagree with a value here. +# Change a pin together with baseline/reference-results.json, never alone. +# +# Host tools +containerlab: 0.75.0 +kind: v0.33.0 +task: 3.51.1 +# Images and components the lab builds or pulls +kindest_node: v1.37.0 +frr: 10.7.1 +remote_host_base: alpine:3.24 +guest_image: ghcr.io/datum-cloud/galactic-debug@sha256:b64e338f845d06f82f2575ae6bb6855da330af7551a8816b7ade2602a85722f2 diff --git a/deploy/containerlab/baseline/reference-results.json b/deploy/containerlab/baseline/reference-results.json new file mode 100644 index 000000000..e77e26727 --- /dev/null +++ b/deploy/containerlab/baseline/reference-results.json @@ -0,0 +1,758 @@ +[ + { + "scenario": "cross-node-a", + "check": "dfw->sjc echo replies", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-a", + "check": "dfw-compute-underlay dfw->sjc requests", + "expect": "-", + "got": "5", + "result": "observed" + }, + { + "scenario": "cross-node-a", + "check": "dfw-compute-underlay sjc->dfw requests", + "expect": "-", + "got": "5", + "result": "observed" + }, + { + "scenario": "cross-node-a", + "check": "dfw-edge-uplink dfw->sjc requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-a", + "check": "dfw-edge-uplink sjc->dfw requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-a", + "check": "dfw-guest dfw->sjc requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-a", + "check": "dfw-guest sjc->dfw requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-a", + "check": "dfw-veth-host dfw->sjc requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-a", + "check": "dfw-veth-host sjc->dfw requests (peer redirect)", + "expect": "-", + "got": "0", + "result": "observed" + }, + { + "scenario": "cross-node-a", + "check": "sjc->dfw echo replies", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-a", + "check": "sjc-compute-underlay dfw->sjc requests", + "expect": "-", + "got": "5", + "result": "observed" + }, + { + "scenario": "cross-node-a", + "check": "sjc-compute-underlay sjc->dfw requests", + "expect": "-", + "got": "5", + "result": "observed" + }, + { + "scenario": "cross-node-a", + "check": "sjc-edge-uplink dfw->sjc requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-a", + "check": "sjc-edge-uplink sjc->dfw requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-a", + "check": "sjc-guest dfw->sjc requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-a", + "check": "sjc-guest sjc->dfw requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-a", + "check": "sjc-veth-host dfw->sjc requests (peer redirect)", + "expect": "-", + "got": "0", + "result": "observed" + }, + { + "scenario": "cross-node-a", + "check": "sjc-veth-host sjc->dfw requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-b", + "check": "dfw->sjc echo replies", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-b", + "check": "dfw-compute-underlay dfw->sjc requests", + "expect": "-", + "got": "5", + "result": "observed" + }, + { + "scenario": "cross-node-b", + "check": "dfw-compute-underlay sjc->dfw requests", + "expect": "-", + "got": "5", + "result": "observed" + }, + { + "scenario": "cross-node-b", + "check": "dfw-edge-uplink dfw->sjc requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-b", + "check": "dfw-edge-uplink sjc->dfw requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-b", + "check": "dfw-guest dfw->sjc requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-b", + "check": "dfw-guest sjc->dfw requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-b", + "check": "dfw-veth-host dfw->sjc requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-b", + "check": "dfw-veth-host sjc->dfw requests (peer redirect)", + "expect": "-", + "got": "0", + "result": "observed" + }, + { + "scenario": "cross-node-b", + "check": "sjc->dfw echo replies", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-b", + "check": "sjc-compute-underlay dfw->sjc requests", + "expect": "-", + "got": "5", + "result": "observed" + }, + { + "scenario": "cross-node-b", + "check": "sjc-compute-underlay sjc->dfw requests", + "expect": "-", + "got": "5", + "result": "observed" + }, + { + "scenario": "cross-node-b", + "check": "sjc-edge-uplink dfw->sjc requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-b", + "check": "sjc-edge-uplink sjc->dfw requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-b", + "check": "sjc-guest dfw->sjc requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-b", + "check": "sjc-guest sjc->dfw requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-b", + "check": "sjc-veth-host dfw->sjc requests (peer redirect)", + "expect": "-", + "got": "0", + "result": "observed" + }, + { + "scenario": "cross-node-b", + "check": "sjc-veth-host sjc->dfw requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "fixture", + "check": "dfw vpc-a guest address", + "expect": "fd20:80:ff01::1:0:0", + "got": "fd20:80:ff01::1:0:0", + "result": "pass" + }, + { + "scenario": "fixture", + "check": "dfw vpc-a guest on compute worker", + "expect": "dfw-worker", + "got": "dfw-worker", + "result": "pass" + }, + { + "scenario": "fixture", + "check": "dfw vpc-b guest address equals vpc-a's", + "expect": "fd20:80:ff01::1:0:0", + "got": "fd20:80:ff01::1:0:0", + "result": "pass" + }, + { + "scenario": "fixture", + "check": "dfw vpc-b guest on compute worker", + "expect": "dfw-worker", + "got": "dfw-worker", + "result": "pass" + }, + { + "scenario": "fixture", + "check": "sjc vpc-a guest address", + "expect": "fd20:80:ff02::1:0:0", + "got": "fd20:80:ff02::1:0:0", + "result": "pass" + }, + { + "scenario": "fixture", + "check": "sjc vpc-a guest on compute worker", + "expect": "sjc-worker", + "got": "sjc-worker", + "result": "pass" + }, + { + "scenario": "fixture", + "check": "sjc vpc-b guest address equals vpc-a's", + "expect": "fd20:80:ff02::1:0:0", + "got": "fd20:80:ff02::1:0:0", + "result": "pass" + }, + { + "scenario": "fixture", + "check": "sjc vpc-b guest on compute worker", + "expect": "sjc-worker", + "got": "sjc-worker", + "result": "pass" + }, + { + "scenario": "gateway-ipv4", + "check": "backend-guest replies to client", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv4", + "check": "backend-guest requests from client", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv4", + "check": "backend-veth-host replies to client", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv4", + "check": "backend-veth-host requests from client (peer redirect)", + "expect": "-", + "got": "no", + "result": "observed" + }, + { + "scenario": "gateway-ipv4", + "check": "dfw-compute-underlay requests", + "expect": "-", + "got": "yes", + "result": "observed" + }, + { + "scenario": "gateway-ipv4", + "check": "public replies out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv4", + "check": "public requests in", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv4", + "check": "tcp flows answered by a backend", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv4", + "check": "udp flows answered by a backend", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv6", + "check": "backend-guest replies to client", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv6", + "check": "backend-guest requests from client", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv6", + "check": "backend-veth-host replies to client", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv6", + "check": "backend-veth-host requests from client (peer redirect)", + "expect": "-", + "got": "no", + "result": "observed" + }, + { + "scenario": "gateway-ipv6", + "check": "dfw-compute-underlay requests", + "expect": "-", + "got": "yes", + "result": "observed" + }, + { + "scenario": "gateway-ipv6", + "check": "peer replies back", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv6", + "check": "peer requests out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv6", + "check": "public replies out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv6", + "check": "public requests in", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv6", + "check": "tcp flows answered by a backend", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv6", + "check": "udp flows answered by a backend", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "isolation", + "check": "dfw-guest-a receives from vpc-a", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "isolation", + "check": "dfw-guest-a receives from vpc-b", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "isolation", + "check": "dfw-guest-b receives from vpc-a", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "isolation", + "check": "dfw-guest-b receives from vpc-b", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "isolation", + "check": "dfw-veth-host-a carries from vpc-a (peer redirect)", + "expect": "-", + "got": "0", + "result": "observed" + }, + { + "scenario": "isolation", + "check": "dfw-veth-host-a carries from vpc-b (peer redirect)", + "expect": "-", + "got": "0", + "result": "observed" + }, + { + "scenario": "isolation", + "check": "dfw-veth-host-b carries from vpc-a (peer redirect)", + "expect": "-", + "got": "0", + "result": "observed" + }, + { + "scenario": "isolation", + "check": "dfw-veth-host-b carries from vpc-b (peer redirect)", + "expect": "-", + "got": "0", + "result": "observed" + }, + { + "scenario": "nat64", + "check": "dfw-compute-underlay SYN out", + "expect": "-", + "got": "yes", + "result": "observed" + }, + { + "scenario": "nat64", + "check": "dfw-compute-underlay SYN-ACK back", + "expect": "-", + "got": "yes", + "result": "observed" + }, + { + "scenario": "nat64", + "check": "dfw-guest-a SYN out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "dfw-guest-a SYN-ACK back", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "dfw-guest-b SYN out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "dfw-guest-b SYN-ACK back", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "dfw-veth-host-a SYN out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "dfw-veth-host-a SYN-ACK back (peer redirect)", + "expect": "-", + "got": "no", + "result": "observed" + }, + { + "scenario": "nat64", + "check": "dfw-veth-host-b SYN out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "dfw-veth-host-b SYN-ACK back (peer redirect)", + "expect": "-", + "got": "no", + "result": "observed" + }, + { + "scenario": "nat64", + "check": "peer SYN-ACK sent", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "peer distinct translated tuples", + "expect": "2", + "got": "2", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "peer sees guest address", + "expect": "no", + "got": "no", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "peer sources are shard addresses", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "public SYN out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "public SYN-ACK back", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "vpc-a fetch status", + "expect": "200", + "got": "200", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "vpc-b fetch status", + "expect": "200", + "got": "200", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "dfw-compute-underlay SYN out", + "expect": "-", + "got": "yes", + "result": "observed" + }, + { + "scenario": "nat66", + "check": "dfw-compute-underlay SYN-ACK back", + "expect": "-", + "got": "yes", + "result": "observed" + }, + { + "scenario": "nat66", + "check": "dfw-guest-a SYN out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "dfw-guest-a SYN-ACK back", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "dfw-guest-b SYN out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "dfw-guest-b SYN-ACK back", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "dfw-veth-host-a SYN out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "dfw-veth-host-a SYN-ACK back (peer redirect)", + "expect": "-", + "got": "no", + "result": "observed" + }, + { + "scenario": "nat66", + "check": "dfw-veth-host-b SYN out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "dfw-veth-host-b SYN-ACK back (peer redirect)", + "expect": "-", + "got": "no", + "result": "observed" + }, + { + "scenario": "nat66", + "check": "peer SYN-ACK sent", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "peer distinct translated tuples", + "expect": "2", + "got": "2", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "peer sees guest address", + "expect": "no", + "got": "no", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "peer sources are shard addresses", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "public SYN out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "public SYN-ACK back", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "vpc-a fetch status", + "expect": "200", + "got": "200", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "vpc-b fetch status", + "expect": "200", + "got": "200", + "result": "pass" + } +] diff --git a/deploy/containerlab/docs/baseline.md b/deploy/containerlab/docs/baseline.md new file mode 100644 index 000000000..224d40562 --- /dev/null +++ b/deploy/containerlab/docs/baseline.md @@ -0,0 +1,149 @@ +# Dedicated-placement baseline + +This page describes a fixture and a set of scripted packet observations of +today's remote paths. The fixture is two VPCs with identical guest addresses +on two physical workers. Services run on dedicated edge nodes, and workloads +run on compute nodes. Later work on the Galactic Worker reuses this fixture +and compares its results against the reference recorded here. + +The fixture has no TAP-backed guest. The tenant TAP capture point belongs to +[#1444](https://github.com/datum-cloud/galactic/issues/1444). + +## Fixture + +`ns80` (`resources/tenants/ns80/`) holds two VPCs with the same guest address +at each site: + +| VPC | Attachment | dfw-worker guest | sjc-worker guest | Kind | +| ---- | ---------- | --------------------- | --------------------- | ---------------------- | +| `80` | `vpc-a` | `fd20:80:ff01::1:0:0` | `fd20:80:ff02::1:0:0` | veth (`galactic-veth`) | +| `81` | `vpc-b` | `fd20:80:ff01::1:0:0` | `fd20:80:ff02::1:0:0` | veth (`galactic-veth`) | + +- **Two physical workers.** `dfw-worker` and `sjc-worker` are each their site's + only compute node. They have distinct SRv6 locators and reach each other over + the routed underlay through their sites' edge nodes and the transit routers. + Gateway and NAT run only on the edge nodes (`dfw-worker2`, `dfw-worker3`, + `sjc-worker2`), so this is dedicated placement. +- **Identical addressing.** The guests take fixed addresses through IPAM's + `addresses` path, so the overlap does not depend on allocation order. Each + guest's gateway lies outside its `/96` and is reached on-link, as on every + `addresses` attachment. +- **Public client and egress peer.** `remote-host` behind `tr4` serves HTTP + on `[2001:db8:1:40::2]:80` and `11.1.40.2:80`. It is the client for gateway + ingress and the peer for NAT egress. +- **Policy.** Every `ns80` attachment is bound to the `private-nat` set + (`resources/tenants/policies/README.md`). + +`task deploy` deploys `ns80` as part of `deploy:scenarios`. `task deploy:ns80` +redeploys it alone, and `task verify:ns80` (part of `task verify`) checks +placement, the overlapping addresses and cross-site reachability. + +## Running the observations + +```bash +cd deploy/containerlab +task deploy # from a destroyed lab: build images, deploy everything +task baseline:twice # pins, two observation runs, compare with the reference +task verify # the lab's full verification still passes +``` + +`task baseline:twice` writes to `bin/baseline//` at the +repository root, or to the directory given after `--`. It fails if a pinned +version disagrees, if any check fails in either run, if the two runs' +`results.json` files differ, or if run 1 differs from +`baseline/reference-results.json`. `task baseline:observe -- DIR` runs one +observation and `task baseline:versions -- FILE` records versions alone. + +Each run directory holds: + +| File | Contents | +| ------------------ | -------------------------------------------------------------------------------------- | +| `results.json` | One row per check: scenario, check, expected, observed, result. Stable values only. | +| `volatile.tsv` | Per-run values: source ports, the isolation nonce, translated tuples, shard addresses. | +| `pcap//` | One pcap per capture point and interface, with tcpdump's log beside it. | +| `logs/` | Probe output and the gateway checks' full logs. | + +`results.json` never holds a port, nonce, timestamp or MAC address. Those +values change on every run, and only stable values let two runs compare equal. + +## Scenarios + +Each scenario runs in its own capture window, in both directions. + +| Scenario | Traffic | Expected | +| -------------- | ----------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------- | +| `cross-node-a` | 5 ICMPv6 echoes dfw → sjc, then 5 sjc → dfw, in VPC 80 | 5 replies each way, every request seen at every expected point | +| `cross-node-b` | The same in VPC 81 | The same | +| `nat66` | Both dfw guests fetch `http://[2001:db8:1:40::2]/` at once from one identical source port | Both get 200, the peer sees 2 distinct translated tuples from shard addresses only | +| `nat64` | The same to `http://[2001:db8:64::b01:2802]/`, under the shards' NAT64 prefix | The same over IPv4 | +| `gateway-ipv6` | `verify-gateway-ingress.sh --node dfw-worker2`: TCP and UDP to `2001:db8:6060:1::1` | Answered flows in both protocols; requests and DSR replies seen at the public side and the backends | +| `gateway-ipv4` | `verify-gateway-ingress-ipv4.sh --node dfw-worker2`: TCP and UDP to `198.51.100.1` | The same; backends see the client as `64:ff9b::b01:2802` | +| `isolation` | 5 UDP datagrams from each VPC's sjc guest to `[fd20:80:ff01::1:0:0]` | Each VPC's dfw guest and host veth see its own 5 and none of the other VPC's | + +The gateway scenarios reuse the lab's ingress checks against `ns60`'s +backends on `dfw-worker`, and pin the VIP to `dfw-worker2` on `tr1` for the +duration of the check. + +The IPv4 check captures on `remote-host` itself and stops every tcpdump +there, so `gateway-ipv4` has no `remote-host` capture point. The check's own +client-side assertions stand in for it. + +### Intermittent TCP loss in the gateway checks + +Both lab ingress checks lose a TCP flow now and then, with or without these +captures: in three back-to-back runs of `verify-gateway-ingress-ipv4.sh --node +dfw-worker2` with no capture running, one answered 6 of 8 TCP flows. The +captures show that the lost flow is delivered. The request reaches the +backend over the gateway, and the backend acknowledges it. Then the `ns60` +responder (`socat ... fork SYSTEM:`) closes without writing a reply, 1 to +2 ms after the client's FIN, and that close returns to the client over the +DSR path. UDP flows are never lost. + +So each gateway scenario's exact flow counts and the check's exit status go +to `volatile.tsv`, and `results.json` records only whether each protocol had +answered flows. + +## Capture points + +| Point | Where | +| --------------- | ----------------------------------------------------------------------------------------------------------------------- | +| Public RX/TX | `tr1`'s members facing both dfw edge nodes (`eth1`, `eth6`, `eth5`, `eth7`), and `remote-host` `eth1` | +| Veth peers | Each guest's `eth0` (an `ns80` guest, or an `ns60` backend for gateway ingress) and its host veth `GH` | +| Remote underlay | `tr2`'s members facing `sjc-worker2` (`eth1`, `eth5`), `tr1`'s dfw edge members, and both compute workers' bond members | + +A point that names several interfaces sums them. The bond member a flow +uses, and the dfw edge node it crosses, are chosen by hashes that can differ +between runs. The link the flow crosses does not change. + +Some points cannot see one direction, because of how today's datapath +delivers packets. Those checks are recorded as observations without an +expectation, and are still compared between runs: + +- **Edge nodes.** The edge nodes forward NAT and gateway traffic in XDP, + which tcpdump on those nodes never sees (#565). Public-side captures are + therefore taken on the transit router's end of each edge uplink. +- **Host veths, inbound.** A host veth carries its guest's outbound packets. + Inbound packets reach the guest through `bpf_redirect_peer` in + `usid_ingress` (`internal/plumbing/ebpf/prog/usid.c`), straight into the + guest's side of the pair, so tcpdump on the host side never sees them. The + guest's own `eth0` is the evidence for inbound delivery, including for + the isolation control. +- **Compute bond members.** These captures are recorded without an + expectation. + +## Pinned versions + +`baseline/pins.yaml` pins the host tools (containerlab, kind, task), the +Kind node version, FRR, the `remote-host` base image and the guest image +digest. `scripts/baseline-versions.sh` checks each pin against its source: +the host tool, the lab's Dockerfiles, the `ns80` manifests and the running +lab. It then records the host and node kernels, Docker, the Galactic +revision and every lab image ID in `versions.yaml`. Change a pin only +together with the reference results. + +## Updating the reference + +After a deliberate change to the fixture or to an expectation, rerun with +`BASELINE_WRITE_REFERENCE=1 task baseline:twice`. The reference is replaced +only when both runs pass and agree. Commit the new +`baseline/reference-results.json` with the change that caused it. diff --git a/deploy/containerlab/docs/tenants.md b/deploy/containerlab/docs/tenants.md index b4792ab25..87d3ee4b5 100644 --- a/deploy/containerlab/docs/tenants.md +++ b/deploy/containerlab/docs/tenants.md @@ -24,6 +24,9 @@ or `policy//` for a site with attachments the others lack, see attachment's status after its pods start (`scripts/publish-attachments.sh`). A pod's interface forwards nothing until then. +`ns80`, two VPCs whose guests share addresses across two workers, is the +dedicated-placement baseline fixture and is described in [baseline.md](baseline.md). + They differ only in scope and addressing: | VPC | Sites | Address families | VRF interface | Notes | diff --git a/deploy/containerlab/resources/tenants/ns80/base/guests.yaml b/deploy/containerlab/resources/tenants/ns80/base/guests.yaml new file mode 100644 index 000000000..cef836845 --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80/base/guests.yaml @@ -0,0 +1,56 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: guest-a + labels: + app: vpc-a +spec: + replicas: 1 + selector: + matchLabels: + app: vpc-a + template: + metadata: + labels: + app: vpc-a + annotations: + v1.multus-cni.io/default-network: ns80/vpc-a + spec: + nodeSelector: + galactic.datumapis.com/node: compute + terminationGracePeriodSeconds: 1 + containers: + - name: guest + # Pinned by digest: baseline/pins.yaml records the same value. + image: ghcr.io/datum-cloud/galactic-debug@sha256:b64e338f845d06f82f2575ae6bb6855da330af7551a8816b7ade2602a85722f2 + imagePullPolicy: IfNotPresent + command: ["sleep", "infinity"] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: guest-b + labels: + app: vpc-b +spec: + replicas: 1 + selector: + matchLabels: + app: vpc-b + template: + metadata: + labels: + app: vpc-b + annotations: + v1.multus-cni.io/default-network: ns80/vpc-b + spec: + nodeSelector: + galactic.datumapis.com/node: compute + terminationGracePeriodSeconds: 1 + containers: + - name: guest + # Pinned by digest: baseline/pins.yaml records the same value. + image: ghcr.io/datum-cloud/galactic-debug@sha256:b64e338f845d06f82f2575ae6bb6855da330af7551a8816b7ade2602a85722f2 + imagePullPolicy: IfNotPresent + command: ["sleep", "infinity"] diff --git a/deploy/containerlab/resources/tenants/ns80/base/kustomization.yaml b/deploy/containerlab/resources/tenants/ns80/base/kustomization.yaml new file mode 100644 index 000000000..ce97d47ab --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80/base/kustomization.yaml @@ -0,0 +1,8 @@ +# ns80: two VPCs, 80 (vpc-a) and 81 (vpc-b), whose guests use identical +# addresses at each site. One guest per VPC runs on each site's compute worker, +# so the pair spans two physical workers. This is the dedicated-placement +# baseline fixture (docs/baseline.md). +namespace: ns80 +resources: + - namespace.yaml + - guests.yaml diff --git a/deploy/containerlab/resources/tenants/ns80/base/namespace.yaml b/deploy/containerlab/resources/tenants/ns80/base/namespace.yaml new file mode 100644 index 000000000..cdbb66953 --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80/base/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ns80 diff --git a/deploy/containerlab/resources/tenants/ns80/dfw/kustomization.yaml b/deploy/containerlab/resources/tenants/ns80/dfw/kustomization.yaml new file mode 100644 index 000000000..ee81d25b5 --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80/dfw/kustomization.yaml @@ -0,0 +1,3 @@ +resources: + - ../base + - nad.yaml diff --git a/deploy/containerlab/resources/tenants/ns80/dfw/nad.yaml b/deploy/containerlab/resources/tenants/ns80/dfw/nad.yaml new file mode 100644 index 000000000..f56bd2932 --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80/dfw/nad.yaml @@ -0,0 +1,71 @@ +# dfw: both VPCs' guests get the same address, fd20:80:ff01::1:0:0. +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: vpc-a + namespace: ns80 +spec: + config: |- + { + "cniVersion": "1.0.0", + "name": "vpc-a", + "plugins": [ + { + "type": "galactic-veth", + "vpc": "80", + "vpcattachment": "80", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "addresses": [ + { + "address": "fd20:80:ff01:0:0:1::/96", + "gateway": "fd20:80:ff01::1" + } + ] + } + }, + { + "type": "galactic-bgp", + "vpc": "80", + "vpcattachment": "80", + "namespace": "galactic-system" + } + ] + } +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: vpc-b + namespace: ns80 +spec: + config: |- + { + "cniVersion": "1.0.0", + "name": "vpc-b", + "plugins": [ + { + "type": "galactic-veth", + "vpc": "81", + "vpcattachment": "81", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "addresses": [ + { + "address": "fd20:80:ff01:0:0:1::/96", + "gateway": "fd20:80:ff01::1" + } + ] + } + }, + { + "type": "galactic-bgp", + "vpc": "81", + "vpcattachment": "81", + "namespace": "galactic-system" + } + ] + } diff --git a/deploy/containerlab/resources/tenants/ns80/policy/base/bindings.yaml b/deploy/containerlab/resources/tenants/ns80/policy/base/bindings.yaml new file mode 100644 index 000000000..c51bb6aba --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80/policy/base/bindings.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: galactic.datumapis.com/v1alpha1 +kind: TrafficPolicyBinding +metadata: + name: vpc-a +spec: + ingressPolicyRef: + name: lab-ingress + egressPolicyRef: + name: lab-egress +--- +apiVersion: galactic.datumapis.com/v1alpha1 +kind: TrafficPolicyBinding +metadata: + name: vpc-b +spec: + ingressPolicyRef: + name: lab-ingress + egressPolicyRef: + name: lab-egress diff --git a/deploy/containerlab/resources/tenants/ns80/policy/base/kustomization.yaml b/deploy/containerlab/resources/tenants/ns80/policy/base/kustomization.yaml new file mode 100644 index 000000000..6db9e7f67 --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80/policy/base/kustomization.yaml @@ -0,0 +1,7 @@ +# ns80's traffic policy: the private-nat set (../../../policies/README.md), +# bound to both VPCs' attachments in both directions. A binding is named after +# its VPCAttachment, which scripts/publish-attachments.sh creates. +namespace: ns80 +resources: + - ../../../policies/private-nat + - bindings.yaml diff --git a/deploy/containerlab/resources/tenants/ns80/policy/kustomization.yaml b/deploy/containerlab/resources/tenants/ns80/policy/kustomization.yaml new file mode 100644 index 000000000..ad82ec479 --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80/policy/kustomization.yaml @@ -0,0 +1,3 @@ +# ns80's traffic policy at every site: both VPCs' attachments (base/). +resources: + - base diff --git a/deploy/containerlab/resources/tenants/ns80/sjc/kustomization.yaml b/deploy/containerlab/resources/tenants/ns80/sjc/kustomization.yaml new file mode 100644 index 000000000..ee81d25b5 --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80/sjc/kustomization.yaml @@ -0,0 +1,3 @@ +resources: + - ../base + - nad.yaml diff --git a/deploy/containerlab/resources/tenants/ns80/sjc/nad.yaml b/deploy/containerlab/resources/tenants/ns80/sjc/nad.yaml new file mode 100644 index 000000000..350526a80 --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80/sjc/nad.yaml @@ -0,0 +1,71 @@ +# sjc: both VPCs' guests get the same address, fd20:80:ff02::1:0:0. +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: vpc-a + namespace: ns80 +spec: + config: |- + { + "cniVersion": "1.0.0", + "name": "vpc-a", + "plugins": [ + { + "type": "galactic-veth", + "vpc": "80", + "vpcattachment": "80", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "addresses": [ + { + "address": "fd20:80:ff02:0:0:1::/96", + "gateway": "fd20:80:ff02::1" + } + ] + } + }, + { + "type": "galactic-bgp", + "vpc": "80", + "vpcattachment": "80", + "namespace": "galactic-system" + } + ] + } +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: vpc-b + namespace: ns80 +spec: + config: |- + { + "cniVersion": "1.0.0", + "name": "vpc-b", + "plugins": [ + { + "type": "galactic-veth", + "vpc": "81", + "vpcattachment": "81", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "addresses": [ + { + "address": "fd20:80:ff02:0:0:1::/96", + "gateway": "fd20:80:ff02::1" + } + ] + } + }, + { + "type": "galactic-bgp", + "vpc": "81", + "vpcattachment": "81", + "namespace": "galactic-system" + } + ] + } diff --git a/deploy/containerlab/scripts/baseline-lib.sh b/deploy/containerlab/scripts/baseline-lib.sh new file mode 100755 index 000000000..8b6541427 --- /dev/null +++ b/deploy/containerlab/scripts/baseline-lib.sh @@ -0,0 +1,131 @@ +#!/bin/bash +# baseline-lib.sh: capture and result helpers for baseline-observe.sh. +# Sourced, not executed. Needs lib.sh and LAB, OUT already set. +# +# A capture point is one or more interfaces read together, because a flow's +# bond member or ECMP next hop can change between runs while the link it +# crosses does not. Each interface streams a pcap back over `docker exec` or +# `kubectl exec`, so nothing is left on a node. + +# Capture targets: +# node:CONTAINER a lab container (Kind node, transit router, remote-host) +# pod:SITE/NAMESPACE/POD a pod's own network namespace +# netns:SITE/NAMESPACE/POD/NETNS a named network namespace inside POD's mount namespace +target_exec() { + local target="$1" + shift + local kind=${target%%:*} rest=${target#*:} + case "${kind}" in + node) docker exec -i "${rest}" "$@" ;; + pod) + IFS=/ read -r site ns pod <<<"${rest}" + docker exec -i "$(control_plane "${site}")" kubectl exec -i -n "${ns}" "${pod}" -- "$@" + ;; + netns) + IFS=/ read -r site ns pod netns <<<"${rest}" + docker exec -i "$(control_plane "${site}")" kubectl exec -i -n "${ns}" "${pod}" -- ip netns exec "${netns}" "$@" + ;; + *) + echo "target_exec: unknown target ${target}" >&2 + return 2 + ;; + esac +} + +# The capture filter on every interface. BGP, BFD and neighbour discovery +# run on every link all the time and say nothing about the paths under test. +CAPTURE_FILTER='not (tcp port 179 or udp port 3784 or udp port 4784 or (icmp6 and ip6[40] >= 133 and ip6[40] <= 137))' + +declare -ga CAP_PIDS=() CAP_STOPS=() + +# cap_start SCENARIO POINT TARGET IFACE [IFACE...] +# Starts one tcpdump per interface. Their pcaps land in +# ${OUT}/pcap/SCENARIO/POINT.IFACE.pcap. +cap_start() { + local scenario="$1" point="$2" target="$3" + shift 3 + local dir="${OUT}/pcap/${scenario}" iface file + mkdir -p "${dir}" + for iface in "$@"; do + file="${dir}/${point}.${iface}.pcap" + target_exec "${target}" timeout 300 tcpdump -U -n -i "${iface}" -w - "${CAPTURE_FILTER}" \ + >"${file}" 2>"${file}.log" & + CAP_PIDS+=($!) + CAP_STOPS+=("${target}|${iface}|${file}") + done +} + +# cap_ready waits until every running capture reports it is listening, so +# no probe is sent before its first capture point is armed. +cap_ready() { + local entry file i + for entry in "${CAP_STOPS[@]}"; do + file=${entry##*|} + for i in $(seq 1 50); do + grep -q 'listening on' "${file}.log" 2>/dev/null && break + if [ "${i}" -eq 50 ]; then + echo "capture never started: ${file}" >&2 + cat "${file}.log" >&2 || true + return 1 + fi + sleep 0.2 + done + done +} + +# cap_stop_all stops every capture started since the last call, waits for +# each to flush and exit, and clears the list. +cap_stop_all() { + local entry target iface pid + sleep 1 + for entry in "${CAP_STOPS[@]}"; do + IFS='|' read -r target iface _ <<<"${entry}" + target_exec "${target}" pkill -INT -f "tcpdump -U -n -i ${iface} -w -" >/dev/null 2>&1 || true + done + for pid in "${CAP_PIDS[@]}"; do + wait "${pid}" 2>/dev/null || true + done + CAP_PIDS=() + CAP_STOPS=() +} + +# hits SCENARIO POINT REGEX counts decoded packets at POINT whose tcpdump +# line matches REGEX (grep -E), summed across POINT's interfaces. Matching +# the decoded text sees inside SRv6 encapsulation, which tcpdump prints as +# "IP6 outer > outer: IP6 inner > inner: ...". +hits() { + local scenario="$1" point="$2" regex="$3" f n total=0 + for f in "${OUT}/pcap/${scenario}/${point}".*.pcap; do + [ -s "${f}" ] || continue + n=$(tcpdump -nn -r "${f}" 2>/dev/null | grep -cE -- "${regex}" || true) + total=$((total + n)) + done + echo "${total}" +} + +# record SCENARIO CHECK EXPECT GOT appends one stable result row. EXPECT is +# a number, "yes"/"no", or "-" for a recorded observation with no +# expectation. Only stable values may be recorded here; anything that +# changes run to run (ports, nonces, timestamps, MACs) goes to volatile. +# FAILED is read by baseline-observe.sh, which sources this file. +# shellcheck disable=SC2034 +FAILED=0 +# shellcheck disable=SC2034 +record() { + local scenario="$1" check="$2" expect="$3" got="$4" ok=pass + if [ "${expect}" != "-" ] && [ "${expect}" != "${got}" ]; then + ok=fail + FAILED=1 + fi + [ "${expect}" = "-" ] && ok=observed + printf '%s\t%s\t%s\t%s\t%s\n' "${scenario}" "${check}" "${expect}" "${got}" "${ok}" >>"${OUT}/results.tsv" + printf ' %-8s %-14s %-58s want %-4s got %s\n' "${ok}" "${scenario}" "${check}" "${expect}" "${got}" +} + +# seen N prints "yes" when N > 0. +seen() { [ "$1" -gt 0 ] && echo yes || echo no; } + +# volatile KEY VALUE records a run-specific detail outside results.json. +volatile() { + printf '%s\t%s\n' "$1" "$2" >>"${OUT}/volatile.tsv" +} diff --git a/deploy/containerlab/scripts/baseline-observe.sh b/deploy/containerlab/scripts/baseline-observe.sh new file mode 100755 index 000000000..d3fe92518 --- /dev/null +++ b/deploy/containerlab/scripts/baseline-observe.sh @@ -0,0 +1,325 @@ +#!/bin/bash +# baseline-observe.sh: observe today's remote paths on the dedicated-placement +# fixture (resources/tenants/ns80, docs/baseline.md) in both directions, with +# packet captures at every capture point, and write the results. +# +# Usage: baseline-observe.sh OUTDIR +# +# Scenarios, each in its own capture window: +# +# cross-node-a/-b ICMPv6 between the dfw and sjc guests of vpc-a, then vpc-b +# nat66, nat64 both dfw guests fetch from remote-host at once, with one +# identical source port, through dfw's egress shards +# gateway-ipv6/-ipv4 +# remote-host to dfw's VIPs through dfw-worker2, answered +# by ns60's backends with direct server return +# isolation UDP to the overlapping address from each VPC's sjc guest: +# only the same VPC's dfw guest may receive it +# +# OUTDIR gets results.json (stable fields only, compared between runs), +# volatile.tsv (ports, nonces and other per-run values), pcap// +# and logs/. Exits non-zero when any expectation fails. +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +source "${SCRIPT_DIR}/lib.sh" + +LAB=${LAB:-gvpc} +OUT=${1:?usage: baseline-observe.sh OUTDIR} +mkdir -p "${OUT}/logs" +OUT=$(cd "${OUT}" && pwd) +: >"${OUT}/results.tsv" +: >"${OUT}/volatile.tsv" +source "${SCRIPT_DIR}/baseline-lib.sh" +trap cap_stop_all EXIT + +NS=ns80 +DFW_CP=$(control_plane dfw) +SJC_CP=$(control_plane sjc) +DFW_ADDR=fd20:80:ff01::1:0:0 +SJC_ADDR=fd20:80:ff02::1:0:0 +CLIENT6=2001:db8:1:40::2 +CLIENT4=11.1.40.2 +# remote-host's IPv4 address as each side writes it in IPv6: under the lab +# shards' NAT64 prefix for egress (resources/galactic-nat/), and under +# 64:ff9b::/96 as a gateway backend sees an IPv4 client. +EGRESS_NAT64=2001:db8:64::b01:2802 +BACKEND_SEES4=64:ff9b::b01:2802 +PINGS=5 +PROBES=5 + +# Capture points shared by several scenarios. Each sums every member of the +# link it names (see baseline-lib.sh). +TR1="node:clab-${LAB}-tr1" +TR2="node:clab-${LAB}-tr2" +REMOTE="node:clab-${LAB}-remote-host" +DFW_COMPUTE_IFS=(eth1 eth3 eth2 eth4) # bonds to dfw-worker2 and dfw-worker3 +SJC_COMPUTE_IFS=(eth1 eth2) # bond to sjc-worker2 +DFW_EDGE_UPLINKS=(eth1 eth6 eth5 eth7) # tr1's members facing dfw-worker2 and dfw-worker3 +SJC_EDGE_UPLINKS=(eth1 eth5) # tr2's members facing sjc-worker2 + +pod_of() { pod_name "$1" "${NS}" "app=$2"; } +host_if() { printf 'G%09s%03sH' "$1" "$2" | tr ' ' '0'; } + +echo "=== fixture ===" +declare -A POD +for site in dfw sjc; do + cp=$(control_plane "${site}") + for app in vpc-a vpc-b; do + pod=$(pod_of "${cp}" "${app}") + if [ -z "${pod}" ]; then + echo "no ${NS} ${app} pod in ${site}: run task deploy:ns80" >&2 + exit 1 + fi + POD[${site}-${app}]=${pod} + record fixture "${site} ${app} guest on compute worker" "${site}-worker" "$(pod_scheduling_node "${cp}" "${NS}" "${pod}")" + done + a=$(pod_ip6 "${cp}" "${NS}" "${POD[${site}-vpc-a]}") + b=$(pod_ip6 "${cp}" "${NS}" "${POD[${site}-vpc-b]}") + want=${DFW_ADDR} + [ "${site}" = sjc ] && want=${SJC_ADDR} + record fixture "${site} vpc-a guest address" "${want}" "${a}" + record fixture "${site} vpc-b guest address equals vpc-a's" "${a}" "${b}" +done + +# --------------------------------------------------------------------------- +# Cross-node workload delivery, per VPC: dfw guest <-> sjc guest. +cross_node() { + local vpc="$1" id="$2" sc="cross-node-${1#vpc-}" + local dfw_pod=${POD[dfw-${vpc}]} sjc_pod=${POD[sjc-${vpc}]} + echo "=== ${sc} ===" + # One unrecorded echo each way first, so neighbour resolution and route + # warm-up never land inside the counted window. + docker exec "${DFW_CP}" kubectl exec -n "${NS}" "${dfw_pod}" -- ping -6 -c 1 -W 2 "${SJC_ADDR}" >/dev/null 2>&1 || true + docker exec "${SJC_CP}" kubectl exec -n "${NS}" "${sjc_pod}" -- ping -6 -c 1 -W 2 "${DFW_ADDR}" >/dev/null 2>&1 || true + + cap_start "${sc}" dfw-guest "pod:dfw/${NS}/${dfw_pod}" eth0 + cap_start "${sc}" dfw-veth-host "node:dfw-worker" "$(host_if "${id}" "${id}")" + cap_start "${sc}" dfw-compute-underlay "node:dfw-worker" "${DFW_COMPUTE_IFS[@]}" + cap_start "${sc}" dfw-edge-uplink "${TR1}" "${DFW_EDGE_UPLINKS[@]}" + cap_start "${sc}" sjc-edge-uplink "${TR2}" "${SJC_EDGE_UPLINKS[@]}" + cap_start "${sc}" sjc-compute-underlay "node:sjc-worker" "${SJC_COMPUTE_IFS[@]}" + cap_start "${sc}" sjc-veth-host "node:sjc-worker" "$(host_if "${id}" "${id}")" + cap_start "${sc}" sjc-guest "pod:sjc/${NS}/${sjc_pod}" eth0 + cap_ready + + local out_ab out_ba + out_ab=$(docker exec "${DFW_CP}" kubectl exec -n "${NS}" "${dfw_pod}" -- \ + ping -6 -c "${PINGS}" -i 0.2 -W 2 "${SJC_ADDR}" 2>&1 || true) + out_ba=$(docker exec "${SJC_CP}" kubectl exec -n "${NS}" "${sjc_pod}" -- \ + ping -6 -c "${PINGS}" -i 0.2 -W 2 "${DFW_ADDR}" 2>&1 || true) + cap_stop_all + printf '%s\n%s\n' "${out_ab}" "${out_ba}" >"${OUT}/logs/${sc}.ping.log" + + record "${sc}" "dfw->sjc echo replies" "${PINGS}" "$(sed -nE 's/.* ([0-9]+) (packets )?received.*/\1/p' <<<"${out_ab}")" + record "${sc}" "sjc->dfw echo replies" "${PINGS}" "$(sed -nE 's/.* ([0-9]+) (packets )?received.*/\1/p' <<<"${out_ba}")" + + local fwd="${DFW_ADDR} > ${SJC_ADDR}: ICMP6, echo request" + local rev="${SJC_ADDR} > ${DFW_ADDR}: ICMP6, echo request" + local point + # Each direction's requests, at every point on the path. Points marked + # "-" are recorded but carry no expectation (docs/baseline.md). + for point in dfw-guest sjc-guest dfw-edge-uplink sjc-edge-uplink; do + record "${sc}" "${point} dfw->sjc requests" "${PINGS}" "$(hits "${sc}" "${point}" "${fwd}")" + record "${sc}" "${point} sjc->dfw requests" "${PINGS}" "$(hits "${sc}" "${point}" "${rev}")" + done + # A host veth carries its guest's outbound packets. Inbound ones arrive by + # bpf_redirect_peer straight into the guest's side of the pair + # (internal/plumbing/ebpf/prog/usid.c), which tcpdump on the host side + # never sees, so those are recorded without an expectation. + record "${sc}" "dfw-veth-host dfw->sjc requests" "${PINGS}" "$(hits "${sc}" dfw-veth-host "${fwd}")" + record "${sc}" "sjc-veth-host sjc->dfw requests" "${PINGS}" "$(hits "${sc}" sjc-veth-host "${rev}")" + record "${sc}" "dfw-veth-host sjc->dfw requests (peer redirect)" - "$(hits "${sc}" dfw-veth-host "${rev}")" + record "${sc}" "sjc-veth-host dfw->sjc requests (peer redirect)" - "$(hits "${sc}" sjc-veth-host "${fwd}")" + for point in dfw-compute-underlay sjc-compute-underlay; do + record "${sc}" "${point} dfw->sjc requests" - "$(hits "${sc}" "${point}" "${fwd}")" + record "${sc}" "${point} sjc->dfw requests" - "$(hits "${sc}" "${point}" "${rev}")" + done +} +cross_node vpc-a 80 +cross_node vpc-b 81 + +# --------------------------------------------------------------------------- +# NAT66 and NAT64 egress: both dfw guests, one identical source port and +# destination, at once. The shard must give them distinct translated tuples +# and return each reply to the guest that sent it. +SHARDS6=$(docker exec "${DFW_CP}" kubectl -n galactic-system get egressshard -o jsonpath='{.items[*].status.shardAddressIPv6}') +SHARDS4=$(docker exec "${DFW_CP}" kubectl -n galactic-system get egressshard -o jsonpath='{.items[*].status.shardAddressIPv4}') +volatile shards "${SHARDS6} ${SHARDS4}" + +nat_scenario() { + local sc="$1" url_host="$2" seen_dst="$3" shards="$4" + local port=$((40000 + RANDOM % 20000)) + volatile "${sc}.guest_source_port" "${port}" + echo "=== ${sc} ===" + local dfw_a=${POD[dfw-vpc-a]} dfw_b=${POD[dfw-vpc-b]} + cap_start "${sc}" dfw-guest-a "pod:dfw/${NS}/${dfw_a}" eth0 + cap_start "${sc}" dfw-guest-b "pod:dfw/${NS}/${dfw_b}" eth0 + cap_start "${sc}" dfw-veth-host-a "node:dfw-worker" "$(host_if 80 80)" + cap_start "${sc}" dfw-veth-host-b "node:dfw-worker" "$(host_if 81 81)" + cap_start "${sc}" dfw-compute-underlay "node:dfw-worker" "${DFW_COMPUTE_IFS[@]}" + cap_start "${sc}" public "${TR1}" "${DFW_EDGE_UPLINKS[@]}" + cap_start "${sc}" peer "${REMOTE}" eth1 + cap_ready + + local fetch="curl -sS -o /dev/null -w %{http_code} --max-time 5 --local-port ${port} http://[${url_host}]/" + docker exec "${DFW_CP}" kubectl exec -n "${NS}" "${dfw_a}" -- sh -c "${fetch}" >"${OUT}/logs/${sc}.a.code" 2>&1 & + local pa=$! + docker exec "${DFW_CP}" kubectl exec -n "${NS}" "${dfw_b}" -- sh -c "${fetch}" >"${OUT}/logs/${sc}.b.code" 2>&1 & + local pb=$! + wait "${pa}" || true + wait "${pb}" || true + cap_stop_all + + record "${sc}" "vpc-a fetch status" 200 "$(tail -c 3 "${OUT}/logs/${sc}.a.code")" + record "${sc}" "vpc-b fetch status" 200 "$(tail -c 3 "${OUT}/logs/${sc}.b.code")" + + local g=${DFW_ADDR} + local out_re="${g}\.${port} > ${url_host}\.80: Flags \[S\]" + local back_re="${url_host}\.80 > ${g}\.${port}: Flags \[S\.\]" + local p + for p in a b; do + record "${sc}" "dfw-guest-${p} SYN out" yes "$(seen "$(hits "${sc}" "dfw-guest-${p}" "${out_re}")")" + record "${sc}" "dfw-guest-${p} SYN-ACK back" yes "$(seen "$(hits "${sc}" "dfw-guest-${p}" "${back_re}")")" + record "${sc}" "dfw-veth-host-${p} SYN out" yes "$(seen "$(hits "${sc}" "dfw-veth-host-${p}" "${out_re}")")" + record "${sc}" "dfw-veth-host-${p} SYN-ACK back (peer redirect)" - "$(seen "$(hits "${sc}" "dfw-veth-host-${p}" "${back_re}")")" + done + record "${sc}" "dfw-compute-underlay SYN out" - "$(seen "$(hits "${sc}" dfw-compute-underlay "${out_re}")")" + record "${sc}" "dfw-compute-underlay SYN-ACK back" - "$(seen "$(hits "${sc}" dfw-compute-underlay "${back_re}")")" + + # On the public side the guest address is gone: each SYN to the peer + # carries a shard's masquerade address and a port the shard chose. + local syn_re="> ${seen_dst}\.80: Flags \[S\]," synack_re="${seen_dst}\.80 > .*: Flags \[S\.\]," + record "${sc}" "public SYN out" yes "$(seen "$(hits "${sc}" public "${syn_re}")")" + record "${sc}" "public SYN-ACK back" yes "$(seen "$(hits "${sc}" public "${synack_re}")")" + local tuples + tuples=$(for f in "${OUT}/pcap/${sc}"/peer.*.pcap; do tcpdump -nn -r "${f}" 2>/dev/null; done | + { grep -E "${syn_re}" || true; } | sed -nE 's/^.* IP6? ([^ ]+) > .*/\1/p' | sort -u) + volatile "${sc}.translated_tuples" "$(paste -sd' ' <<<"${tuples}")" + record "${sc}" "peer distinct translated tuples" 2 "$(grep -c . <<<"${tuples}" || true)" + local leaked=no in_shards=yes t + for t in ${tuples}; do + [[ "${t}" == "${g}".* ]] && leaked=yes + [[ " ${shards} " == *" ${t%.*} "* ]] || in_shards=no + done + record "${sc}" "peer sees guest address" no "${leaked}" + record "${sc}" "peer sources are shard addresses" yes "${in_shards}" + record "${sc}" "peer SYN-ACK sent" yes "$(seen "$(hits "${sc}" peer "${synack_re}")")" +} +nat_scenario nat66 "${CLIENT6}" "${CLIENT6}" "${SHARDS6}" +nat_scenario nat64 "${EGRESS_NAT64}" "${CLIENT4}" "${SHARDS4}" + +# --------------------------------------------------------------------------- +# Gateway ingress over IPv6 and IPv4, through dfw-worker2. The lab's own +# checks drive the flows and assert replies, backend identity and DSR; the +# captures add what each point saw. +DFW_BACKEND_IFS=$(docker exec "${DFW_CP}" kubectl get vpcattachment -n ns60 -o json | + jq -r '.items[] | select(.status.node == "dfw-worker") | .status.hostInterface' | sort | paste -sd' ') +mapfile -t DFW_BACKENDS < <(docker exec "${DFW_CP}" kubectl get pods -n ns60 -l app=backend -o json | + jq -r '.items[] | select(.spec.nodeName == "dfw-worker") | .metadata.name' | sort) +# backend_hits SCENARIO REGEX sums REGEX over every dfw backend's eth0. +backend_hits() { + local i total=0 + for i in "${!DFW_BACKENDS[@]}"; do + total=$((total + $(hits "$1" "backend-guest-${i}" "$2"))) + done + echo "${total}" +} +gateway_scenario() { + local sc="$1" script="$2" vip="$3" client="$4" backend_client="$5" peer_capture="$6" + echo "=== ${sc} ===" + # verify-gateway-ingress-ipv4.sh captures on remote-host itself and stops + # every tcpdump there, ours included, so its own client-side checks stand + # in for the peer capture point. + [ "${peer_capture}" = yes ] && cap_start "${sc}" peer "${REMOTE}" eth1 + cap_start "${sc}" public "${TR1}" "${DFW_EDGE_UPLINKS[@]}" + cap_start "${sc}" dfw-compute-underlay "node:dfw-worker" "${DFW_COMPUTE_IFS[@]}" + # shellcheck disable=SC2086 # one interface per backend attachment + cap_start "${sc}" backend-veth-host "node:dfw-worker" ${DFW_BACKEND_IFS} + local i + for i in "${!DFW_BACKENDS[@]}"; do + cap_start "${sc}" "backend-guest-${i}" "pod:dfw/ns60/${DFW_BACKENDS[${i}]}" eth0 + done + cap_ready + local rc=0 + LAB=${LAB} "${SCRIPT_DIR}/${script}" --node dfw-worker2 >"${OUT}/logs/${sc}.log" 2>&1 || rc=$? + cap_stop_all + # The lab's ingress checks intermittently lose a TCP flow whose request + # reached a backend: the ns60 responder closes without writing a reply + # (docs/baseline.md). So the exact flow counts and the check's exit status + # are per-run values; the stable result is that each protocol had answered + # flows, which shows delivery in both directions. + volatile "${sc}.check_exit" "${rc}" + local proto answered + for proto in tcp udp; do + answered=$(sed -nE "s/^ ${proto}: ([0-9]+)\/[0-9]+ flows answered.*/\1/p" "${OUT}/logs/${sc}.log" | head -1) + volatile "${sc}.${proto}_flows_answered" "${answered:-0}/8" + record "${sc}" "${proto} flows answered by a backend" yes "$(seen "${answered:-0}")" + done + local in_re="${client}\.[0-9]+ > ${vip}\.80:" out_re="${vip}\.80 > ${client}\.[0-9]+:" + if [ "${peer_capture}" = yes ]; then + record "${sc}" "peer requests out" yes "$(seen "$(hits "${sc}" peer "${in_re}")")" + record "${sc}" "peer replies back" yes "$(seen "$(hits "${sc}" peer "${out_re}")")" + fi + record "${sc}" "public requests in" yes "$(seen "$(hits "${sc}" public "${in_re}")")" + record "${sc}" "public replies out" yes "$(seen "$(hits "${sc}" public "${out_re}")")" + record "${sc}" "backend-guest requests from client" yes \ + "$(seen "$(backend_hits "${sc}" "${backend_client}\.[0-9]+ > ")")" + record "${sc}" "backend-guest replies to client" yes \ + "$(seen "$(backend_hits "${sc}" "> ${backend_client}\.[0-9]+:")")" + record "${sc}" "backend-veth-host replies to client" yes \ + "$(seen "$(hits "${sc}" backend-veth-host "> ${backend_client}\.[0-9]+:")")" + record "${sc}" "backend-veth-host requests from client (peer redirect)" - \ + "$(seen "$(hits "${sc}" backend-veth-host "${backend_client}\.[0-9]+ > ")")" + record "${sc}" "dfw-compute-underlay requests" - "$(seen "$(hits "${sc}" dfw-compute-underlay "${in_re}")")" +} +gateway_scenario gateway-ipv6 verify-gateway-ingress.sh 2001:db8:6060:1::1 "${CLIENT6}" "${CLIENT6}" yes +gateway_scenario gateway-ipv4 verify-gateway-ingress-ipv4.sh 198.51.100.1 "${CLIENT4}" "${BACKEND_SEES4}" no + +# --------------------------------------------------------------------------- +# Negative isolation control. Each VPC's sjc guest sends PROBES datagrams to +# the address both dfw guests hold. Only the dfw guest in the sender's VPC +# may see them; the other VPC's guest, and its host veth, must see none. +echo "=== isolation ===" +sc=isolation +iport=$((20000 + RANDOM % 10000)) +nonce=$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') +volatile isolation.udp_port "${iport}" +volatile isolation.nonce "${nonce}" +cap_start "${sc}" dfw-guest-a "pod:dfw/${NS}/${POD[dfw-vpc-a]}" eth0 +cap_start "${sc}" dfw-guest-b "pod:dfw/${NS}/${POD[dfw-vpc-b]}" eth0 +cap_start "${sc}" dfw-veth-host-a "node:dfw-worker" "$(host_if 80 80)" +cap_start "${sc}" dfw-veth-host-b "node:dfw-worker" "$(host_if 81 81)" +cap_ready +# A different source port per VPC, so each one's probes are told apart at +# the receiving end. +declare -A SPORT=([a]=$((iport + 1)) [b]=$((iport + 2))) +for from in a b; do + for i in $(seq 1 "${PROBES}"); do + echo "${nonce}-${from}-${i}" | docker exec -i "${SJC_CP}" kubectl exec -i -n "${NS}" "${POD[sjc-vpc-${from}]}" -- \ + socat -u - "UDP6-SENDTO:[${DFW_ADDR}]:${iport},sourceport=${SPORT[${from}]},reuseaddr" || true + sleep 0.1 + done +done +cap_stop_all +for to in a b; do + for from in a b; do + sp=${SPORT[${from}]} + want=0 + [ "${from}" = "${to}" ] && want=${PROBES} + re="${SJC_ADDR}\.${sp} > ${DFW_ADDR}\.${iport}: UDP" + record "${sc}" "dfw-guest-${to} receives from vpc-${from}" "${want}" "$(hits "${sc}" "dfw-guest-${to}" "${re}")" + # Inbound, so delivered by peer redirect past the host veth: recorded + # only. The guest's own eth0 is the isolation evidence. + record "${sc}" "dfw-veth-host-${to} carries from vpc-${from} (peer redirect)" - "$(hits "${sc}" "dfw-veth-host-${to}" "${re}")" + done +done + +# --------------------------------------------------------------------------- +jq -R -s 'split("\n") | map(select(length > 0) | split("\t") + | {scenario: .[0], check: .[1], expect: .[2], got: .[3], result: .[4]}) + | sort_by(.scenario, .check)' "${OUT}/results.tsv" >"${OUT}/results.json" +total=$(jq length "${OUT}/results.json") +failed=$(jq '[.[] | select(.result == "fail")] | length' "${OUT}/results.json") +echo "=== ${total} checks, ${failed} failed; results in ${OUT}/results.json ===" +exit "${FAILED}" diff --git a/deploy/containerlab/scripts/baseline-twice.sh b/deploy/containerlab/scripts/baseline-twice.sh new file mode 100755 index 000000000..93e99e3e2 --- /dev/null +++ b/deploy/containerlab/scripts/baseline-twice.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# baseline-twice.sh: record the pinned versions, run baseline-observe.sh twice +# in a row, and require both runs to pass with identical results.json files +# that also match the committed baseline/reference-results.json. +# +# Usage: baseline-twice.sh [OUTDIR] +# OUTDIR defaults to bin/baseline/ at the repository root. +# +# Set BASELINE_WRITE_REFERENCE=1 to replace the reference with run-1's +# results once both runs agree, after a deliberate change to the fixture or +# its expectations. +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +LAB_DIR=$(cd "${SCRIPT_DIR}/.." && pwd) +REPO=$(cd "${LAB_DIR}/../.." && pwd) +REFERENCE="${LAB_DIR}/baseline/reference-results.json" +OUT=${1:-${REPO}/bin/baseline/$(date -u +%Y%m%dT%H%M%SZ)} +mkdir -p "${OUT}" + +rc=0 +"${SCRIPT_DIR}/baseline-versions.sh" "${OUT}/versions.yaml" || rc=1 +for run in 1 2; do + echo "##### run ${run} #####" + "${SCRIPT_DIR}/baseline-observe.sh" "${OUT}/run-${run}" 2>&1 | tee "${OUT}/run-${run}.log" || + rc=1 +done + +if diff -u "${OUT}/run-1/results.json" "${OUT}/run-2/results.json" >"${OUT}/runs.diff"; then + echo "runs 1 and 2: identical results ($(sha256sum <"${OUT}/run-1/results.json" | cut -c1-16))" +else + echo "FAIL: runs 1 and 2 differ:" >&2 + cat "${OUT}/runs.diff" >&2 + rc=1 +fi + +if [ "${rc}" -eq 0 ] && [ "${BASELINE_WRITE_REFERENCE:-0}" = 1 ]; then + cp "${OUT}/run-1/results.json" "${REFERENCE}" + echo "wrote ${REFERENCE}" +elif [ ! -f "${REFERENCE}" ]; then + echo "FAIL: no ${REFERENCE}; rerun with BASELINE_WRITE_REFERENCE=1 to create it" >&2 + rc=1 +elif diff -u "${REFERENCE}" "${OUT}/run-1/results.json" >"${OUT}/reference.diff"; then + echo "run 1 matches baseline/reference-results.json" +else + echo "FAIL: run 1 differs from baseline/reference-results.json:" >&2 + cat "${OUT}/reference.diff" >&2 + rc=1 +fi + +echo "evidence in ${OUT}" +exit "${rc}" diff --git a/deploy/containerlab/scripts/baseline-versions.sh b/deploy/containerlab/scripts/baseline-versions.sh new file mode 100755 index 000000000..98c1732e0 --- /dev/null +++ b/deploy/containerlab/scripts/baseline-versions.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# baseline-versions.sh: check the pinned versions in baseline/pins.yaml +# against the host tools, the lab's Dockerfiles, the ns80 fixture and the +# running lab, and record every version the baseline depends on. +# +# Usage: baseline-versions.sh OUTFILE +# +# OUTFILE gets a YAML record: each pin with its expected and observed value, +# then the unpinned values (kernel, Docker, Galactic revision, image IDs and +# digests). Exits non-zero if any pin disagrees. +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +source "${SCRIPT_DIR}/lib.sh" + +LAB=${LAB:-gvpc} +OUTFILE=${1:?usage: baseline-versions.sh OUTFILE} +LAB_DIR=$(cd "${SCRIPT_DIR}/.." && pwd) +REPO=$(cd "${LAB_DIR}/../.." && pwd) +PINS="${LAB_DIR}/baseline/pins.yaml" + +pin() { awk -v k="$1" -F': ' '$1 == k {print $2; exit}' "${PINS}"; } +dockerfile_arg() { sed -nE "s/^ARG $2=(.*)/\1/p" "$1" | head -1; } + +mkdir -p "$(dirname "${OUTFILE}")" +rc=0 +{ + echo "# Recorded by scripts/baseline-versions.sh" + echo "pins:" +} >"${OUTFILE}" + +# check KEY SOURCE OBSERVED +check() { + local want + want=$(pin "$1") + local ok=match + if [ "${want}" != "$3" ]; then + ok=MISMATCH + rc=1 + fi + printf ' - {pin: %s, source: "%s", want: "%s", got: "%s", result: %s}\n' "$1" "$2" "${want}" "$3" "${ok}" >>"${OUTFILE}" + printf '%-9s %-17s %-40s want %s got %s\n' "${ok}" "$1" "$2" "${want}" "$3" +} + +check containerlab "containerlab version" "$(containerlab version 2>/dev/null | sed -nE 's/^ *version: *//p' | head -1)" +check kind "kind version" "$(kind version | awk '{print $2}')" +check task "task --version" "$(task --version | sed -E 's/^Task version: //; s/ .*//')" +check kindest_node "kindest-node-galactic Dockerfile" \ + "$(dockerfile_arg "${LAB_DIR}/containers/kindest-node-galactic/Dockerfile" KINDEST_VER)" +check kindest_node "dfw API server" \ + "$(docker exec "$(control_plane dfw)" kubectl version -o json | jq -r .serverVersion.gitVersion)" +check frr "fabric-router Dockerfile" "$(dockerfile_arg "${REPO}/containers/fabric-router/Dockerfile" FRR_VERSION)" +check frr "tr1 vtysh" \ + "$(docker exec "clab-${LAB}-tr1" vtysh -c 'show version' | sed -nE 's/^FRRouting ([0-9.]+).*/\1/p' | head -1)" +check remote_host_base "remote-host Dockerfile" "$(sed -nE 's/^FROM (.*)/\1/p' "${REPO}/containers/remote-host/Dockerfile" | head -1)" +for f in base/guests.yaml dfw/tap-guest.yaml; do + for img in $(sed -nE 's/^ *image: *//p' "${LAB_DIR}/resources/tenants/ns80/${f}" | sort -u); do + check guest_image "ns80/${f}" "${img}" + done +done +# What the running guests were actually started from. +for site in dfw sjc; do + got=$(docker exec "$(control_plane "${site}")" kubectl get pods -n ns80 -o json | + jq -r '[.items[].status.containerStatuses[]?.imageID] | unique | join(",")') + check guest_image "ns80 pods in ${site}" "ghcr.io/datum-cloud/galactic-debug@${got##*@}" +done + +{ + echo "recorded:" + echo " host_kernel: \"$(uname -r)\"" + echo " docker: \"$(docker version -f '{{.Server.Version}}')\"" + echo " kubectl_client: \"$(kubectl version --client -o json | jq -r .clientVersion.gitVersion)\"" + echo " node_kernel: \"$(docker exec dfw-worker uname -r)\"" + echo " galactic_revision: \"$(git -C "${REPO}" rev-parse HEAD)\"" + echo " galactic_tree: \"$(git -C "${REPO}" diff --quiet HEAD && echo clean || echo dirty)\"" + echo " local_images:" + docker images --no-trunc --format '{{.Repository}}:{{.Tag}} {{.ID}}' | + grep -E '^(kindest/node:galactic|fabric-router:|remote-host:|galactic-[a-z-]+:|fabric-[a-z-]+:|masque-[a-z-]+:)' | + sort | awk '{printf " - {image: \"%s\", id: \"%s\"}\n", $1, $2}' + echo " dfw_worker_images:" + docker exec dfw-worker crictl images -o json | + jq -r '.images[] | select(any(.repoTags[]?; test("galactic|fabric|masque"))) | "\(.repoTags[0]) \(.id)"' | + sort | awk '{printf " - {image: \"%s\", id: \"%s\"}\n", $1, $2}' +} >>"${OUTFILE}" + +echo "versions recorded in ${OUTFILE}" +exit "${rc}" diff --git a/deploy/containerlab/scripts/verify-ns80.sh b/deploy/containerlab/scripts/verify-ns80.sh new file mode 100755 index 000000000..51f5e4bd2 --- /dev/null +++ b/deploy/containerlab/scripts/verify-ns80.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# verify-ns80.sh: check the dedicated-placement fixture (resources/tenants/ns80) +# is in place and forwarding. Each VPC's guests sit on dfw's and sjc's compute +# workers, both VPCs' guests hold the same address at each site, and each VPC +# reaches its own guest across sites in both directions. The full +# observations, with captures and the isolation control, are +# task baseline:observe. +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +source "${SCRIPT_DIR}/lib.sh" + +NS=ns80 +DFW=$(control_plane dfw) +SJC=$(control_plane sjc) +declare -A WANT=([dfw]=fd20:80:ff01::1:0:0 [sjc]=fd20:80:ff02::1:0:0) + +declare -A POD +for site in dfw sjc; do + cp=$(control_plane "${site}") + for app in vpc-a vpc-b; do + pod=$(pod_name "${cp}" "${NS}" "app=${app}") + if [ -z "${pod}" ]; then + echo "ns80 ${app} pod not found in ${site}" >&2 + exit 1 + fi + node=$(pod_scheduling_node "${cp}" "${NS}" "${pod}") + if [ "${node}" != "${site}-worker" ]; then + echo "ns80 ${app} in ${site} runs on ${node}, want ${site}-worker" >&2 + exit 1 + fi + addr=$(pod_ip6 "${cp}" "${NS}" "${pod}") + if [ "${addr}" != "${WANT[${site}]}" ]; then + echo "ns80 ${app} in ${site} has address '${addr}', want ${WANT[${site}]}" >&2 + exit 1 + fi + POD[${site}-${app}]=${pod} + echo "${site} ${app}: ${pod} on ${node}, ${addr}" + done +done + +for app in vpc-a vpc-b; do + echo "--- ${app}: dfw -> sjc ---" + ping_pod "${DFW}" "${NS}" "${POD[dfw-${app}]}" -6 "${WANT[sjc]}" + echo "--- ${app}: sjc -> dfw ---" + ping_pod "${SJC}" "${NS}" "${POD[sjc-${app}]}" -6 "${WANT[dfw]}" +done + +echo "ns80: both VPCs reach across sites in both directions" From 2b59e168592a8abc43687fb2750c3de12d79be53 Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Wed, 18 Nov 2026 22:21:12 -0500 Subject: [PATCH 2/2] fix(containerlab): Drop the removed TAP manifest from the pin check The TAP stand-in moved to datum-cloud/galactic#1444, so baseline-versions.sh no longer has a dfw/tap-guest.yaml to read and logged a sed error on every run. --- deploy/containerlab/scripts/baseline-versions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/containerlab/scripts/baseline-versions.sh b/deploy/containerlab/scripts/baseline-versions.sh index 98c1732e0..c917e7b56 100755 --- a/deploy/containerlab/scripts/baseline-versions.sh +++ b/deploy/containerlab/scripts/baseline-versions.sh @@ -53,7 +53,7 @@ check frr "fabric-router Dockerfile" "$(dockerfile_arg "${REPO}/containers/fabri check frr "tr1 vtysh" \ "$(docker exec "clab-${LAB}-tr1" vtysh -c 'show version' | sed -nE 's/^FRRouting ([0-9.]+).*/\1/p' | head -1)" check remote_host_base "remote-host Dockerfile" "$(sed -nE 's/^FROM (.*)/\1/p' "${REPO}/containers/remote-host/Dockerfile" | head -1)" -for f in base/guests.yaml dfw/tap-guest.yaml; do +for f in base/guests.yaml; do for img in $(sed -nE 's/^ *image: *//p' "${LAB_DIR}/resources/tenants/ns80/${f}" | sort -u); do check guest_image "ns80/${f}" "${img}" done