diff --git a/deploy/containerlab/AGENTS.md b/deploy/containerlab/AGENTS.md index 187f4955..e9abac5f 100644 --- a/deploy/containerlab/AGENTS.md +++ b/deploy/containerlab/AGENTS.md @@ -45,3 +45,4 @@ Full documentation — topology, addressing, tasks — is in [README.md](README.md). Verification commands are in [docs/verification.md](docs/verification.md). Deploying the five test VPCs and verifying cross-site/same-node connectivity is documented in [docs/tenants.md](docs/tenants.md). +The dedicated-placement baseline (`ns80`, `task baseline:twice`) is documented in [docs/baseline.md](docs/baseline.md). diff --git a/deploy/containerlab/README.md b/deploy/containerlab/README.md index 24cca5ee..edab89a9 100644 --- a/deploy/containerlab/README.md +++ b/deploy/containerlab/README.md @@ -599,7 +599,9 @@ PublicNAT and PublicService paths and the built-in default. See See [docs/verification.md](docs/verification.md) for transit fabric, FRR, and galactic-router health checks, and [docs/tenants.md](docs/tenants.md) for deploying and verifying -the `ns10`/`ns20`/`ns30`/`ns40` test VPCs. +the `ns10`/`ns20`/`ns30`/`ns40` test VPCs. [docs/baseline.md](docs/baseline.md) +covers the dedicated-placement baseline: the `ns80` fixture, its scripted packet +observations and pinned versions (`task baseline:twice`). `task verify` (and its constituent `task verify:scenarios`) also pings every VPC's pods end-to-end via `task verify:ns10`/`ns20`/`ns30`/`ns40` — diff --git a/deploy/containerlab/Taskfile.yaml b/deploy/containerlab/Taskfile.yaml index d597c37e..709d970e 100644 --- a/deploy/containerlab/Taskfile.yaml +++ b/deploy/containerlab/Taskfile.yaml @@ -434,6 +434,7 @@ tasks: - task: "deploy:ns40" - task: "deploy:ns60" - task: "deploy:ns71" + - task: "deploy:ns80" "deploy:ns10": desc: Deploy ns10 test VPC (IPv6-only, fd20 ULA) @@ -477,6 +478,15 @@ tasks: - ./scripts/deploy-ns.sh ns71 sjc - ./scripts/publish-attachments.sh ns71 sjc + # The dedicated-placement baseline fixture (docs/baseline.md): two VPCs + # with identical guest addresses on dfw's and sjc's compute workers. + "deploy:ns80": + desc: Deploy ns80 test VPCs (overlapping guests on two workers; baseline fixture) + cmds: + - ./scripts/deploy-tenant-policy.sh ns80 dfw sjc + - ./scripts/deploy-ns.sh ns80 dfw sjc + - ./scripts/publish-attachments.sh ns80 dfw sjc + verify: desc: Run all verification checks cmds: @@ -1296,6 +1306,7 @@ tasks: - task: "verify:ns20" - task: "verify:ns30" - task: "verify:ns40" + - task: "verify:ns80" # Not idempotent while it runs: it edits ns60's dfw policy and unbinds # ns30/private-b's ingress for a moment, and restores both on exit. @@ -1364,6 +1375,30 @@ tasks: cmds: - ./scripts/verify-ns40.sh + "verify:ns80": + desc: Verify ns80 placement, overlapping addresses and cross-site ping in both VPCs + cmds: + - ./scripts/verify-ns80.sh + + # The dedicated-placement baseline (docs/baseline.md). Not part of verify: + # each observation takes a few minutes and runs the gateway ingress checks + # again under capture. Evidence lands in bin/baseline/ at the repository + # root. + "baseline:versions": + desc: Check baseline/pins.yaml against the host, the Dockerfiles and the running lab, and record versions + cmds: + - LAB={{.LAB}} ./scripts/baseline-versions.sh {{.CLI_ARGS | default "../../bin/baseline/versions.yaml"}} + + "baseline:observe": + desc: Observe today's remote paths on ns80 once, with captures (OUTDIR via task baseline:observe -- DIR) + cmds: + - LAB={{.LAB}} ./scripts/baseline-observe.sh {{.CLI_ARGS | default "../../bin/baseline/observe"}} + + "baseline:twice": + desc: Record versions, observe twice, and require identical results matching baseline/reference-results.json + cmds: + - LAB={{.LAB}} ./scripts/baseline-twice.sh {{.CLI_ARGS}} + clean: desc: Destroy the lab and delete artifacts cmds: diff --git a/deploy/containerlab/baseline/pins.yaml b/deploy/containerlab/baseline/pins.yaml new file mode 100644 index 00000000..c14493a9 --- /dev/null +++ b/deploy/containerlab/baseline/pins.yaml @@ -0,0 +1,14 @@ +# Versions the dedicated-placement baseline (docs/baseline.md) was recorded +# with. scripts/baseline-versions.sh fails when the host tools, the lab's +# Dockerfiles, the fixture or the running lab disagree with a value here. +# Change a pin together with baseline/reference-results.json, never alone. +# +# Host tools +containerlab: 0.75.0 +kind: v0.33.0 +task: 3.51.1 +# Images and components the lab builds or pulls +kindest_node: v1.37.0 +frr: 10.7.1 +remote_host_base: alpine:3.24 +guest_image: ghcr.io/datum-cloud/galactic-debug@sha256:b64e338f845d06f82f2575ae6bb6855da330af7551a8816b7ade2602a85722f2 diff --git a/deploy/containerlab/baseline/reference-results.json b/deploy/containerlab/baseline/reference-results.json new file mode 100644 index 00000000..e77e2672 --- /dev/null +++ b/deploy/containerlab/baseline/reference-results.json @@ -0,0 +1,758 @@ +[ + { + "scenario": "cross-node-a", + "check": "dfw->sjc echo replies", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-a", + "check": "dfw-compute-underlay dfw->sjc requests", + "expect": "-", + "got": "5", + "result": "observed" + }, + { + "scenario": "cross-node-a", + "check": "dfw-compute-underlay sjc->dfw requests", + "expect": "-", + "got": "5", + "result": "observed" + }, + { + "scenario": "cross-node-a", + "check": "dfw-edge-uplink dfw->sjc requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-a", + "check": "dfw-edge-uplink sjc->dfw requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-a", + "check": "dfw-guest dfw->sjc requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-a", + "check": "dfw-guest sjc->dfw requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-a", + "check": "dfw-veth-host dfw->sjc requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-a", + "check": "dfw-veth-host sjc->dfw requests (peer redirect)", + "expect": "-", + "got": "0", + "result": "observed" + }, + { + "scenario": "cross-node-a", + "check": "sjc->dfw echo replies", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-a", + "check": "sjc-compute-underlay dfw->sjc requests", + "expect": "-", + "got": "5", + "result": "observed" + }, + { + "scenario": "cross-node-a", + "check": "sjc-compute-underlay sjc->dfw requests", + "expect": "-", + "got": "5", + "result": "observed" + }, + { + "scenario": "cross-node-a", + "check": "sjc-edge-uplink dfw->sjc requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-a", + "check": "sjc-edge-uplink sjc->dfw requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-a", + "check": "sjc-guest dfw->sjc requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-a", + "check": "sjc-guest sjc->dfw requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-a", + "check": "sjc-veth-host dfw->sjc requests (peer redirect)", + "expect": "-", + "got": "0", + "result": "observed" + }, + { + "scenario": "cross-node-a", + "check": "sjc-veth-host sjc->dfw requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-b", + "check": "dfw->sjc echo replies", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-b", + "check": "dfw-compute-underlay dfw->sjc requests", + "expect": "-", + "got": "5", + "result": "observed" + }, + { + "scenario": "cross-node-b", + "check": "dfw-compute-underlay sjc->dfw requests", + "expect": "-", + "got": "5", + "result": "observed" + }, + { + "scenario": "cross-node-b", + "check": "dfw-edge-uplink dfw->sjc requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-b", + "check": "dfw-edge-uplink sjc->dfw requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-b", + "check": "dfw-guest dfw->sjc requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-b", + "check": "dfw-guest sjc->dfw requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-b", + "check": "dfw-veth-host dfw->sjc requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-b", + "check": "dfw-veth-host sjc->dfw requests (peer redirect)", + "expect": "-", + "got": "0", + "result": "observed" + }, + { + "scenario": "cross-node-b", + "check": "sjc->dfw echo replies", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-b", + "check": "sjc-compute-underlay dfw->sjc requests", + "expect": "-", + "got": "5", + "result": "observed" + }, + { + "scenario": "cross-node-b", + "check": "sjc-compute-underlay sjc->dfw requests", + "expect": "-", + "got": "5", + "result": "observed" + }, + { + "scenario": "cross-node-b", + "check": "sjc-edge-uplink dfw->sjc requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-b", + "check": "sjc-edge-uplink sjc->dfw requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-b", + "check": "sjc-guest dfw->sjc requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-b", + "check": "sjc-guest sjc->dfw requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "cross-node-b", + "check": "sjc-veth-host dfw->sjc requests (peer redirect)", + "expect": "-", + "got": "0", + "result": "observed" + }, + { + "scenario": "cross-node-b", + "check": "sjc-veth-host sjc->dfw requests", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "fixture", + "check": "dfw vpc-a guest address", + "expect": "fd20:80:ff01::1:0:0", + "got": "fd20:80:ff01::1:0:0", + "result": "pass" + }, + { + "scenario": "fixture", + "check": "dfw vpc-a guest on compute worker", + "expect": "dfw-worker", + "got": "dfw-worker", + "result": "pass" + }, + { + "scenario": "fixture", + "check": "dfw vpc-b guest address equals vpc-a's", + "expect": "fd20:80:ff01::1:0:0", + "got": "fd20:80:ff01::1:0:0", + "result": "pass" + }, + { + "scenario": "fixture", + "check": "dfw vpc-b guest on compute worker", + "expect": "dfw-worker", + "got": "dfw-worker", + "result": "pass" + }, + { + "scenario": "fixture", + "check": "sjc vpc-a guest address", + "expect": "fd20:80:ff02::1:0:0", + "got": "fd20:80:ff02::1:0:0", + "result": "pass" + }, + { + "scenario": "fixture", + "check": "sjc vpc-a guest on compute worker", + "expect": "sjc-worker", + "got": "sjc-worker", + "result": "pass" + }, + { + "scenario": "fixture", + "check": "sjc vpc-b guest address equals vpc-a's", + "expect": "fd20:80:ff02::1:0:0", + "got": "fd20:80:ff02::1:0:0", + "result": "pass" + }, + { + "scenario": "fixture", + "check": "sjc vpc-b guest on compute worker", + "expect": "sjc-worker", + "got": "sjc-worker", + "result": "pass" + }, + { + "scenario": "gateway-ipv4", + "check": "backend-guest replies to client", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv4", + "check": "backend-guest requests from client", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv4", + "check": "backend-veth-host replies to client", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv4", + "check": "backend-veth-host requests from client (peer redirect)", + "expect": "-", + "got": "no", + "result": "observed" + }, + { + "scenario": "gateway-ipv4", + "check": "dfw-compute-underlay requests", + "expect": "-", + "got": "yes", + "result": "observed" + }, + { + "scenario": "gateway-ipv4", + "check": "public replies out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv4", + "check": "public requests in", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv4", + "check": "tcp flows answered by a backend", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv4", + "check": "udp flows answered by a backend", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv6", + "check": "backend-guest replies to client", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv6", + "check": "backend-guest requests from client", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv6", + "check": "backend-veth-host replies to client", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv6", + "check": "backend-veth-host requests from client (peer redirect)", + "expect": "-", + "got": "no", + "result": "observed" + }, + { + "scenario": "gateway-ipv6", + "check": "dfw-compute-underlay requests", + "expect": "-", + "got": "yes", + "result": "observed" + }, + { + "scenario": "gateway-ipv6", + "check": "peer replies back", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv6", + "check": "peer requests out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv6", + "check": "public replies out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv6", + "check": "public requests in", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv6", + "check": "tcp flows answered by a backend", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "gateway-ipv6", + "check": "udp flows answered by a backend", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "isolation", + "check": "dfw-guest-a receives from vpc-a", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "isolation", + "check": "dfw-guest-a receives from vpc-b", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "isolation", + "check": "dfw-guest-b receives from vpc-a", + "expect": "0", + "got": "0", + "result": "pass" + }, + { + "scenario": "isolation", + "check": "dfw-guest-b receives from vpc-b", + "expect": "5", + "got": "5", + "result": "pass" + }, + { + "scenario": "isolation", + "check": "dfw-veth-host-a carries from vpc-a (peer redirect)", + "expect": "-", + "got": "0", + "result": "observed" + }, + { + "scenario": "isolation", + "check": "dfw-veth-host-a carries from vpc-b (peer redirect)", + "expect": "-", + "got": "0", + "result": "observed" + }, + { + "scenario": "isolation", + "check": "dfw-veth-host-b carries from vpc-a (peer redirect)", + "expect": "-", + "got": "0", + "result": "observed" + }, + { + "scenario": "isolation", + "check": "dfw-veth-host-b carries from vpc-b (peer redirect)", + "expect": "-", + "got": "0", + "result": "observed" + }, + { + "scenario": "nat64", + "check": "dfw-compute-underlay SYN out", + "expect": "-", + "got": "yes", + "result": "observed" + }, + { + "scenario": "nat64", + "check": "dfw-compute-underlay SYN-ACK back", + "expect": "-", + "got": "yes", + "result": "observed" + }, + { + "scenario": "nat64", + "check": "dfw-guest-a SYN out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "dfw-guest-a SYN-ACK back", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "dfw-guest-b SYN out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "dfw-guest-b SYN-ACK back", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "dfw-veth-host-a SYN out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "dfw-veth-host-a SYN-ACK back (peer redirect)", + "expect": "-", + "got": "no", + "result": "observed" + }, + { + "scenario": "nat64", + "check": "dfw-veth-host-b SYN out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "dfw-veth-host-b SYN-ACK back (peer redirect)", + "expect": "-", + "got": "no", + "result": "observed" + }, + { + "scenario": "nat64", + "check": "peer SYN-ACK sent", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "peer distinct translated tuples", + "expect": "2", + "got": "2", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "peer sees guest address", + "expect": "no", + "got": "no", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "peer sources are shard addresses", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "public SYN out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "public SYN-ACK back", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "vpc-a fetch status", + "expect": "200", + "got": "200", + "result": "pass" + }, + { + "scenario": "nat64", + "check": "vpc-b fetch status", + "expect": "200", + "got": "200", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "dfw-compute-underlay SYN out", + "expect": "-", + "got": "yes", + "result": "observed" + }, + { + "scenario": "nat66", + "check": "dfw-compute-underlay SYN-ACK back", + "expect": "-", + "got": "yes", + "result": "observed" + }, + { + "scenario": "nat66", + "check": "dfw-guest-a SYN out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "dfw-guest-a SYN-ACK back", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "dfw-guest-b SYN out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "dfw-guest-b SYN-ACK back", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "dfw-veth-host-a SYN out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "dfw-veth-host-a SYN-ACK back (peer redirect)", + "expect": "-", + "got": "no", + "result": "observed" + }, + { + "scenario": "nat66", + "check": "dfw-veth-host-b SYN out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "dfw-veth-host-b SYN-ACK back (peer redirect)", + "expect": "-", + "got": "no", + "result": "observed" + }, + { + "scenario": "nat66", + "check": "peer SYN-ACK sent", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "peer distinct translated tuples", + "expect": "2", + "got": "2", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "peer sees guest address", + "expect": "no", + "got": "no", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "peer sources are shard addresses", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "public SYN out", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "public SYN-ACK back", + "expect": "yes", + "got": "yes", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "vpc-a fetch status", + "expect": "200", + "got": "200", + "result": "pass" + }, + { + "scenario": "nat66", + "check": "vpc-b fetch status", + "expect": "200", + "got": "200", + "result": "pass" + } +] diff --git a/deploy/containerlab/docs/baseline.md b/deploy/containerlab/docs/baseline.md new file mode 100644 index 00000000..224d4056 --- /dev/null +++ b/deploy/containerlab/docs/baseline.md @@ -0,0 +1,149 @@ +# Dedicated-placement baseline + +This page describes a fixture and a set of scripted packet observations of +today's remote paths. The fixture is two VPCs with identical guest addresses +on two physical workers. Services run on dedicated edge nodes, and workloads +run on compute nodes. Later work on the Galactic Worker reuses this fixture +and compares its results against the reference recorded here. + +The fixture has no TAP-backed guest. The tenant TAP capture point belongs to +[#1444](https://github.com/datum-cloud/galactic/issues/1444). + +## Fixture + +`ns80` (`resources/tenants/ns80/`) holds two VPCs with the same guest address +at each site: + +| VPC | Attachment | dfw-worker guest | sjc-worker guest | Kind | +| ---- | ---------- | --------------------- | --------------------- | ---------------------- | +| `80` | `vpc-a` | `fd20:80:ff01::1:0:0` | `fd20:80:ff02::1:0:0` | veth (`galactic-veth`) | +| `81` | `vpc-b` | `fd20:80:ff01::1:0:0` | `fd20:80:ff02::1:0:0` | veth (`galactic-veth`) | + +- **Two physical workers.** `dfw-worker` and `sjc-worker` are each their site's + only compute node. They have distinct SRv6 locators and reach each other over + the routed underlay through their sites' edge nodes and the transit routers. + Gateway and NAT run only on the edge nodes (`dfw-worker2`, `dfw-worker3`, + `sjc-worker2`), so this is dedicated placement. +- **Identical addressing.** The guests take fixed addresses through IPAM's + `addresses` path, so the overlap does not depend on allocation order. Each + guest's gateway lies outside its `/96` and is reached on-link, as on every + `addresses` attachment. +- **Public client and egress peer.** `remote-host` behind `tr4` serves HTTP + on `[2001:db8:1:40::2]:80` and `11.1.40.2:80`. It is the client for gateway + ingress and the peer for NAT egress. +- **Policy.** Every `ns80` attachment is bound to the `private-nat` set + (`resources/tenants/policies/README.md`). + +`task deploy` deploys `ns80` as part of `deploy:scenarios`. `task deploy:ns80` +redeploys it alone, and `task verify:ns80` (part of `task verify`) checks +placement, the overlapping addresses and cross-site reachability. + +## Running the observations + +```bash +cd deploy/containerlab +task deploy # from a destroyed lab: build images, deploy everything +task baseline:twice # pins, two observation runs, compare with the reference +task verify # the lab's full verification still passes +``` + +`task baseline:twice` writes to `bin/baseline//` at the +repository root, or to the directory given after `--`. It fails if a pinned +version disagrees, if any check fails in either run, if the two runs' +`results.json` files differ, or if run 1 differs from +`baseline/reference-results.json`. `task baseline:observe -- DIR` runs one +observation and `task baseline:versions -- FILE` records versions alone. + +Each run directory holds: + +| File | Contents | +| ------------------ | -------------------------------------------------------------------------------------- | +| `results.json` | One row per check: scenario, check, expected, observed, result. Stable values only. | +| `volatile.tsv` | Per-run values: source ports, the isolation nonce, translated tuples, shard addresses. | +| `pcap//` | One pcap per capture point and interface, with tcpdump's log beside it. | +| `logs/` | Probe output and the gateway checks' full logs. | + +`results.json` never holds a port, nonce, timestamp or MAC address. Those +values change on every run, and only stable values let two runs compare equal. + +## Scenarios + +Each scenario runs in its own capture window, in both directions. + +| Scenario | Traffic | Expected | +| -------------- | ----------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------- | +| `cross-node-a` | 5 ICMPv6 echoes dfw → sjc, then 5 sjc → dfw, in VPC 80 | 5 replies each way, every request seen at every expected point | +| `cross-node-b` | The same in VPC 81 | The same | +| `nat66` | Both dfw guests fetch `http://[2001:db8:1:40::2]/` at once from one identical source port | Both get 200, the peer sees 2 distinct translated tuples from shard addresses only | +| `nat64` | The same to `http://[2001:db8:64::b01:2802]/`, under the shards' NAT64 prefix | The same over IPv4 | +| `gateway-ipv6` | `verify-gateway-ingress.sh --node dfw-worker2`: TCP and UDP to `2001:db8:6060:1::1` | Answered flows in both protocols; requests and DSR replies seen at the public side and the backends | +| `gateway-ipv4` | `verify-gateway-ingress-ipv4.sh --node dfw-worker2`: TCP and UDP to `198.51.100.1` | The same; backends see the client as `64:ff9b::b01:2802` | +| `isolation` | 5 UDP datagrams from each VPC's sjc guest to `[fd20:80:ff01::1:0:0]` | Each VPC's dfw guest and host veth see its own 5 and none of the other VPC's | + +The gateway scenarios reuse the lab's ingress checks against `ns60`'s +backends on `dfw-worker`, and pin the VIP to `dfw-worker2` on `tr1` for the +duration of the check. + +The IPv4 check captures on `remote-host` itself and stops every tcpdump +there, so `gateway-ipv4` has no `remote-host` capture point. The check's own +client-side assertions stand in for it. + +### Intermittent TCP loss in the gateway checks + +Both lab ingress checks lose a TCP flow now and then, with or without these +captures: in three back-to-back runs of `verify-gateway-ingress-ipv4.sh --node +dfw-worker2` with no capture running, one answered 6 of 8 TCP flows. The +captures show that the lost flow is delivered. The request reaches the +backend over the gateway, and the backend acknowledges it. Then the `ns60` +responder (`socat ... fork SYSTEM:`) closes without writing a reply, 1 to +2 ms after the client's FIN, and that close returns to the client over the +DSR path. UDP flows are never lost. + +So each gateway scenario's exact flow counts and the check's exit status go +to `volatile.tsv`, and `results.json` records only whether each protocol had +answered flows. + +## Capture points + +| Point | Where | +| --------------- | ----------------------------------------------------------------------------------------------------------------------- | +| Public RX/TX | `tr1`'s members facing both dfw edge nodes (`eth1`, `eth6`, `eth5`, `eth7`), and `remote-host` `eth1` | +| Veth peers | Each guest's `eth0` (an `ns80` guest, or an `ns60` backend for gateway ingress) and its host veth `GH` | +| Remote underlay | `tr2`'s members facing `sjc-worker2` (`eth1`, `eth5`), `tr1`'s dfw edge members, and both compute workers' bond members | + +A point that names several interfaces sums them. The bond member a flow +uses, and the dfw edge node it crosses, are chosen by hashes that can differ +between runs. The link the flow crosses does not change. + +Some points cannot see one direction, because of how today's datapath +delivers packets. Those checks are recorded as observations without an +expectation, and are still compared between runs: + +- **Edge nodes.** The edge nodes forward NAT and gateway traffic in XDP, + which tcpdump on those nodes never sees (#565). Public-side captures are + therefore taken on the transit router's end of each edge uplink. +- **Host veths, inbound.** A host veth carries its guest's outbound packets. + Inbound packets reach the guest through `bpf_redirect_peer` in + `usid_ingress` (`internal/plumbing/ebpf/prog/usid.c`), straight into the + guest's side of the pair, so tcpdump on the host side never sees them. The + guest's own `eth0` is the evidence for inbound delivery, including for + the isolation control. +- **Compute bond members.** These captures are recorded without an + expectation. + +## Pinned versions + +`baseline/pins.yaml` pins the host tools (containerlab, kind, task), the +Kind node version, FRR, the `remote-host` base image and the guest image +digest. `scripts/baseline-versions.sh` checks each pin against its source: +the host tool, the lab's Dockerfiles, the `ns80` manifests and the running +lab. It then records the host and node kernels, Docker, the Galactic +revision and every lab image ID in `versions.yaml`. Change a pin only +together with the reference results. + +## Updating the reference + +After a deliberate change to the fixture or to an expectation, rerun with +`BASELINE_WRITE_REFERENCE=1 task baseline:twice`. The reference is replaced +only when both runs pass and agree. Commit the new +`baseline/reference-results.json` with the change that caused it. diff --git a/deploy/containerlab/docs/tenants.md b/deploy/containerlab/docs/tenants.md index b4792ab2..87d3ee4b 100644 --- a/deploy/containerlab/docs/tenants.md +++ b/deploy/containerlab/docs/tenants.md @@ -24,6 +24,9 @@ or `policy//` for a site with attachments the others lack, see attachment's status after its pods start (`scripts/publish-attachments.sh`). A pod's interface forwards nothing until then. +`ns80`, two VPCs whose guests share addresses across two workers, is the +dedicated-placement baseline fixture and is described in [baseline.md](baseline.md). + They differ only in scope and addressing: | VPC | Sites | Address families | VRF interface | Notes | diff --git a/deploy/containerlab/resources/tenants/ns80/base/guests.yaml b/deploy/containerlab/resources/tenants/ns80/base/guests.yaml new file mode 100644 index 00000000..cef83684 --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80/base/guests.yaml @@ -0,0 +1,56 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: guest-a + labels: + app: vpc-a +spec: + replicas: 1 + selector: + matchLabels: + app: vpc-a + template: + metadata: + labels: + app: vpc-a + annotations: + v1.multus-cni.io/default-network: ns80/vpc-a + spec: + nodeSelector: + galactic.datumapis.com/node: compute + terminationGracePeriodSeconds: 1 + containers: + - name: guest + # Pinned by digest: baseline/pins.yaml records the same value. + image: ghcr.io/datum-cloud/galactic-debug@sha256:b64e338f845d06f82f2575ae6bb6855da330af7551a8816b7ade2602a85722f2 + imagePullPolicy: IfNotPresent + command: ["sleep", "infinity"] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: guest-b + labels: + app: vpc-b +spec: + replicas: 1 + selector: + matchLabels: + app: vpc-b + template: + metadata: + labels: + app: vpc-b + annotations: + v1.multus-cni.io/default-network: ns80/vpc-b + spec: + nodeSelector: + galactic.datumapis.com/node: compute + terminationGracePeriodSeconds: 1 + containers: + - name: guest + # Pinned by digest: baseline/pins.yaml records the same value. + image: ghcr.io/datum-cloud/galactic-debug@sha256:b64e338f845d06f82f2575ae6bb6855da330af7551a8816b7ade2602a85722f2 + imagePullPolicy: IfNotPresent + command: ["sleep", "infinity"] diff --git a/deploy/containerlab/resources/tenants/ns80/base/kustomization.yaml b/deploy/containerlab/resources/tenants/ns80/base/kustomization.yaml new file mode 100644 index 00000000..ce97d47a --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80/base/kustomization.yaml @@ -0,0 +1,8 @@ +# ns80: two VPCs, 80 (vpc-a) and 81 (vpc-b), whose guests use identical +# addresses at each site. One guest per VPC runs on each site's compute worker, +# so the pair spans two physical workers. This is the dedicated-placement +# baseline fixture (docs/baseline.md). +namespace: ns80 +resources: + - namespace.yaml + - guests.yaml diff --git a/deploy/containerlab/resources/tenants/ns80/base/namespace.yaml b/deploy/containerlab/resources/tenants/ns80/base/namespace.yaml new file mode 100644 index 00000000..cdbb6695 --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80/base/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ns80 diff --git a/deploy/containerlab/resources/tenants/ns80/dfw/kustomization.yaml b/deploy/containerlab/resources/tenants/ns80/dfw/kustomization.yaml new file mode 100644 index 00000000..ee81d25b --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80/dfw/kustomization.yaml @@ -0,0 +1,3 @@ +resources: + - ../base + - nad.yaml diff --git a/deploy/containerlab/resources/tenants/ns80/dfw/nad.yaml b/deploy/containerlab/resources/tenants/ns80/dfw/nad.yaml new file mode 100644 index 00000000..f56bd293 --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80/dfw/nad.yaml @@ -0,0 +1,71 @@ +# dfw: both VPCs' guests get the same address, fd20:80:ff01::1:0:0. +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: vpc-a + namespace: ns80 +spec: + config: |- + { + "cniVersion": "1.0.0", + "name": "vpc-a", + "plugins": [ + { + "type": "galactic-veth", + "vpc": "80", + "vpcattachment": "80", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "addresses": [ + { + "address": "fd20:80:ff01:0:0:1::/96", + "gateway": "fd20:80:ff01::1" + } + ] + } + }, + { + "type": "galactic-bgp", + "vpc": "80", + "vpcattachment": "80", + "namespace": "galactic-system" + } + ] + } +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: vpc-b + namespace: ns80 +spec: + config: |- + { + "cniVersion": "1.0.0", + "name": "vpc-b", + "plugins": [ + { + "type": "galactic-veth", + "vpc": "81", + "vpcattachment": "81", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "addresses": [ + { + "address": "fd20:80:ff01:0:0:1::/96", + "gateway": "fd20:80:ff01::1" + } + ] + } + }, + { + "type": "galactic-bgp", + "vpc": "81", + "vpcattachment": "81", + "namespace": "galactic-system" + } + ] + } diff --git a/deploy/containerlab/resources/tenants/ns80/policy/base/bindings.yaml b/deploy/containerlab/resources/tenants/ns80/policy/base/bindings.yaml new file mode 100644 index 00000000..c51bb6ab --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80/policy/base/bindings.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: galactic.datumapis.com/v1alpha1 +kind: TrafficPolicyBinding +metadata: + name: vpc-a +spec: + ingressPolicyRef: + name: lab-ingress + egressPolicyRef: + name: lab-egress +--- +apiVersion: galactic.datumapis.com/v1alpha1 +kind: TrafficPolicyBinding +metadata: + name: vpc-b +spec: + ingressPolicyRef: + name: lab-ingress + egressPolicyRef: + name: lab-egress diff --git a/deploy/containerlab/resources/tenants/ns80/policy/base/kustomization.yaml b/deploy/containerlab/resources/tenants/ns80/policy/base/kustomization.yaml new file mode 100644 index 00000000..6db9e7f6 --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80/policy/base/kustomization.yaml @@ -0,0 +1,7 @@ +# ns80's traffic policy: the private-nat set (../../../policies/README.md), +# bound to both VPCs' attachments in both directions. A binding is named after +# its VPCAttachment, which scripts/publish-attachments.sh creates. +namespace: ns80 +resources: + - ../../../policies/private-nat + - bindings.yaml diff --git a/deploy/containerlab/resources/tenants/ns80/policy/kustomization.yaml b/deploy/containerlab/resources/tenants/ns80/policy/kustomization.yaml new file mode 100644 index 00000000..ad82ec47 --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80/policy/kustomization.yaml @@ -0,0 +1,3 @@ +# ns80's traffic policy at every site: both VPCs' attachments (base/). +resources: + - base diff --git a/deploy/containerlab/resources/tenants/ns80/sjc/kustomization.yaml b/deploy/containerlab/resources/tenants/ns80/sjc/kustomization.yaml new file mode 100644 index 00000000..ee81d25b --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80/sjc/kustomization.yaml @@ -0,0 +1,3 @@ +resources: + - ../base + - nad.yaml diff --git a/deploy/containerlab/resources/tenants/ns80/sjc/nad.yaml b/deploy/containerlab/resources/tenants/ns80/sjc/nad.yaml new file mode 100644 index 00000000..350526a8 --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns80/sjc/nad.yaml @@ -0,0 +1,71 @@ +# sjc: both VPCs' guests get the same address, fd20:80:ff02::1:0:0. +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: vpc-a + namespace: ns80 +spec: + config: |- + { + "cniVersion": "1.0.0", + "name": "vpc-a", + "plugins": [ + { + "type": "galactic-veth", + "vpc": "80", + "vpcattachment": "80", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "addresses": [ + { + "address": "fd20:80:ff02:0:0:1::/96", + "gateway": "fd20:80:ff02::1" + } + ] + } + }, + { + "type": "galactic-bgp", + "vpc": "80", + "vpcattachment": "80", + "namespace": "galactic-system" + } + ] + } +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: vpc-b + namespace: ns80 +spec: + config: |- + { + "cniVersion": "1.0.0", + "name": "vpc-b", + "plugins": [ + { + "type": "galactic-veth", + "vpc": "81", + "vpcattachment": "81", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "addresses": [ + { + "address": "fd20:80:ff02:0:0:1::/96", + "gateway": "fd20:80:ff02::1" + } + ] + } + }, + { + "type": "galactic-bgp", + "vpc": "81", + "vpcattachment": "81", + "namespace": "galactic-system" + } + ] + } diff --git a/deploy/containerlab/scripts/baseline-lib.sh b/deploy/containerlab/scripts/baseline-lib.sh new file mode 100755 index 00000000..8b654142 --- /dev/null +++ b/deploy/containerlab/scripts/baseline-lib.sh @@ -0,0 +1,131 @@ +#!/bin/bash +# baseline-lib.sh: capture and result helpers for baseline-observe.sh. +# Sourced, not executed. Needs lib.sh and LAB, OUT already set. +# +# A capture point is one or more interfaces read together, because a flow's +# bond member or ECMP next hop can change between runs while the link it +# crosses does not. Each interface streams a pcap back over `docker exec` or +# `kubectl exec`, so nothing is left on a node. + +# Capture targets: +# node:CONTAINER a lab container (Kind node, transit router, remote-host) +# pod:SITE/NAMESPACE/POD a pod's own network namespace +# netns:SITE/NAMESPACE/POD/NETNS a named network namespace inside POD's mount namespace +target_exec() { + local target="$1" + shift + local kind=${target%%:*} rest=${target#*:} + case "${kind}" in + node) docker exec -i "${rest}" "$@" ;; + pod) + IFS=/ read -r site ns pod <<<"${rest}" + docker exec -i "$(control_plane "${site}")" kubectl exec -i -n "${ns}" "${pod}" -- "$@" + ;; + netns) + IFS=/ read -r site ns pod netns <<<"${rest}" + docker exec -i "$(control_plane "${site}")" kubectl exec -i -n "${ns}" "${pod}" -- ip netns exec "${netns}" "$@" + ;; + *) + echo "target_exec: unknown target ${target}" >&2 + return 2 + ;; + esac +} + +# The capture filter on every interface. BGP, BFD and neighbour discovery +# run on every link all the time and say nothing about the paths under test. +CAPTURE_FILTER='not (tcp port 179 or udp port 3784 or udp port 4784 or (icmp6 and ip6[40] >= 133 and ip6[40] <= 137))' + +declare -ga CAP_PIDS=() CAP_STOPS=() + +# cap_start SCENARIO POINT TARGET IFACE [IFACE...] +# Starts one tcpdump per interface. Their pcaps land in +# ${OUT}/pcap/SCENARIO/POINT.IFACE.pcap. +cap_start() { + local scenario="$1" point="$2" target="$3" + shift 3 + local dir="${OUT}/pcap/${scenario}" iface file + mkdir -p "${dir}" + for iface in "$@"; do + file="${dir}/${point}.${iface}.pcap" + target_exec "${target}" timeout 300 tcpdump -U -n -i "${iface}" -w - "${CAPTURE_FILTER}" \ + >"${file}" 2>"${file}.log" & + CAP_PIDS+=($!) + CAP_STOPS+=("${target}|${iface}|${file}") + done +} + +# cap_ready waits until every running capture reports it is listening, so +# no probe is sent before its first capture point is armed. +cap_ready() { + local entry file i + for entry in "${CAP_STOPS[@]}"; do + file=${entry##*|} + for i in $(seq 1 50); do + grep -q 'listening on' "${file}.log" 2>/dev/null && break + if [ "${i}" -eq 50 ]; then + echo "capture never started: ${file}" >&2 + cat "${file}.log" >&2 || true + return 1 + fi + sleep 0.2 + done + done +} + +# cap_stop_all stops every capture started since the last call, waits for +# each to flush and exit, and clears the list. +cap_stop_all() { + local entry target iface pid + sleep 1 + for entry in "${CAP_STOPS[@]}"; do + IFS='|' read -r target iface _ <<<"${entry}" + target_exec "${target}" pkill -INT -f "tcpdump -U -n -i ${iface} -w -" >/dev/null 2>&1 || true + done + for pid in "${CAP_PIDS[@]}"; do + wait "${pid}" 2>/dev/null || true + done + CAP_PIDS=() + CAP_STOPS=() +} + +# hits SCENARIO POINT REGEX counts decoded packets at POINT whose tcpdump +# line matches REGEX (grep -E), summed across POINT's interfaces. Matching +# the decoded text sees inside SRv6 encapsulation, which tcpdump prints as +# "IP6 outer > outer: IP6 inner > inner: ...". +hits() { + local scenario="$1" point="$2" regex="$3" f n total=0 + for f in "${OUT}/pcap/${scenario}/${point}".*.pcap; do + [ -s "${f}" ] || continue + n=$(tcpdump -nn -r "${f}" 2>/dev/null | grep -cE -- "${regex}" || true) + total=$((total + n)) + done + echo "${total}" +} + +# record SCENARIO CHECK EXPECT GOT appends one stable result row. EXPECT is +# a number, "yes"/"no", or "-" for a recorded observation with no +# expectation. Only stable values may be recorded here; anything that +# changes run to run (ports, nonces, timestamps, MACs) goes to volatile. +# FAILED is read by baseline-observe.sh, which sources this file. +# shellcheck disable=SC2034 +FAILED=0 +# shellcheck disable=SC2034 +record() { + local scenario="$1" check="$2" expect="$3" got="$4" ok=pass + if [ "${expect}" != "-" ] && [ "${expect}" != "${got}" ]; then + ok=fail + FAILED=1 + fi + [ "${expect}" = "-" ] && ok=observed + printf '%s\t%s\t%s\t%s\t%s\n' "${scenario}" "${check}" "${expect}" "${got}" "${ok}" >>"${OUT}/results.tsv" + printf ' %-8s %-14s %-58s want %-4s got %s\n' "${ok}" "${scenario}" "${check}" "${expect}" "${got}" +} + +# seen N prints "yes" when N > 0. +seen() { [ "$1" -gt 0 ] && echo yes || echo no; } + +# volatile KEY VALUE records a run-specific detail outside results.json. +volatile() { + printf '%s\t%s\n' "$1" "$2" >>"${OUT}/volatile.tsv" +} diff --git a/deploy/containerlab/scripts/baseline-observe.sh b/deploy/containerlab/scripts/baseline-observe.sh new file mode 100755 index 00000000..d3fe9251 --- /dev/null +++ b/deploy/containerlab/scripts/baseline-observe.sh @@ -0,0 +1,325 @@ +#!/bin/bash +# baseline-observe.sh: observe today's remote paths on the dedicated-placement +# fixture (resources/tenants/ns80, docs/baseline.md) in both directions, with +# packet captures at every capture point, and write the results. +# +# Usage: baseline-observe.sh OUTDIR +# +# Scenarios, each in its own capture window: +# +# cross-node-a/-b ICMPv6 between the dfw and sjc guests of vpc-a, then vpc-b +# nat66, nat64 both dfw guests fetch from remote-host at once, with one +# identical source port, through dfw's egress shards +# gateway-ipv6/-ipv4 +# remote-host to dfw's VIPs through dfw-worker2, answered +# by ns60's backends with direct server return +# isolation UDP to the overlapping address from each VPC's sjc guest: +# only the same VPC's dfw guest may receive it +# +# OUTDIR gets results.json (stable fields only, compared between runs), +# volatile.tsv (ports, nonces and other per-run values), pcap// +# and logs/. Exits non-zero when any expectation fails. +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +source "${SCRIPT_DIR}/lib.sh" + +LAB=${LAB:-gvpc} +OUT=${1:?usage: baseline-observe.sh OUTDIR} +mkdir -p "${OUT}/logs" +OUT=$(cd "${OUT}" && pwd) +: >"${OUT}/results.tsv" +: >"${OUT}/volatile.tsv" +source "${SCRIPT_DIR}/baseline-lib.sh" +trap cap_stop_all EXIT + +NS=ns80 +DFW_CP=$(control_plane dfw) +SJC_CP=$(control_plane sjc) +DFW_ADDR=fd20:80:ff01::1:0:0 +SJC_ADDR=fd20:80:ff02::1:0:0 +CLIENT6=2001:db8:1:40::2 +CLIENT4=11.1.40.2 +# remote-host's IPv4 address as each side writes it in IPv6: under the lab +# shards' NAT64 prefix for egress (resources/galactic-nat/), and under +# 64:ff9b::/96 as a gateway backend sees an IPv4 client. +EGRESS_NAT64=2001:db8:64::b01:2802 +BACKEND_SEES4=64:ff9b::b01:2802 +PINGS=5 +PROBES=5 + +# Capture points shared by several scenarios. Each sums every member of the +# link it names (see baseline-lib.sh). +TR1="node:clab-${LAB}-tr1" +TR2="node:clab-${LAB}-tr2" +REMOTE="node:clab-${LAB}-remote-host" +DFW_COMPUTE_IFS=(eth1 eth3 eth2 eth4) # bonds to dfw-worker2 and dfw-worker3 +SJC_COMPUTE_IFS=(eth1 eth2) # bond to sjc-worker2 +DFW_EDGE_UPLINKS=(eth1 eth6 eth5 eth7) # tr1's members facing dfw-worker2 and dfw-worker3 +SJC_EDGE_UPLINKS=(eth1 eth5) # tr2's members facing sjc-worker2 + +pod_of() { pod_name "$1" "${NS}" "app=$2"; } +host_if() { printf 'G%09s%03sH' "$1" "$2" | tr ' ' '0'; } + +echo "=== fixture ===" +declare -A POD +for site in dfw sjc; do + cp=$(control_plane "${site}") + for app in vpc-a vpc-b; do + pod=$(pod_of "${cp}" "${app}") + if [ -z "${pod}" ]; then + echo "no ${NS} ${app} pod in ${site}: run task deploy:ns80" >&2 + exit 1 + fi + POD[${site}-${app}]=${pod} + record fixture "${site} ${app} guest on compute worker" "${site}-worker" "$(pod_scheduling_node "${cp}" "${NS}" "${pod}")" + done + a=$(pod_ip6 "${cp}" "${NS}" "${POD[${site}-vpc-a]}") + b=$(pod_ip6 "${cp}" "${NS}" "${POD[${site}-vpc-b]}") + want=${DFW_ADDR} + [ "${site}" = sjc ] && want=${SJC_ADDR} + record fixture "${site} vpc-a guest address" "${want}" "${a}" + record fixture "${site} vpc-b guest address equals vpc-a's" "${a}" "${b}" +done + +# --------------------------------------------------------------------------- +# Cross-node workload delivery, per VPC: dfw guest <-> sjc guest. +cross_node() { + local vpc="$1" id="$2" sc="cross-node-${1#vpc-}" + local dfw_pod=${POD[dfw-${vpc}]} sjc_pod=${POD[sjc-${vpc}]} + echo "=== ${sc} ===" + # One unrecorded echo each way first, so neighbour resolution and route + # warm-up never land inside the counted window. + docker exec "${DFW_CP}" kubectl exec -n "${NS}" "${dfw_pod}" -- ping -6 -c 1 -W 2 "${SJC_ADDR}" >/dev/null 2>&1 || true + docker exec "${SJC_CP}" kubectl exec -n "${NS}" "${sjc_pod}" -- ping -6 -c 1 -W 2 "${DFW_ADDR}" >/dev/null 2>&1 || true + + cap_start "${sc}" dfw-guest "pod:dfw/${NS}/${dfw_pod}" eth0 + cap_start "${sc}" dfw-veth-host "node:dfw-worker" "$(host_if "${id}" "${id}")" + cap_start "${sc}" dfw-compute-underlay "node:dfw-worker" "${DFW_COMPUTE_IFS[@]}" + cap_start "${sc}" dfw-edge-uplink "${TR1}" "${DFW_EDGE_UPLINKS[@]}" + cap_start "${sc}" sjc-edge-uplink "${TR2}" "${SJC_EDGE_UPLINKS[@]}" + cap_start "${sc}" sjc-compute-underlay "node:sjc-worker" "${SJC_COMPUTE_IFS[@]}" + cap_start "${sc}" sjc-veth-host "node:sjc-worker" "$(host_if "${id}" "${id}")" + cap_start "${sc}" sjc-guest "pod:sjc/${NS}/${sjc_pod}" eth0 + cap_ready + + local out_ab out_ba + out_ab=$(docker exec "${DFW_CP}" kubectl exec -n "${NS}" "${dfw_pod}" -- \ + ping -6 -c "${PINGS}" -i 0.2 -W 2 "${SJC_ADDR}" 2>&1 || true) + out_ba=$(docker exec "${SJC_CP}" kubectl exec -n "${NS}" "${sjc_pod}" -- \ + ping -6 -c "${PINGS}" -i 0.2 -W 2 "${DFW_ADDR}" 2>&1 || true) + cap_stop_all + printf '%s\n%s\n' "${out_ab}" "${out_ba}" >"${OUT}/logs/${sc}.ping.log" + + record "${sc}" "dfw->sjc echo replies" "${PINGS}" "$(sed -nE 's/.* ([0-9]+) (packets )?received.*/\1/p' <<<"${out_ab}")" + record "${sc}" "sjc->dfw echo replies" "${PINGS}" "$(sed -nE 's/.* ([0-9]+) (packets )?received.*/\1/p' <<<"${out_ba}")" + + local fwd="${DFW_ADDR} > ${SJC_ADDR}: ICMP6, echo request" + local rev="${SJC_ADDR} > ${DFW_ADDR}: ICMP6, echo request" + local point + # Each direction's requests, at every point on the path. Points marked + # "-" are recorded but carry no expectation (docs/baseline.md). + for point in dfw-guest sjc-guest dfw-edge-uplink sjc-edge-uplink; do + record "${sc}" "${point} dfw->sjc requests" "${PINGS}" "$(hits "${sc}" "${point}" "${fwd}")" + record "${sc}" "${point} sjc->dfw requests" "${PINGS}" "$(hits "${sc}" "${point}" "${rev}")" + done + # A host veth carries its guest's outbound packets. Inbound ones arrive by + # bpf_redirect_peer straight into the guest's side of the pair + # (internal/plumbing/ebpf/prog/usid.c), which tcpdump on the host side + # never sees, so those are recorded without an expectation. + record "${sc}" "dfw-veth-host dfw->sjc requests" "${PINGS}" "$(hits "${sc}" dfw-veth-host "${fwd}")" + record "${sc}" "sjc-veth-host sjc->dfw requests" "${PINGS}" "$(hits "${sc}" sjc-veth-host "${rev}")" + record "${sc}" "dfw-veth-host sjc->dfw requests (peer redirect)" - "$(hits "${sc}" dfw-veth-host "${rev}")" + record "${sc}" "sjc-veth-host dfw->sjc requests (peer redirect)" - "$(hits "${sc}" sjc-veth-host "${fwd}")" + for point in dfw-compute-underlay sjc-compute-underlay; do + record "${sc}" "${point} dfw->sjc requests" - "$(hits "${sc}" "${point}" "${fwd}")" + record "${sc}" "${point} sjc->dfw requests" - "$(hits "${sc}" "${point}" "${rev}")" + done +} +cross_node vpc-a 80 +cross_node vpc-b 81 + +# --------------------------------------------------------------------------- +# NAT66 and NAT64 egress: both dfw guests, one identical source port and +# destination, at once. The shard must give them distinct translated tuples +# and return each reply to the guest that sent it. +SHARDS6=$(docker exec "${DFW_CP}" kubectl -n galactic-system get egressshard -o jsonpath='{.items[*].status.shardAddressIPv6}') +SHARDS4=$(docker exec "${DFW_CP}" kubectl -n galactic-system get egressshard -o jsonpath='{.items[*].status.shardAddressIPv4}') +volatile shards "${SHARDS6} ${SHARDS4}" + +nat_scenario() { + local sc="$1" url_host="$2" seen_dst="$3" shards="$4" + local port=$((40000 + RANDOM % 20000)) + volatile "${sc}.guest_source_port" "${port}" + echo "=== ${sc} ===" + local dfw_a=${POD[dfw-vpc-a]} dfw_b=${POD[dfw-vpc-b]} + cap_start "${sc}" dfw-guest-a "pod:dfw/${NS}/${dfw_a}" eth0 + cap_start "${sc}" dfw-guest-b "pod:dfw/${NS}/${dfw_b}" eth0 + cap_start "${sc}" dfw-veth-host-a "node:dfw-worker" "$(host_if 80 80)" + cap_start "${sc}" dfw-veth-host-b "node:dfw-worker" "$(host_if 81 81)" + cap_start "${sc}" dfw-compute-underlay "node:dfw-worker" "${DFW_COMPUTE_IFS[@]}" + cap_start "${sc}" public "${TR1}" "${DFW_EDGE_UPLINKS[@]}" + cap_start "${sc}" peer "${REMOTE}" eth1 + cap_ready + + local fetch="curl -sS -o /dev/null -w %{http_code} --max-time 5 --local-port ${port} http://[${url_host}]/" + docker exec "${DFW_CP}" kubectl exec -n "${NS}" "${dfw_a}" -- sh -c "${fetch}" >"${OUT}/logs/${sc}.a.code" 2>&1 & + local pa=$! + docker exec "${DFW_CP}" kubectl exec -n "${NS}" "${dfw_b}" -- sh -c "${fetch}" >"${OUT}/logs/${sc}.b.code" 2>&1 & + local pb=$! + wait "${pa}" || true + wait "${pb}" || true + cap_stop_all + + record "${sc}" "vpc-a fetch status" 200 "$(tail -c 3 "${OUT}/logs/${sc}.a.code")" + record "${sc}" "vpc-b fetch status" 200 "$(tail -c 3 "${OUT}/logs/${sc}.b.code")" + + local g=${DFW_ADDR} + local out_re="${g}\.${port} > ${url_host}\.80: Flags \[S\]" + local back_re="${url_host}\.80 > ${g}\.${port}: Flags \[S\.\]" + local p + for p in a b; do + record "${sc}" "dfw-guest-${p} SYN out" yes "$(seen "$(hits "${sc}" "dfw-guest-${p}" "${out_re}")")" + record "${sc}" "dfw-guest-${p} SYN-ACK back" yes "$(seen "$(hits "${sc}" "dfw-guest-${p}" "${back_re}")")" + record "${sc}" "dfw-veth-host-${p} SYN out" yes "$(seen "$(hits "${sc}" "dfw-veth-host-${p}" "${out_re}")")" + record "${sc}" "dfw-veth-host-${p} SYN-ACK back (peer redirect)" - "$(seen "$(hits "${sc}" "dfw-veth-host-${p}" "${back_re}")")" + done + record "${sc}" "dfw-compute-underlay SYN out" - "$(seen "$(hits "${sc}" dfw-compute-underlay "${out_re}")")" + record "${sc}" "dfw-compute-underlay SYN-ACK back" - "$(seen "$(hits "${sc}" dfw-compute-underlay "${back_re}")")" + + # On the public side the guest address is gone: each SYN to the peer + # carries a shard's masquerade address and a port the shard chose. + local syn_re="> ${seen_dst}\.80: Flags \[S\]," synack_re="${seen_dst}\.80 > .*: Flags \[S\.\]," + record "${sc}" "public SYN out" yes "$(seen "$(hits "${sc}" public "${syn_re}")")" + record "${sc}" "public SYN-ACK back" yes "$(seen "$(hits "${sc}" public "${synack_re}")")" + local tuples + tuples=$(for f in "${OUT}/pcap/${sc}"/peer.*.pcap; do tcpdump -nn -r "${f}" 2>/dev/null; done | + { grep -E "${syn_re}" || true; } | sed -nE 's/^.* IP6? ([^ ]+) > .*/\1/p' | sort -u) + volatile "${sc}.translated_tuples" "$(paste -sd' ' <<<"${tuples}")" + record "${sc}" "peer distinct translated tuples" 2 "$(grep -c . <<<"${tuples}" || true)" + local leaked=no in_shards=yes t + for t in ${tuples}; do + [[ "${t}" == "${g}".* ]] && leaked=yes + [[ " ${shards} " == *" ${t%.*} "* ]] || in_shards=no + done + record "${sc}" "peer sees guest address" no "${leaked}" + record "${sc}" "peer sources are shard addresses" yes "${in_shards}" + record "${sc}" "peer SYN-ACK sent" yes "$(seen "$(hits "${sc}" peer "${synack_re}")")" +} +nat_scenario nat66 "${CLIENT6}" "${CLIENT6}" "${SHARDS6}" +nat_scenario nat64 "${EGRESS_NAT64}" "${CLIENT4}" "${SHARDS4}" + +# --------------------------------------------------------------------------- +# Gateway ingress over IPv6 and IPv4, through dfw-worker2. The lab's own +# checks drive the flows and assert replies, backend identity and DSR; the +# captures add what each point saw. +DFW_BACKEND_IFS=$(docker exec "${DFW_CP}" kubectl get vpcattachment -n ns60 -o json | + jq -r '.items[] | select(.status.node == "dfw-worker") | .status.hostInterface' | sort | paste -sd' ') +mapfile -t DFW_BACKENDS < <(docker exec "${DFW_CP}" kubectl get pods -n ns60 -l app=backend -o json | + jq -r '.items[] | select(.spec.nodeName == "dfw-worker") | .metadata.name' | sort) +# backend_hits SCENARIO REGEX sums REGEX over every dfw backend's eth0. +backend_hits() { + local i total=0 + for i in "${!DFW_BACKENDS[@]}"; do + total=$((total + $(hits "$1" "backend-guest-${i}" "$2"))) + done + echo "${total}" +} +gateway_scenario() { + local sc="$1" script="$2" vip="$3" client="$4" backend_client="$5" peer_capture="$6" + echo "=== ${sc} ===" + # verify-gateway-ingress-ipv4.sh captures on remote-host itself and stops + # every tcpdump there, ours included, so its own client-side checks stand + # in for the peer capture point. + [ "${peer_capture}" = yes ] && cap_start "${sc}" peer "${REMOTE}" eth1 + cap_start "${sc}" public "${TR1}" "${DFW_EDGE_UPLINKS[@]}" + cap_start "${sc}" dfw-compute-underlay "node:dfw-worker" "${DFW_COMPUTE_IFS[@]}" + # shellcheck disable=SC2086 # one interface per backend attachment + cap_start "${sc}" backend-veth-host "node:dfw-worker" ${DFW_BACKEND_IFS} + local i + for i in "${!DFW_BACKENDS[@]}"; do + cap_start "${sc}" "backend-guest-${i}" "pod:dfw/ns60/${DFW_BACKENDS[${i}]}" eth0 + done + cap_ready + local rc=0 + LAB=${LAB} "${SCRIPT_DIR}/${script}" --node dfw-worker2 >"${OUT}/logs/${sc}.log" 2>&1 || rc=$? + cap_stop_all + # The lab's ingress checks intermittently lose a TCP flow whose request + # reached a backend: the ns60 responder closes without writing a reply + # (docs/baseline.md). So the exact flow counts and the check's exit status + # are per-run values; the stable result is that each protocol had answered + # flows, which shows delivery in both directions. + volatile "${sc}.check_exit" "${rc}" + local proto answered + for proto in tcp udp; do + answered=$(sed -nE "s/^ ${proto}: ([0-9]+)\/[0-9]+ flows answered.*/\1/p" "${OUT}/logs/${sc}.log" | head -1) + volatile "${sc}.${proto}_flows_answered" "${answered:-0}/8" + record "${sc}" "${proto} flows answered by a backend" yes "$(seen "${answered:-0}")" + done + local in_re="${client}\.[0-9]+ > ${vip}\.80:" out_re="${vip}\.80 > ${client}\.[0-9]+:" + if [ "${peer_capture}" = yes ]; then + record "${sc}" "peer requests out" yes "$(seen "$(hits "${sc}" peer "${in_re}")")" + record "${sc}" "peer replies back" yes "$(seen "$(hits "${sc}" peer "${out_re}")")" + fi + record "${sc}" "public requests in" yes "$(seen "$(hits "${sc}" public "${in_re}")")" + record "${sc}" "public replies out" yes "$(seen "$(hits "${sc}" public "${out_re}")")" + record "${sc}" "backend-guest requests from client" yes \ + "$(seen "$(backend_hits "${sc}" "${backend_client}\.[0-9]+ > ")")" + record "${sc}" "backend-guest replies to client" yes \ + "$(seen "$(backend_hits "${sc}" "> ${backend_client}\.[0-9]+:")")" + record "${sc}" "backend-veth-host replies to client" yes \ + "$(seen "$(hits "${sc}" backend-veth-host "> ${backend_client}\.[0-9]+:")")" + record "${sc}" "backend-veth-host requests from client (peer redirect)" - \ + "$(seen "$(hits "${sc}" backend-veth-host "${backend_client}\.[0-9]+ > ")")" + record "${sc}" "dfw-compute-underlay requests" - "$(seen "$(hits "${sc}" dfw-compute-underlay "${in_re}")")" +} +gateway_scenario gateway-ipv6 verify-gateway-ingress.sh 2001:db8:6060:1::1 "${CLIENT6}" "${CLIENT6}" yes +gateway_scenario gateway-ipv4 verify-gateway-ingress-ipv4.sh 198.51.100.1 "${CLIENT4}" "${BACKEND_SEES4}" no + +# --------------------------------------------------------------------------- +# Negative isolation control. Each VPC's sjc guest sends PROBES datagrams to +# the address both dfw guests hold. Only the dfw guest in the sender's VPC +# may see them; the other VPC's guest, and its host veth, must see none. +echo "=== isolation ===" +sc=isolation +iport=$((20000 + RANDOM % 10000)) +nonce=$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') +volatile isolation.udp_port "${iport}" +volatile isolation.nonce "${nonce}" +cap_start "${sc}" dfw-guest-a "pod:dfw/${NS}/${POD[dfw-vpc-a]}" eth0 +cap_start "${sc}" dfw-guest-b "pod:dfw/${NS}/${POD[dfw-vpc-b]}" eth0 +cap_start "${sc}" dfw-veth-host-a "node:dfw-worker" "$(host_if 80 80)" +cap_start "${sc}" dfw-veth-host-b "node:dfw-worker" "$(host_if 81 81)" +cap_ready +# A different source port per VPC, so each one's probes are told apart at +# the receiving end. +declare -A SPORT=([a]=$((iport + 1)) [b]=$((iport + 2))) +for from in a b; do + for i in $(seq 1 "${PROBES}"); do + echo "${nonce}-${from}-${i}" | docker exec -i "${SJC_CP}" kubectl exec -i -n "${NS}" "${POD[sjc-vpc-${from}]}" -- \ + socat -u - "UDP6-SENDTO:[${DFW_ADDR}]:${iport},sourceport=${SPORT[${from}]},reuseaddr" || true + sleep 0.1 + done +done +cap_stop_all +for to in a b; do + for from in a b; do + sp=${SPORT[${from}]} + want=0 + [ "${from}" = "${to}" ] && want=${PROBES} + re="${SJC_ADDR}\.${sp} > ${DFW_ADDR}\.${iport}: UDP" + record "${sc}" "dfw-guest-${to} receives from vpc-${from}" "${want}" "$(hits "${sc}" "dfw-guest-${to}" "${re}")" + # Inbound, so delivered by peer redirect past the host veth: recorded + # only. The guest's own eth0 is the isolation evidence. + record "${sc}" "dfw-veth-host-${to} carries from vpc-${from} (peer redirect)" - "$(hits "${sc}" "dfw-veth-host-${to}" "${re}")" + done +done + +# --------------------------------------------------------------------------- +jq -R -s 'split("\n") | map(select(length > 0) | split("\t") + | {scenario: .[0], check: .[1], expect: .[2], got: .[3], result: .[4]}) + | sort_by(.scenario, .check)' "${OUT}/results.tsv" >"${OUT}/results.json" +total=$(jq length "${OUT}/results.json") +failed=$(jq '[.[] | select(.result == "fail")] | length' "${OUT}/results.json") +echo "=== ${total} checks, ${failed} failed; results in ${OUT}/results.json ===" +exit "${FAILED}" diff --git a/deploy/containerlab/scripts/baseline-twice.sh b/deploy/containerlab/scripts/baseline-twice.sh new file mode 100755 index 00000000..93e99e3e --- /dev/null +++ b/deploy/containerlab/scripts/baseline-twice.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# baseline-twice.sh: record the pinned versions, run baseline-observe.sh twice +# in a row, and require both runs to pass with identical results.json files +# that also match the committed baseline/reference-results.json. +# +# Usage: baseline-twice.sh [OUTDIR] +# OUTDIR defaults to bin/baseline/ at the repository root. +# +# Set BASELINE_WRITE_REFERENCE=1 to replace the reference with run-1's +# results once both runs agree, after a deliberate change to the fixture or +# its expectations. +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +LAB_DIR=$(cd "${SCRIPT_DIR}/.." && pwd) +REPO=$(cd "${LAB_DIR}/../.." && pwd) +REFERENCE="${LAB_DIR}/baseline/reference-results.json" +OUT=${1:-${REPO}/bin/baseline/$(date -u +%Y%m%dT%H%M%SZ)} +mkdir -p "${OUT}" + +rc=0 +"${SCRIPT_DIR}/baseline-versions.sh" "${OUT}/versions.yaml" || rc=1 +for run in 1 2; do + echo "##### run ${run} #####" + "${SCRIPT_DIR}/baseline-observe.sh" "${OUT}/run-${run}" 2>&1 | tee "${OUT}/run-${run}.log" || + rc=1 +done + +if diff -u "${OUT}/run-1/results.json" "${OUT}/run-2/results.json" >"${OUT}/runs.diff"; then + echo "runs 1 and 2: identical results ($(sha256sum <"${OUT}/run-1/results.json" | cut -c1-16))" +else + echo "FAIL: runs 1 and 2 differ:" >&2 + cat "${OUT}/runs.diff" >&2 + rc=1 +fi + +if [ "${rc}" -eq 0 ] && [ "${BASELINE_WRITE_REFERENCE:-0}" = 1 ]; then + cp "${OUT}/run-1/results.json" "${REFERENCE}" + echo "wrote ${REFERENCE}" +elif [ ! -f "${REFERENCE}" ]; then + echo "FAIL: no ${REFERENCE}; rerun with BASELINE_WRITE_REFERENCE=1 to create it" >&2 + rc=1 +elif diff -u "${REFERENCE}" "${OUT}/run-1/results.json" >"${OUT}/reference.diff"; then + echo "run 1 matches baseline/reference-results.json" +else + echo "FAIL: run 1 differs from baseline/reference-results.json:" >&2 + cat "${OUT}/reference.diff" >&2 + rc=1 +fi + +echo "evidence in ${OUT}" +exit "${rc}" diff --git a/deploy/containerlab/scripts/baseline-versions.sh b/deploy/containerlab/scripts/baseline-versions.sh new file mode 100755 index 00000000..c917e7b5 --- /dev/null +++ b/deploy/containerlab/scripts/baseline-versions.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# baseline-versions.sh: check the pinned versions in baseline/pins.yaml +# against the host tools, the lab's Dockerfiles, the ns80 fixture and the +# running lab, and record every version the baseline depends on. +# +# Usage: baseline-versions.sh OUTFILE +# +# OUTFILE gets a YAML record: each pin with its expected and observed value, +# then the unpinned values (kernel, Docker, Galactic revision, image IDs and +# digests). Exits non-zero if any pin disagrees. +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +source "${SCRIPT_DIR}/lib.sh" + +LAB=${LAB:-gvpc} +OUTFILE=${1:?usage: baseline-versions.sh OUTFILE} +LAB_DIR=$(cd "${SCRIPT_DIR}/.." && pwd) +REPO=$(cd "${LAB_DIR}/../.." && pwd) +PINS="${LAB_DIR}/baseline/pins.yaml" + +pin() { awk -v k="$1" -F': ' '$1 == k {print $2; exit}' "${PINS}"; } +dockerfile_arg() { sed -nE "s/^ARG $2=(.*)/\1/p" "$1" | head -1; } + +mkdir -p "$(dirname "${OUTFILE}")" +rc=0 +{ + echo "# Recorded by scripts/baseline-versions.sh" + echo "pins:" +} >"${OUTFILE}" + +# check KEY SOURCE OBSERVED +check() { + local want + want=$(pin "$1") + local ok=match + if [ "${want}" != "$3" ]; then + ok=MISMATCH + rc=1 + fi + printf ' - {pin: %s, source: "%s", want: "%s", got: "%s", result: %s}\n' "$1" "$2" "${want}" "$3" "${ok}" >>"${OUTFILE}" + printf '%-9s %-17s %-40s want %s got %s\n' "${ok}" "$1" "$2" "${want}" "$3" +} + +check containerlab "containerlab version" "$(containerlab version 2>/dev/null | sed -nE 's/^ *version: *//p' | head -1)" +check kind "kind version" "$(kind version | awk '{print $2}')" +check task "task --version" "$(task --version | sed -E 's/^Task version: //; s/ .*//')" +check kindest_node "kindest-node-galactic Dockerfile" \ + "$(dockerfile_arg "${LAB_DIR}/containers/kindest-node-galactic/Dockerfile" KINDEST_VER)" +check kindest_node "dfw API server" \ + "$(docker exec "$(control_plane dfw)" kubectl version -o json | jq -r .serverVersion.gitVersion)" +check frr "fabric-router Dockerfile" "$(dockerfile_arg "${REPO}/containers/fabric-router/Dockerfile" FRR_VERSION)" +check frr "tr1 vtysh" \ + "$(docker exec "clab-${LAB}-tr1" vtysh -c 'show version' | sed -nE 's/^FRRouting ([0-9.]+).*/\1/p' | head -1)" +check remote_host_base "remote-host Dockerfile" "$(sed -nE 's/^FROM (.*)/\1/p' "${REPO}/containers/remote-host/Dockerfile" | head -1)" +for f in base/guests.yaml; do + for img in $(sed -nE 's/^ *image: *//p' "${LAB_DIR}/resources/tenants/ns80/${f}" | sort -u); do + check guest_image "ns80/${f}" "${img}" + done +done +# What the running guests were actually started from. +for site in dfw sjc; do + got=$(docker exec "$(control_plane "${site}")" kubectl get pods -n ns80 -o json | + jq -r '[.items[].status.containerStatuses[]?.imageID] | unique | join(",")') + check guest_image "ns80 pods in ${site}" "ghcr.io/datum-cloud/galactic-debug@${got##*@}" +done + +{ + echo "recorded:" + echo " host_kernel: \"$(uname -r)\"" + echo " docker: \"$(docker version -f '{{.Server.Version}}')\"" + echo " kubectl_client: \"$(kubectl version --client -o json | jq -r .clientVersion.gitVersion)\"" + echo " node_kernel: \"$(docker exec dfw-worker uname -r)\"" + echo " galactic_revision: \"$(git -C "${REPO}" rev-parse HEAD)\"" + echo " galactic_tree: \"$(git -C "${REPO}" diff --quiet HEAD && echo clean || echo dirty)\"" + echo " local_images:" + docker images --no-trunc --format '{{.Repository}}:{{.Tag}} {{.ID}}' | + grep -E '^(kindest/node:galactic|fabric-router:|remote-host:|galactic-[a-z-]+:|fabric-[a-z-]+:|masque-[a-z-]+:)' | + sort | awk '{printf " - {image: \"%s\", id: \"%s\"}\n", $1, $2}' + echo " dfw_worker_images:" + docker exec dfw-worker crictl images -o json | + jq -r '.images[] | select(any(.repoTags[]?; test("galactic|fabric|masque"))) | "\(.repoTags[0]) \(.id)"' | + sort | awk '{printf " - {image: \"%s\", id: \"%s\"}\n", $1, $2}' +} >>"${OUTFILE}" + +echo "versions recorded in ${OUTFILE}" +exit "${rc}" diff --git a/deploy/containerlab/scripts/verify-ns80.sh b/deploy/containerlab/scripts/verify-ns80.sh new file mode 100755 index 00000000..51f5e4bd --- /dev/null +++ b/deploy/containerlab/scripts/verify-ns80.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# verify-ns80.sh: check the dedicated-placement fixture (resources/tenants/ns80) +# is in place and forwarding. Each VPC's guests sit on dfw's and sjc's compute +# workers, both VPCs' guests hold the same address at each site, and each VPC +# reaches its own guest across sites in both directions. The full +# observations, with captures and the isolation control, are +# task baseline:observe. +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +source "${SCRIPT_DIR}/lib.sh" + +NS=ns80 +DFW=$(control_plane dfw) +SJC=$(control_plane sjc) +declare -A WANT=([dfw]=fd20:80:ff01::1:0:0 [sjc]=fd20:80:ff02::1:0:0) + +declare -A POD +for site in dfw sjc; do + cp=$(control_plane "${site}") + for app in vpc-a vpc-b; do + pod=$(pod_name "${cp}" "${NS}" "app=${app}") + if [ -z "${pod}" ]; then + echo "ns80 ${app} pod not found in ${site}" >&2 + exit 1 + fi + node=$(pod_scheduling_node "${cp}" "${NS}" "${pod}") + if [ "${node}" != "${site}-worker" ]; then + echo "ns80 ${app} in ${site} runs on ${node}, want ${site}-worker" >&2 + exit 1 + fi + addr=$(pod_ip6 "${cp}" "${NS}" "${pod}") + if [ "${addr}" != "${WANT[${site}]}" ]; then + echo "ns80 ${app} in ${site} has address '${addr}', want ${WANT[${site}]}" >&2 + exit 1 + fi + POD[${site}-${app}]=${pod} + echo "${site} ${app}: ${pod} on ${node}, ${addr}" + done +done + +for app in vpc-a vpc-b; do + echo "--- ${app}: dfw -> sjc ---" + ping_pod "${DFW}" "${NS}" "${POD[dfw-${app}]}" -6 "${WANT[sjc]}" + echo "--- ${app}: sjc -> dfw ---" + ping_pod "${SJC}" "${NS}" "${POD[sjc-${app}]}" -6 "${WANT[dfw]}" +done + +echo "ns80: both VPCs reach across sites in both directions"