From 1777e631316064e386479b52f9527d611c95edec Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Wed, 18 Nov 2026 18:37:54 -0500 Subject: [PATCH 1/5] feat(ownership): Move the fence and add an atomic Takeover Move internal/edge/ownership to internal/ownership. CNI, the installer, galactic-router and Worker now all write through it, not only galactic-edge. DefaultDir keeps its path, so no node state moves. Add Takeover. It advances a scope's generation and names the new owner under one lock, and the returned ticket requires a snapshot commit. Worker previously ran Fence and then Adopt as two calls. A Worker that died between them left the scope transitioning, and every Exclusive writer was refused until Worker came back. CNI ADD writes shared VPC state with Exclusive, so workload attach failed while Worker was down. Worker now uses Takeover both to take over a scope from an earlier incarnation and to fence a scope under repair when a claim is released. docs/worker records this as D23. Takeover also completes a transfer that an earlier Fence left open. Add ExclusiveIf, an Exclusive write whose check reads the scope's record under the lock, so a writer can refuse a scope by its owner with no race. Add WorkerOwnerPrefix and WorkerOwned. Fix tap-mode CNI DEL so it no longer removes a successor's state. DEL removed the ifindex-keyed map rows and the router advertisement record by host interface name before tap.Delete checked the owner stamp. A predecessor's delayed DEL therefore stripped the rows of the TAP a replacement container had taken over. tap.Release now checks the stamp, removes the rows and deletes the tap as one sequence. DEL removes the router advertisement record only when this container still owns the tap. The veth path already checked ownership first. --- cmd/galactic-edge/main.go | 2 +- cmd/galactic-router/root.go | 2 +- cmd/galactic-worker/root.go | 2 +- config/galactic-router/base/daemonset.yaml | 2 +- docs/cni/gc-cmd-sequence.md | 2 +- docs/edge/provider-contract.md | 8 +- docs/worker/README.md | 3 +- docs/worker/contract.md | 18 +-- docs/worker/ownership.md | 40 ++--- internal/cni/tap/tap.go | 27 ++++ internal/cni/tap/tap_test.go | 52 +++++++ internal/cnitap/ops_del.go | 65 +++------ internal/edge/agent/agent.go | 4 +- internal/edge/agent/agent_test.go | 2 +- internal/edge/agent/fixture_test.go | 2 +- internal/edge/agent/gate.go | 2 +- internal/edge/agent/gate_test.go | 2 +- internal/edge/agent/maintenance_test.go | 2 +- internal/edge/agent/replacement_test.go | 2 +- internal/edge/maintenance/maintenance.go | 4 +- internal/edge/maintenance/maintenance_test.go | 2 +- internal/{edge => }/ownership/ownership.go | 42 +++++- .../ownership/ownership_netns_test.go | 0 .../{edge => }/ownership/ownership_test.go | 0 internal/ownership/takeover_test.go | 138 ++++++++++++++++++ internal/worker/claims.go | 12 +- internal/worker/harness_test.go | 2 +- internal/worker/module/module.go | 4 +- internal/worker/reconcile.go | 37 ++--- internal/worker/recovery_test.go | 5 +- internal/worker/worker.go | 14 +- 31 files changed, 362 insertions(+), 137 deletions(-) rename internal/{edge => }/ownership/ownership.go (89%) rename internal/{edge => }/ownership/ownership_netns_test.go (100%) rename internal/{edge => }/ownership/ownership_test.go (100%) create mode 100644 internal/ownership/takeover_test.go diff --git a/cmd/galactic-edge/main.go b/cmd/galactic-edge/main.go index d8b4612e5..2528030a1 100644 --- a/cmd/galactic-edge/main.go +++ b/cmd/galactic-edge/main.go @@ -42,12 +42,12 @@ import ( "go.datum.net/galactic/internal/edge/agent" "go.datum.net/galactic/internal/edge/diag" "go.datum.net/galactic/internal/edge/hostattach" - "go.datum.net/galactic/internal/edge/ownership" "go.datum.net/galactic/internal/edge/route" "go.datum.net/galactic/internal/edge/timing" "go.datum.net/galactic/internal/logging" "go.datum.net/galactic/internal/metadata" "go.datum.net/galactic/internal/nodevpc" + "go.datum.net/galactic/internal/ownership" "go.datum.net/galactic/internal/plumbing/ebpf/attach" "go.datum.net/galactic/internal/plumbing/sysctl" ) diff --git a/cmd/galactic-router/root.go b/cmd/galactic-router/root.go index a10fb479b..c29182395 100644 --- a/cmd/galactic-router/root.go +++ b/cmd/galactic-router/root.go @@ -33,11 +33,11 @@ import ( "go.datum.net/galactic/internal/config" "go.datum.net/galactic/internal/controller" "go.datum.net/galactic/internal/edge/maintenance" - "go.datum.net/galactic/internal/edge/ownership" "go.datum.net/galactic/internal/gc" "go.datum.net/galactic/internal/hash" "go.datum.net/galactic/internal/logging" "go.datum.net/galactic/internal/metadata" + "go.datum.net/galactic/internal/ownership" "go.datum.net/galactic/internal/plumbing/ebpf/attach" "go.datum.net/galactic/internal/plumbing/ebpf/vipxlatmap" "go.datum.net/galactic/internal/plumbing/loaddr" diff --git a/cmd/galactic-worker/root.go b/cmd/galactic-worker/root.go index 4c74babba..29e8c28eb 100644 --- a/cmd/galactic-worker/root.go +++ b/cmd/galactic-worker/root.go @@ -27,9 +27,9 @@ import ( workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" "go.datum.net/galactic/internal/config" - "go.datum.net/galactic/internal/edge/ownership" "go.datum.net/galactic/internal/logging" "go.datum.net/galactic/internal/metadata" + "go.datum.net/galactic/internal/ownership" "go.datum.net/galactic/internal/worker" "go.datum.net/galactic/internal/worker/auth" "go.datum.net/galactic/internal/worker/module" diff --git a/config/galactic-router/base/daemonset.yaml b/config/galactic-router/base/daemonset.yaml index 4706d9201..05f53f8de 100644 --- a/config/galactic-router/base/daemonset.yaml +++ b/config/galactic-router/base/daemonset.yaml @@ -165,7 +165,7 @@ spec: - name: cni-state mountPath: /var/lib/cni # The node's write fence, shared with galactic-edge - # (internal/edge/ownership). GC deletes a VPC's VRF and + # (internal/ownership). GC deletes a VPC's VRF and # BGPVRFInstance only under that VPC's fence, after checking no # edge attachment uses it. Without this mount GC reclaims # neither. diff --git a/docs/cni/gc-cmd-sequence.md b/docs/cni/gc-cmd-sequence.md index ae6203834..cc8a40616 100644 --- a/docs/cni/gc-cmd-sequence.md +++ b/docs/cni/gc-cmd-sequence.md @@ -286,7 +286,7 @@ sequenceDiagram counts a VPC as used while an `EdgeAttachment` on this node is not `Released`, read uncached, or a host link stamped `galactic-edge:hostattach:` for it remains. Each delete runs inside - `Reclaim`, which takes the VPC's write fence (`internal/edge/ownership`, + `Reclaim`, which takes the VPC's write fence (`internal/ownership`, `/var/lib/galactic-edge/ownership`) that `galactic-edge` holds while it programs, checks again and deletes only if nothing uses the VPC. An attachment admitted after collection therefore keeps the VRF, and a change diff --git a/docs/edge/provider-contract.md b/docs/edge/provider-contract.md index d977b7b82..cdfb0a402 100644 --- a/docs/edge/provider-contract.md +++ b/docs/edge/provider-contract.md @@ -26,7 +26,7 @@ This document is the contract the [Galactic Edge foundation](../designs/galactic | Node packaging | A new `galactic-edge` DaemonSet on nodes labeled `galactic.datumapis.com/node=edge` and `galactic.datumapis.com/edge=enabled`. The `edge=enabled` label is new, and [node labels](../operations/node-labels.md) gains it when the DaemonSet ships. `galactic-edge` is the only process that changes host state for an edge attachment. It writes only the CNI datapath's maps under `/sys/fs/bpf/galactic`, and never opens `/sys/fs/bpf/galactic-edge`, which is `galactic-gateway`'s pin directory despite the name (`internal/plumbing/ebpf/edgeattach/attach.go:38`). | | Provider privilege | None on the host. A provider pod gets no host namespace, hostPath, `NET_ADMIN` or BPF access. It opens the packet device the node agent created in its own network namespace. | | Claim store | The cell's API server, compare-and-swap on `resourceVersion` with a UID precondition, as `internal/edge/claim` requires. Claims never span cells. | -| Node-local write fence | `internal/edge/ownership`, with its state and lock files in the hostPath `/var/lib/galactic-edge/ownership`, shared by every process that writes a fenced scope on the node. | +| Node-local write fence | `internal/ownership`, with its state and lock files in the hostPath `/var/lib/galactic-edge/ownership`, shared by every process that writes a fenced scope on the node. | | Forwarding gate | A new map of the CNI datapath, with one row per edge host link and per accepted prefix holding a `CLOCK_BOOTTIME` deadline. The datapath compares it with `bpf_ktime_get_boot_ns` on every packet, the pattern `serviceroutemap` already uses for authorization deadlines. | | Allocation | Attachment addresses come from the service's client pool, allocated by the leader of `galactic-controller`. Delegated client prefixes are routes the provider requests inside that pool; the prefix conflict check makes them exclusive. An edge attachment joins the VPC's canonical host VRF and allocates no host table or uSID Argument of its own. | | Node VPC state | When an attachment is its gVPC's only user on a node, `galactic-edge` creates the gVPC's canonical host VRF, the node's `BGPVRFInstance` for it and its `vrf_table` row, as CNI ADD does, through `nodevpc.Ensure` under the same node locks. `galactic-router` GC stays the only remover: it counts edge attachments as users and deletes the state once nothing on the node uses the gVPC. The CNI, the ingress sidecar and GC keep their roles, so no second reference count exists (#1288). | @@ -124,7 +124,7 @@ A ValidatingAdmissionPolicy reads only its own parameter object, so the provider - Every create uses a derived name. `AlreadyExists` is answered by reading the object and comparing its immutable fields; a mismatch is refused with `IdentityConflict`, never overwritten. - Claim writes are compare-and-swap. A lost reply is recovered as `claim.Instance.Renew` does: the writer rereads, and validity always runs from the start of the latest call. -- Node mutations go through `ownership.Authority.Commit` (`internal/edge/ownership/ownership.go:162`), which refuses a ticket issued before another commit. A retry built from an old view cannot undo newer state. +- Node mutations go through `ownership.Authority.Commit` (`internal/ownership/ownership.go:162`), which refuses a ticket issued before another commit. A retry built from an old view cannot undo newer state. ### Readiness @@ -300,7 +300,7 @@ A drain stops new routes and attachments for its scope at once. Forwarding conti | Provider process restarts in its pod | New holder ID. It takes over its own instance lease after a release or `TakeoverWait`, then core re-accepts its routes at once (see [Acceptance](#acceptance)). | | Provider pod replaced, same node | New pod UID and `EdgeInstance`. Routes move by takeover. The old attachment drains. | | Replacement on another node | As above. The old node's gate closes at its own deadline even if that node cannot reach the API. | -| `galactic-edge` restarts | New ownership owner ID, `Fence`, `Adopt`, then a snapshot commit (`internal/edge/ownership/ownership.go:212`, `:226`, `:169`). It extends no deadline until it has read a renewal newer than the one applied. | +| `galactic-edge` restarts | New ownership owner ID, `Fence`, `Adopt`, then a snapshot commit (`internal/ownership/ownership.go:212`, `:226`, `:169`). It extends no deadline until it has read a renewal newer than the one applied. | | `galactic-router` restarts | Advertises an edge prefix only while its gate deadline, read from the pinned map, is in the future. | | Node reboots | Pinned maps and leases from the previous boot are void: the boot ID changed. Every claim on that node is reacquired through the takeover rules. | @@ -352,7 +352,7 @@ The revocation bound works like this. `galactic-edge` extends a gate only after ## Foundation resources and their writers -Foundation-owned host state is what `galactic-edge` creates for an attachment, plus the shared state it writes into. Each gVPC on the node is one scope of `internal/edge/ownership`, which `galactic-edge` owns and writes only in snapshot commits. A writer that can remove shared state an edge attachment uses does so only through `maintenance.Guard.Reclaim` (`internal/edge/maintenance`): under the scope's lock (`ownership.Authority.Exclusive`), and never between `Fence` and `Adopt`, it checks again that no edge attachment uses the gVPC; a refused check writes nothing, and a write makes every ticket `galactic-edge` holds for the scope stale, so a change prepared before it is prepared again. A gVPC is in use while the API has an `EdgeAttachment` on the node for it that is not `Released`, which covers one still being prepared, or the host has a link stamped for it. Every other writer in the table below either writes only rows an edge attachment never uses, or only adds rows; the table gives the reason for each. No lab or end-to-end environment runs `galactic-edge` yet. +Foundation-owned host state is what `galactic-edge` creates for an attachment, plus the shared state it writes into. Each gVPC on the node is one scope of `internal/ownership`, which `galactic-edge` owns and writes only in snapshot commits. A writer that can remove shared state an edge attachment uses does so only through `maintenance.Guard.Reclaim` (`internal/edge/maintenance`): under the scope's lock (`ownership.Authority.Exclusive`), and never between `Fence` and `Adopt`, it checks again that no edge attachment uses the gVPC; a refused check writes nothing, and a write makes every ticket `galactic-edge` holds for the scope stale, so a change prepared before it is prepared again. A gVPC is in use while the API has an `EdgeAttachment` on the node for it that is not `Released`, which covers one still being prepared, or the host has a link stamped for it. Every other writer in the table below either writes only rows an edge attachment never uses, or only adds rows; the table gives the reason for each. No lab or end-to-end environment runs `galactic-edge` yet. ### Owned by `galactic-edge` diff --git a/docs/worker/README.md b/docs/worker/README.md index f5d1dcbb6..8221f42da 100644 --- a/docs/worker/README.md +++ b/docs/worker/README.md @@ -42,7 +42,7 @@ PR. "Confirmed" means the product owner approved the decision on the date given. | D10 | Trusted sources for locator, VPC, owner, backend SID and policy | Accepted | [Contract](contract.md#trusted-sources) | | D11 | Own locator resolves to Local, any other to Fabric, with no fallback from Local to Fabric | Proposed | [Contract](contract.md#resolution) | | D12 | One mutation authority means owning the scope's fence, or being a class's only writer; CNI writes with `Exclusive` | Accepted | [Ownership](ownership.md#what-one-mutation-authority-means) | -| D13 | Bootstrap, adoption and transfer use the `internal/edge/ownership` fence, generalized | Accepted | [Ownership](ownership.md#bootstrap-adoption-and-transfer) | +| D13 | Bootstrap, adoption and transfer use the `internal/ownership` fence, generalized | Accepted | [Ownership](ownership.md#bootstrap-adoption-and-transfer) | | D14 | The connector is `cmd/galactic-edge` with a Worker-backed `agent.Datapath` | Accepted | [Connector](connector.md) | | D15 | Branch workflow with start commit `451eb40` | Accepted | [Branch workflow](branch-workflow.md) | | D16 | NIC: Broadcom NetXtreme-C/E (`bnxt_en`) in generic mode; native mode on lab veth devices | Confirmed 2026-11-18 | [Qualification](qualification.md#platform) | @@ -52,3 +52,4 @@ PR. "Confirmed" means the product owner approved the decision on the date given. | D20 | Shard and backend destinations report `DestinationKindUnsupported` until #1437 and #1438 add their resolvers inside Worker | Accepted | [Contract](contract.md#claims) | | D21 | The peer check maps `SO_PEERCRED` to a pod by the cgroup path, so Worker needs `hostPID` and the host `/proc` | Accepted | [Contract](contract.md#transport-identity-and-access) | | D22 | The allowlist is a YAML file passed with `--allowlist` | Accepted | [Contract](contract.md#transport-identity-and-access) | +| D23 | Worker takes a scope over with `Takeover`, which fences and adopts under one lock, so no crash leaves a scope refusing CNI writes | Accepted | [Ownership](ownership.md#bootstrap-adoption-and-transfer) | diff --git a/docs/worker/contract.md b/docs/worker/contract.md index e5a51b669..1d4ca3ee8 100644 --- a/docs/worker/contract.md +++ b/docs/worker/contract.md @@ -210,8 +210,8 @@ qualified for detecting a dead service agent (D5). `AuthorizationRevoked` and withdraws the claim the same way ([#1440](https://github.com/datum-cloud/galactic/issues/1440) tests revocation). -- **Worker restart.** Worker reads desired state from disk, then for each scope - fences, adopts and reconciles a full snapshot, as [ownership](ownership.md#bootstrap-adoption-and-transfer) +- **Worker restart.** Worker reads desired state from disk, then takes each + scope over and reconciles a full snapshot, as [ownership](ownership.md#bootstrap-adoption-and-transfer) describes. Programmed and observed state are rebuilt by reading the host. Forwarding continues while Worker is down, because the datapath does not call Worker. @@ -253,10 +253,10 @@ D8 and D9 were confirmed on 2026-11-18. Each case names the decision it exercises and the Task that owns its test. -| Case | Start | Event | Worker action | Result | Decision | Test owner | -| -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Duplicate request | Claim at generation 4, `Ready=True`. | The client's response was lost, so it sends `Request` again with generation 4 and the same spec. | Compares generation and spec hash with desired state. Takes no ticket and writes nothing. | `OK` with the current status. Host state is byte-for-byte unchanged. The same generation with a different spec gets `FAILED_PRECONDITION`/`GenerationConflict`. | D1, D3 | [#1433](https://github.com/datum-cloud/galactic/issues/1433) unit test: duplicate and out-of-order requests | -| Stale release | Claim updated from generation 4 to 5, programming in progress. | A delayed `Release` with generation 4 arrives. | Compares the generation with desired state and refuses before taking a ticket. | `ABORTED`/`StaleGeneration`, carrying current generation 5. The claim and host state are unchanged, and programming of generation 5 continues. | D3, D4 | [#1433](https://github.com/datum-cloud/galactic/issues/1433) unit test: release from a stale generation and from a non-owner | -| Unauthorized cross-VPC claim | The caller has a claim on an attachment in VPC A. VPC B uses the same guest addresses. | `Update` adds a destination whose attachment is in VPC B. | Resolves the destination's VPC from its attachment record and its identity from NetworkFabricIdentity, finds B ≠ A, and refuses. Records nothing durable, takes no ticket, logs the refusal and counts it in a metric. | `PERMISSION_DENIED`/`VPCNotAuthorized`. The existing generation keeps running. Routes, rules, links and pinned maps are unchanged. | D1, D10 | [#1433](https://github.com/datum-cloud/galactic/issues/1433) zero-mutation snapshot test; [#1442](https://github.com/datum-cloud/galactic/issues/1442) observes the unauthorized destination in the lab | -| Local locator with a missing TAP | Claim admitted; one destination is a guest on this node. | Resolution finds the node's own locator, but the guest's TAP does not exist (not yet attached, or being replaced). | Resolves the destination to Local and does not fall back to Fabric. Programs the other destinations. Leaves the missing one unprogrammed and watches CNI attachment events. When CNI ADD creates the TAP, programs it on the new ifindex and reads it back. | Until the TAP appears: `Resolved=True`, `Programmed=False` and `Ready=False`, with reason `HandoffInterfaceMissing` naming the destination. Afterwards: `Ready=True`. | D2, D11 | [#1435](https://github.com/datum-cloud/galactic/issues/1435) root test for Local programming; [#1434](https://github.com/datum-cloud/galactic/issues/1434) for CNI attachment observation | -| Withdrawal during repair | A commit failed part-way, so the VPC scope is resync-required and Worker's repair holds a ticket while it prepares a snapshot. | The owner sends `Release` with the current generation. | Records the release durably and answers. Fences the scope, which invalidates the repair's ticket, then adopts it and commits a snapshot computed from desired state without the released claim. The repair's late commit fails with a transitioning or stale error and is dropped. | `Release` returns `OK`. Status shows `Withdrawing` with scope "one claim" until the read-back confirms removal, then `Observe` reports the claim released. No row of the released claim survives the snapshot. | D3, D12, D13 | [#1433](https://github.com/datum-cloud/galactic/issues/1433) crash and interleaving tests; [#1436](https://github.com/datum-cloud/galactic/issues/1436) for Fabric rows | +| Case | Start | Event | Worker action | Result | Decision | Test owner | +| -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Duplicate request | Claim at generation 4, `Ready=True`. | The client's response was lost, so it sends `Request` again with generation 4 and the same spec. | Compares generation and spec hash with desired state. Takes no ticket and writes nothing. | `OK` with the current status. Host state is byte-for-byte unchanged. The same generation with a different spec gets `FAILED_PRECONDITION`/`GenerationConflict`. | D1, D3 | [#1433](https://github.com/datum-cloud/galactic/issues/1433) unit test: duplicate and out-of-order requests | +| Stale release | Claim updated from generation 4 to 5, programming in progress. | A delayed `Release` with generation 4 arrives. | Compares the generation with desired state and refuses before taking a ticket. | `ABORTED`/`StaleGeneration`, carrying current generation 5. The claim and host state are unchanged, and programming of generation 5 continues. | D3, D4 | [#1433](https://github.com/datum-cloud/galactic/issues/1433) unit test: release from a stale generation and from a non-owner | +| Unauthorized cross-VPC claim | The caller has a claim on an attachment in VPC A. VPC B uses the same guest addresses. | `Update` adds a destination whose attachment is in VPC B. | Resolves the destination's VPC from its attachment record and its identity from NetworkFabricIdentity, finds B ≠ A, and refuses. Records nothing durable, takes no ticket, logs the refusal and counts it in a metric. | `PERMISSION_DENIED`/`VPCNotAuthorized`. The existing generation keeps running. Routes, rules, links and pinned maps are unchanged. | D1, D10 | [#1433](https://github.com/datum-cloud/galactic/issues/1433) zero-mutation snapshot test; [#1442](https://github.com/datum-cloud/galactic/issues/1442) observes the unauthorized destination in the lab | +| Local locator with a missing TAP | Claim admitted; one destination is a guest on this node. | Resolution finds the node's own locator, but the guest's TAP does not exist (not yet attached, or being replaced). | Resolves the destination to Local and does not fall back to Fabric. Programs the other destinations. Leaves the missing one unprogrammed and watches CNI attachment events. When CNI ADD creates the TAP, programs it on the new ifindex and reads it back. | Until the TAP appears: `Resolved=True`, `Programmed=False` and `Ready=False`, with reason `HandoffInterfaceMissing` naming the destination. Afterwards: `Ready=True`. | D2, D11 | [#1435](https://github.com/datum-cloud/galactic/issues/1435) root test for Local programming; [#1434](https://github.com/datum-cloud/galactic/issues/1434) for CNI attachment observation | +| Withdrawal during repair | A commit failed part-way, so the VPC scope is resync-required and Worker's repair holds a ticket while it prepares a snapshot. | The owner sends `Release` with the current generation. | Records the release durably and answers. Takes the scope over again with `Takeover`, which invalidates the repair's ticket, and commits a snapshot computed from desired state without the released claim. The repair's late commit fails with a transitioning or stale error and is dropped. | `Release` returns `OK`. Status shows `Withdrawing` with scope "one claim" until the read-back confirms removal, then `Observe` reports the claim released. No row of the released claim survives the snapshot. | D3, D12, D23 | [#1433](https://github.com/datum-cloud/galactic/issues/1433) crash and interleaving tests; [#1436](https://github.com/datum-cloud/galactic/issues/1436) for Fabric rows | diff --git a/docs/worker/ownership.md b/docs/worker/ownership.md index d45379f26..0ef2c5b04 100644 --- a/docs/worker/ownership.md +++ b/docs/worker/ownership.md @@ -19,11 +19,11 @@ workload while Worker is down. CNI never owns a scope. It writes with `Exclusive` whether or not Worker runs, and Worker adopts that state when it starts. A write that bypasses the fence is a defect. -The fence is `internal/edge/ownership`, generalized from its current VPC scope -(`internal/edge/ownership/ownership.go:150-317`). Its state directory must -outlive every writer, as the package documentation requires. Worker uses the -package in place, with its `VPCScope` names and its shared `DefaultDir`; -relocating it is part of [#1434](https://github.com/datum-cloud/galactic/issues/1434). +The fence is `internal/ownership`, generalized from the VPC scope Galactic +Edge introduced. Its state directory must outlive every writer, as the package +documentation requires. Every writer uses the package's `VPCScope` names and its +shared `DefaultDir`, `/var/lib/galactic-edge/ownership`, which the galactic-router +and galactic-cni DaemonSets mount from the host. A resource class has one of two kinds of authority: @@ -66,20 +66,24 @@ Each step uses the fence operations that exist today (D13). 1. **Bootstrap with Worker down.** CNI ADD writes the VPC scope with `Exclusive`, which needs no owner. Workload attach never waits for Worker. - `Exclusive` returns `ErrTransitioning` only between `Fence` and `Adopt`, and - CNI retries within its own ADD deadline. + `Exclusive` returns `ErrTransitioning` only while a `Fence` waits for its + `Adopt`, and CNI retries within its own ADD deadline. Worker never leaves a + scope in that state: it takes scopes over with `Takeover` (D23), which + fences and adopts under one lock, so a Worker that dies while taking a + scope over cannot block CNI until it returns. 2. **Adoption of preexisting state.** When Worker starts, it takes each scope it - finds (from `vrf_table`, the VRF devices and `tables.json`). It uses `Claim` - for an unowned scope, or `Fence` then `Adopt` for one an earlier Worker - incarnation owned. Both return a ticket that requires `CommitSnapshot`, which - Worker runs with its complete desired state read from disk, merged with the - host's current rows. Until that snapshot commits, Worker refuses claim - changes for the scope with `UNAVAILABLE`/`Resyncing`. Each Worker + finds (from the VRF devices and the table reservations in `tables.json`). + It uses `Claim` for an unowned scope, or `Takeover` for one an earlier + Worker incarnation owned, including one an older incarnation left between + `Fence` and `Adopt`. Both return a ticket that requires `CommitSnapshot`, + which Worker runs with its complete desired state read from disk, merged + with the host's current rows. Until that snapshot commits, Worker refuses + claim changes for the scope with `UNAVAILABLE`/`Resyncing`. Each Worker incarnation's owner ID starts with `galactic-worker:`, which is how a new - incarnation recognizes a scope an earlier one owned. Worker never fences a - scope another writer owns: its claims there report `Programmed=False` with - reason `ScopeOwnedElsewhere` until that writer's transfer step (3) hands the - scope over. + incarnation recognizes a scope an earlier one owned. Worker never takes + over a scope another writer owns: its claims there report + `Programmed=False` with reason `ScopeOwnedElsewhere` until that writer's + transfer step (3) hands the scope over. 3. **Ownership transfer.** A legacy writer moves to Worker in two steps. First it writes with `Exclusive` on the shared fence instead of writing directly. Then Worker adopts the scope, and the legacy writer calls Worker instead of @@ -89,7 +93,7 @@ Each step uses the fence operations that exist today (D13). a full snapshot before accepting incremental changes, so a half-applied change is neither assumed done nor assumed undone. 5. **Loss of the state directory.** Stop every writer, reconcile host state, and - create a new authority, as `internal/edge/ownership` requires. Never restore + create a new authority, as `internal/ownership` requires. Never restore the directory while writers run. ## Defects this transfer removes diff --git a/internal/cni/tap/tap.go b/internal/cni/tap/tap.go index cc0d3381d..d17ed5304 100644 --- a/internal/cni/tap/tap.go +++ b/internal/cni/tap/tap.go @@ -273,6 +273,33 @@ func Delete(vpc, vpcAttachment, ownerID string) error { return nil } +// Release tears down ownerID's host tap for the attachment. While the tap +// carries ownerID's stamp, it passes the tap's ifindex to unregister, which +// removes the map rows keyed on it, and then deletes the tap. A tap another +// container's ADD has taken over is left with its rows, and owned is false; +// the caller must then also leave the attachment's other per-interface state +// alone. A missing tap is owned, so cleanup keyed by name still runs. +// +// An unregister failure is logged and does not stop the deletion. +func Release(vpc, vpcAttachment, ownerID string, unregister func(ifindex uint32) error) (owned bool, err error) { + tapName := intf.GenerateInterfaceNameHost(vpc, vpcAttachment) + tapLink, err := netlink.LinkByName(tapName) + switch { + case err != nil && !isLinkNotFoundError(err): + return false, fmt.Errorf("look up tap %q: %w", tapName, err) + case err == nil && !OwnedBy(tapLink, ownerID): + slog.Info("tap: leaving host tap and its state alone, another container has taken this attachment over", + "tap", tapName, "owner", tapLink.Attrs().Alias, "requestedBy", ownerID) + return false, nil + case err == nil: + if err := unregister(uint32(tapLink.Attrs().Index)); err != nil { + slog.Warn("tap: failed to remove the tap's eBPF interface state", "err", err, + "tap", tapName, "owner", ownerID) + } + } + return true, Delete(vpc, vpcAttachment, ownerID) +} + // ensureMTU sets the tap's MTU to mtu, leaving the kernel default in place // when none is configured. // diff --git a/internal/cni/tap/tap_test.go b/internal/cni/tap/tap_test.go index 22271dc30..70dc0b6bc 100644 --- a/internal/cni/tap/tap_test.go +++ b/internal/cni/tap/tap_test.go @@ -450,3 +450,55 @@ func findLink(t *testing.T, name string) (netlink.Link, error) { } return link, nil } + +// TestReleaseLeavesSuccessorTap replays a predecessor container's DEL after a +// successor's ADD took the tap over: the successor's tap and the rows keyed on +// its ifindex stay. The owner's own DEL then removes both, and a repeat finds +// nothing to unregister. +func TestReleaseLeavesSuccessorTap(t *testing.T) { + requireRoot(t) + + vpc, att := "rl", "c1" + addVRFStandIn(t, vpc) + t.Cleanup(func() { _ = Delete(vpc, att, "successor") }) + if _, err := Add(vpc, att, "successor", 1500); err != nil { + t.Fatalf("Add = %v", err) + } + link, err := findLink(t, intf.GenerateInterfaceNameHost(vpc, att)) + if err != nil { + t.Fatal(err) + } + var unregistered []uint32 + unregister := func(ifindex uint32) error { + unregistered = append(unregistered, ifindex) + return nil + } + + owned, err := Release(vpc, att, "predecessor", unregister) + if err != nil || owned { + t.Fatalf("stale Release = (%t, %v), want (false, nil)", owned, err) + } + if len(unregistered) != 0 { + t.Fatalf("stale Release unregistered ifindexes %v", unregistered) + } + if _, err := findLink(t, link.Attrs().Name); err != nil { + t.Fatalf("stale Release removed the successor's tap: %v", err) + } + + owned, err = Release(vpc, att, "successor", unregister) + if err != nil || !owned { + t.Fatalf("owner Release = (%t, %v), want (true, nil)", owned, err) + } + if len(unregistered) != 1 || unregistered[0] != uint32(link.Attrs().Index) { + t.Fatalf("owner Release unregistered %v, want [%d]", unregistered, link.Attrs().Index) + } + if _, err := netlink.LinkByName(link.Attrs().Name); err == nil { + t.Fatal("owner Release left the tap") + } + + owned, err = Release(vpc, att, "successor", unregister) + if err != nil || !owned || len(unregistered) != 1 { + t.Fatalf("repeated Release = (%t, %v) with %v unregistered, want (true, nil) and no new call", + owned, err, unregistered) + } +} diff --git a/internal/cnitap/ops_del.go b/internal/cnitap/ops_del.go index fd3c6e220..77aae6951 100644 --- a/internal/cnitap/ops_del.go +++ b/internal/cnitap/ops_del.go @@ -11,7 +11,6 @@ import ( "github.com/containernetworking/cni/pkg/types" type100 "github.com/containernetworking/cni/pkg/types/100" "github.com/containernetworking/plugins/pkg/ipam" - "github.com/vishvananda/netlink" "go.datum.net/galactic/internal/cni/tap" "go.datum.net/galactic/internal/plumbing/dan" @@ -45,18 +44,29 @@ func cmdDel(args *skel.CmdArgs) error { } } - // Unregister this attachment's ifindex-keyed map rows. Private to this - // attachment's own ifindex, so it belongs alongside the device deletion - // below rather than the shared cleanup deferred to GC. Resolved and removed - // before the interface is torn down, since there is nothing to resolve an - // ifindex from afterward. - unregisterIfindexVRFEntry(vpc, vpcAtt, args.ContainerID) + // Remove this attachment's ifindex-keyed map rows and its tap, but only + // while the tap still belongs to this container: the host name is keyed + // by (vpc, vpcAttachment) alone, so a replacement container's ADD can + // already have taken this same device over by the time this DEL, for a + // predecessor still terminating, arrives. The rows are removed before the + // tap, since there is nothing to resolve its ifindex from afterward. A VMM + // still holding the device's descriptor open can make the kernel delete + // lazily rather than immediately, but never blocks or fails this call. + owned, err := tap.Release(vpc, vpcAtt, args.ContainerID, func(ifindex uint32) error { + return attachreg.UnregisterInterface(attach.PinDir, ifindex) + }) + if err != nil { + slog.Warn("DEL: failed to delete tap device", "err", err, + "containerID", args.ContainerID, "vpc", vpc, "vpcAttachment", vpcAtt) + } - // Stop the node daemon resending Router Advertisements for this attachment. - // Best-effort and unconditional: a missing record, from an attachment that - // never had an IPv6 gateway allocated, is not an error, and DEL must stay - // idempotent. - removeRadvState(vpc, vpcAtt, args.ContainerID) + // Stop the node daemon resending Router Advertisements for this + // attachment, unless a successor owns the tap and its advertisements. + // Best-effort: a missing record, from an attachment that never had an IPv6 + // gateway allocated, is not an error, and DEL must stay idempotent. + if owned { + removeRadvState(vpc, vpcAtt, args.ContainerID) + } // Remove the sandbox's DAN file. Nothing in Kata deletes it, so files would // accumulate in a tmpfs directory for the node's uptime. @@ -69,21 +79,6 @@ func cmdDel(args *skel.CmdArgs) error { "containerID", args.ContainerID, "dir", cniConfig.DANDir) } - // Delete this attachment's tap device, but only while it still belongs to - // this container: the host name is keyed by (vpc, vpcAttachment) alone, so - // a replacement container's ADD can already have taken this same device - // over by the time this DEL, for a predecessor still terminating, arrives. - // tap.Delete checks the ownership stamp Add leaves and leaves a - // successor's tap alone rather than tearing it down by name. - // - // A VMM still holding the device's descriptor open can make the kernel - // delete lazily rather than immediately, but never blocks or fails this - // call. - if err := tap.Delete(vpc, vpcAtt, args.ContainerID); err != nil { - slog.Warn("DEL: failed to delete tap device", "err", err, - "containerID", args.ContainerID, "vpc", vpc, "vpcAttachment", vpcAtt) - } - // Shared resources, the VRF and the BGP CRDs, are keyed by attachment or by // node and may still be in use by another VM. Deleting them here races // cmdAdd during a restart, so cleanup is left to garbage collection. @@ -96,22 +91,6 @@ func cmdDel(args *skel.CmdArgs) error { return nil } -// unregisterIfindexVRFEntry removes every pinned map row keyed on this -// attachment's host ifindex (see attachreg.UnregisterInterface), mirroring the -// veth plugin's identical helper but resolving a tap device's host-side -// interface rather than a veth pair's. -func unregisterIfindexVRFEntry(vpc, vpcAttachment, containerID string) { - hostName := intf.GenerateInterfaceNameHost(vpc, vpcAttachment) - link, err := netlink.LinkByName(hostName) - if err != nil { - return - } - if err := attachreg.UnregisterInterface(attach.PinDir, uint32(link.Attrs().Index)); err != nil { - slog.Warn("DEL: failed to remove eBPF interface state", "err", err, - "containerID", containerID, "vpc", vpc, "vpcAttachment", vpcAttachment, "hostInterface", hostName) - } -} - // removeRadvState removes this attachment's router-advertisement resend record, // if any. Private to this attachment's own host interface, so it belongs // alongside the device deletion rather than the shared cleanup deferred to diff --git a/internal/edge/agent/agent.go b/internal/edge/agent/agent.go index 2e5c40cbb..a0d5d41f4 100644 --- a/internal/edge/agent/agent.go +++ b/internal/edge/agent/agent.go @@ -16,7 +16,7 @@ // A stamped link is released only under its own ID, so cleanup of an earlier // attachment never touches a replacement's. // -// Each VPC on the node is one ownership scope (internal/edge/ownership). A +// Each VPC on the node is one ownership scope (internal/ownership). A // process takes over the scopes its predecessors held with Fence and Adopt, // and writes each scope only inside a commit, so a write prepared by an // earlier process, or before another writer's commit, never lands. @@ -47,8 +47,8 @@ import ( "go.datum.net/galactic/internal/edge/claim" "go.datum.net/galactic/internal/edge/diag" "go.datum.net/galactic/internal/edge/hostattach" - "go.datum.net/galactic/internal/edge/ownership" "go.datum.net/galactic/internal/edge/timing" + "go.datum.net/galactic/internal/ownership" "go.datum.net/galactic/internal/plumbing/ebpf/attachreg" "go.datum.net/galactic/internal/plumbing/intf" ) diff --git a/internal/edge/agent/agent_test.go b/internal/edge/agent/agent_test.go index 8561b18d4..7e2932200 100644 --- a/internal/edge/agent/agent_test.go +++ b/internal/edge/agent/agent_test.go @@ -21,7 +21,7 @@ import ( galacticv1alpha1 "go.datum.net/galactic/api/v1alpha1" "go.datum.net/galactic/internal/edge/diag" - "go.datum.net/galactic/internal/edge/ownership" + "go.datum.net/galactic/internal/ownership" "go.datum.net/galactic/internal/plumbing/ebpf/forwardgatemap" ) diff --git a/internal/edge/agent/fixture_test.go b/internal/edge/agent/fixture_test.go index a50590dc6..2981c9531 100644 --- a/internal/edge/agent/fixture_test.go +++ b/internal/edge/agent/fixture_test.go @@ -32,7 +32,7 @@ import ( "go.datum.net/galactic/internal/edge/diag" "go.datum.net/galactic/internal/edge/diag/diagtest" "go.datum.net/galactic/internal/edge/hostattach" - "go.datum.net/galactic/internal/edge/ownership" + "go.datum.net/galactic/internal/ownership" "go.datum.net/galactic/internal/plumbing/ebpf/forwardgatemap" "go.datum.net/galactic/internal/plumbing/intf" ) diff --git a/internal/edge/agent/gate.go b/internal/edge/agent/gate.go index ae4657c7f..469cda406 100644 --- a/internal/edge/agent/gate.go +++ b/internal/edge/agent/gate.go @@ -21,9 +21,9 @@ import ( "go.datum.net/galactic/internal/edge/claim" "go.datum.net/galactic/internal/edge/diag" "go.datum.net/galactic/internal/edge/hostattach" - "go.datum.net/galactic/internal/edge/ownership" "go.datum.net/galactic/internal/edge/provider" "go.datum.net/galactic/internal/edge/route" + "go.datum.net/galactic/internal/ownership" "go.datum.net/galactic/internal/plumbing/ebpf/forwardgatemap" ) diff --git a/internal/edge/agent/gate_test.go b/internal/edge/agent/gate_test.go index b24c5068c..23175d332 100644 --- a/internal/edge/agent/gate_test.go +++ b/internal/edge/agent/gate_test.go @@ -21,9 +21,9 @@ import ( "go.datum.net/galactic/internal/edge/claim" "go.datum.net/galactic/internal/edge/diag" "go.datum.net/galactic/internal/edge/diag/diagtest" - "go.datum.net/galactic/internal/edge/ownership" "go.datum.net/galactic/internal/edge/provider" "go.datum.net/galactic/internal/edge/route" + "go.datum.net/galactic/internal/ownership" "go.datum.net/galactic/internal/plumbing/ebpf/forwardgatemap" ) diff --git a/internal/edge/agent/maintenance_test.go b/internal/edge/agent/maintenance_test.go index 7eb2d0553..5c6ad897a 100644 --- a/internal/edge/agent/maintenance_test.go +++ b/internal/edge/agent/maintenance_test.go @@ -15,8 +15,8 @@ import ( galacticv1alpha1 "go.datum.net/galactic/api/v1alpha1" "go.datum.net/galactic/internal/edge/hostattach" "go.datum.net/galactic/internal/edge/maintenance" - "go.datum.net/galactic/internal/edge/ownership" "go.datum.net/galactic/internal/gc" + "go.datum.net/galactic/internal/ownership" "go.datum.net/galactic/internal/plumbing/intf" ) diff --git a/internal/edge/agent/replacement_test.go b/internal/edge/agent/replacement_test.go index 7be2288e6..f336160bf 100644 --- a/internal/edge/agent/replacement_test.go +++ b/internal/edge/agent/replacement_test.go @@ -45,10 +45,10 @@ import ( galacticv1alpha1 "go.datum.net/galactic/api/v1alpha1" "go.datum.net/galactic/internal/edge/claim" "go.datum.net/galactic/internal/edge/hostattach" - "go.datum.net/galactic/internal/edge/ownership" edgeprovider "go.datum.net/galactic/internal/edge/provider" "go.datum.net/galactic/internal/edge/route" "go.datum.net/galactic/internal/edge/timing" + "go.datum.net/galactic/internal/ownership" "go.datum.net/galactic/internal/plumbing/ebpf/attach" "go.datum.net/galactic/internal/plumbing/ebpf/forwardgatemap" "go.datum.net/galactic/internal/plumbing/ebpf/usidmap" diff --git a/internal/edge/maintenance/maintenance.go b/internal/edge/maintenance/maintenance.go index 7a5cae755..b4e2c134b 100644 --- a/internal/edge/maintenance/maintenance.go +++ b/internal/edge/maintenance/maintenance.go @@ -16,7 +16,7 @@ // attachment whose object is gone or unreadable but whose host state is // not yet released. // -// Reclaim decides again under the VPC's write fence (internal/edge/ownership), +// Reclaim decides again under the VPC's write fence (internal/ownership), // which galactic-edge holds while it programs, so a reclaim decided from an // earlier view cannot remove state an attachment programmed since. package maintenance @@ -33,7 +33,7 @@ import ( galacticv1alpha1 "go.datum.net/galactic/api/v1alpha1" "go.datum.net/galactic/internal/edge/hostattach" - "go.datum.net/galactic/internal/edge/ownership" + "go.datum.net/galactic/internal/ownership" ) // ErrInUse is returned by Reclaim when an edge attachment uses the VPC. diff --git a/internal/edge/maintenance/maintenance_test.go b/internal/edge/maintenance/maintenance_test.go index 519c31472..b60c842c7 100644 --- a/internal/edge/maintenance/maintenance_test.go +++ b/internal/edge/maintenance/maintenance_test.go @@ -24,7 +24,7 @@ import ( galacticv1alpha1 "go.datum.net/galactic/api/v1alpha1" "go.datum.net/galactic/internal/edge/hostattach" - "go.datum.net/galactic/internal/edge/ownership" + "go.datum.net/galactic/internal/ownership" ) const node = "edge-1" diff --git a/internal/edge/ownership/ownership.go b/internal/ownership/ownership.go similarity index 89% rename from internal/edge/ownership/ownership.go rename to internal/ownership/ownership.go index a7e2e6c89..69eae449b 100644 --- a/internal/edge/ownership/ownership.go +++ b/internal/ownership/ownership.go @@ -256,11 +256,18 @@ func (a *Authority) commit(t Ticket, snapshot bool, mutate func() error) (Ticket // authority for the same scope, and mutate must finish every write before it // returns. func (a *Authority) Exclusive(scope string, check, mutate func() error) error { + return a.ExclusiveIf(scope, func(Record) error { return check() }, mutate) +} + +// ExclusiveIf is Exclusive with a check that also reads the scope's record as +// it stands under the lock, so a writer can refuse a scope by its owner +// without a window in which the owner changes. +func (a *Authority) ExclusiveIf(scope string, check func(Record) error, mutate func() error) error { return a.lockedStaged(scope, func(r *Record, persist func() error) error { if r.Transitioning { return ErrTransitioning } - if err := check(); err != nil { + if err := check(*r); err != nil { return err } pending := r.Pending @@ -313,6 +320,39 @@ func (a *Authority) Adopt(scope, owner string, generation uint64) (Ticket, error return t, err } +// Takeover moves scope to owner in one step under the scope's lock: it +// advances the generation, so no ticket issued before it can commit, names +// owner, and returns a ticket that requires a snapshot commit. from decides, +// from the record as it stands, whether the current owner may be replaced; +// when it refuses, Takeover returns ErrNotOwner and changes nothing. +// +// Takeover leaves no window between fencing and adopting, so a process that +// dies during the transfer cannot leave the scope transitioning and refuse +// every Exclusive writer until it returns. It also completes a transfer an +// earlier Fence left open. +func (a *Authority) Takeover(scope, owner string, from func(Record) bool) (Ticket, error) { + if owner == "" { + return Ticket{}, ErrNoOwner + } + var t Ticket + err := a.locked(scope, func(r *Record) (bool, error) { + if !from(*r) { + return false, fmt.Errorf("%w: %q owns %q", ErrNotOwner, r.Owner, scope) + } + r.Owner, r.Generation, r.Transitioning, r.Pending = owner, r.Generation+1, false, true + t = ticket(scope, *r) + return true, nil + }) + return t, err +} + +// WorkerOwnerPrefix starts the owner ID of every galactic-worker +// incarnation. +const WorkerOwnerPrefix = "galactic-worker:" + +// WorkerOwned reports whether owner is a galactic-worker incarnation. +func WorkerOwned(owner string) bool { return strings.HasPrefix(owner, WorkerOwnerPrefix) } + // Read returns the scope's record. func (a *Authority) Read(scope string) (Record, error) { var out Record diff --git a/internal/edge/ownership/ownership_netns_test.go b/internal/ownership/ownership_netns_test.go similarity index 100% rename from internal/edge/ownership/ownership_netns_test.go rename to internal/ownership/ownership_netns_test.go diff --git a/internal/edge/ownership/ownership_test.go b/internal/ownership/ownership_test.go similarity index 100% rename from internal/edge/ownership/ownership_test.go rename to internal/ownership/ownership_test.go diff --git a/internal/ownership/takeover_test.go b/internal/ownership/takeover_test.go new file mode 100644 index 000000000..c45e741bf --- /dev/null +++ b/internal/ownership/takeover_test.go @@ -0,0 +1,138 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ownership + +import ( + "errors" + "testing" +) + +func anyOwner(Record) bool { return true } + +// TestTakeoverFencesAndAdoptsInOneStep moves a scope to a new owner with no +// transitioning window: tickets issued before it are stale, the new ticket +// requires a snapshot, and an Exclusive writer is never refused. +func TestTakeoverFencesAndAdoptsInOneStep(t *testing.T) { + a, old := owned(t) + tk, err := a.Takeover(scope, WorkerOwnerPrefix+"n:2", anyOwner) + if err != nil { + t.Fatal(err) + } + if !tk.Resync { + t.Fatal("takeover ticket does not require a snapshot") + } + rec, err := a.Read(scope) + if err != nil { + t.Fatal(err) + } + if rec.Transitioning || !rec.Pending || rec.Owner != WorkerOwnerPrefix+"n:2" { + t.Fatalf("record after takeover = %+v", rec) + } + if _, err := a.Commit(old, mustNotRun(t)); !errors.Is(err, ErrNotOwner) { + t.Fatalf("commit by the replaced owner = %v, want ErrNotOwner", err) + } + if err := a.Exclusive(scope, noCheck, func() error { return nil }); err != nil { + t.Fatalf("exclusive write after takeover = %v", err) + } + // The exclusive write staled the takeover ticket; a fresh one commits + // only as a snapshot. + if _, err := a.CommitSnapshot(tk, mustNotRun(t)); !errors.Is(err, ErrStale) { + t.Fatalf("snapshot from a ticket issued before an exclusive write = %v, want ErrStale", err) + } + next, err := a.Begin(scope, WorkerOwnerPrefix+"n:2") + if err != nil { + t.Fatal(err) + } + if _, err := a.Commit(next, mustNotRun(t)); !errors.Is(err, ErrResyncRequired) { + t.Fatalf("incremental commit before the snapshot = %v, want ErrResyncRequired", err) + } + if _, err := a.CommitSnapshot(next, func() error { return nil }); err != nil { + t.Fatal(err) + } +} + +// TestTakeoverRefusedLeavesRecord changes nothing when from refuses the +// current owner. +func TestTakeoverRefusedLeavesRecord(t *testing.T) { + a, tk := owned(t) + before, err := a.Read(scope) + if err != nil { + t.Fatal(err) + } + _, err = a.Takeover(scope, WorkerOwnerPrefix+"n:2", func(r Record) bool { return WorkerOwned(r.Owner) }) + if !errors.Is(err, ErrNotOwner) { + t.Fatalf("takeover of another writer's scope = %v, want ErrNotOwner", err) + } + if after, _ := a.Read(scope); after != before { + t.Fatalf("record changed: %+v, was %+v", after, before) + } + if _, err := a.Commit(tk, func() error { return nil }); err != nil { + t.Fatalf("owner's ticket after a refused takeover = %v", err) + } +} + +// TestTakeoverCompletesAnOpenFence ends a transfer an earlier Fence left +// open, as when a process died between Fence and Adopt. +func TestTakeoverCompletesAnOpenFence(t *testing.T) { + a, _ := owned(t) + gen, err := a.Fence(scope) + if err != nil { + t.Fatal(err) + } + if err := a.Exclusive(scope, noCheck, mustNotRun(t)); !errors.Is(err, ErrTransitioning) { + t.Fatalf("exclusive write during the open fence = %v, want ErrTransitioning", err) + } + tk, err := a.Takeover(scope, WorkerOwnerPrefix+"n:3", anyOwner) + if err != nil { + t.Fatal(err) + } + if tk.Generation <= gen { + t.Fatalf("takeover generation %d, want above the fence's %d", tk.Generation, gen) + } + if _, err := a.Adopt(scope, "late", gen); !errors.Is(err, ErrStale) { + t.Fatalf("adopt of the superseded fence = %v, want ErrStale", err) + } + if err := a.Exclusive(scope, noCheck, func() error { return nil }); err != nil { + t.Fatalf("exclusive write after takeover = %v", err) + } +} + +// TestExclusiveIfSeesTheRecord refuses a write by the owner it reads under +// the lock, leaving the record unchanged. +func TestExclusiveIfSeesTheRecord(t *testing.T) { + a, _ := owned(t) + errOwned := errors.New("owned") + before, _ := a.Read(scope) + err := a.ExclusiveIf(scope, func(r Record) error { + if r.Owner == "cni" { + return errOwned + } + return nil + }, mustNotRun(t)) + if !errors.Is(err, errOwned) { + t.Fatalf("ExclusiveIf = %v, want the check's error", err) + } + if after, _ := a.Read(scope); after != before { + t.Fatalf("record changed: %+v, was %+v", after, before) + } +} + +func TestWorkerOwned(t *testing.T) { + for _, tc := range []struct { + name string + owner string + want bool + }{ + {"Worker", WorkerOwnerPrefix + "node:abc", true}, + {"Edge", "galactic-edge:node:abc", false}, + {"Unowned", "", false}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := WorkerOwned(tc.owner); got != tc.want { + t.Fatalf("WorkerOwned(%q) = %t, want %t", tc.owner, got, tc.want) + } + }) + } +} diff --git a/internal/worker/claims.go b/internal/worker/claims.go index 35cc854bd..5d63f57ce 100644 --- a/internal/worker/claims.go +++ b/internal/worker/claims.go @@ -412,8 +412,9 @@ func (w *Worker) withdrawLocked(k module.ClaimKey, gen uint64, scope, reason str // fenceIfRepairing fences a scope whose last commit failed, so a repair that // holds a ticket while preparing its snapshot cannot commit state that still -// includes a released claim. The next reconcile adopts the scope and commits -// a snapshot from current desired state. +// includes a released claim. It takes the scope over again in one step, which +// advances the generation and requires a snapshot, so the next reconcile +// commits one from current desired state. func (w *Worker) fenceIfRepairing(vpc string) { w.mu.Lock() s := w.scopes[vpc] @@ -422,14 +423,9 @@ func (w *Worker) fenceIfRepairing(vpc string) { if !repairing { return } - gen, err := w.d.Fence.Fence(module.Scope{VPC: vpc}.Name()) - if err != nil { + if _, err := w.d.Fence.Takeover(module.Scope{VPC: vpc}.Name(), w.owner, workerOrUnowned); err != nil { w.log.Error("fence scope for release", "vpc", vpc, "error", err) - return } - w.mu.Lock() - w.scope(vpc).fenced = gen - w.mu.Unlock() } // refuse counts a refusal and returns it. diff --git a/internal/worker/harness_test.go b/internal/worker/harness_test.go index 27994f90c..0aef9952b 100644 --- a/internal/worker/harness_test.go +++ b/internal/worker/harness_test.go @@ -26,7 +26,7 @@ import ( "google.golang.org/protobuf/proto" workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" - "go.datum.net/galactic/internal/edge/ownership" + "go.datum.net/galactic/internal/ownership" "go.datum.net/galactic/internal/worker/auth" "go.datum.net/galactic/internal/worker/module" "go.datum.net/galactic/internal/worker/module/fake" diff --git a/internal/worker/module/module.go b/internal/worker/module/module.go index 8ae8da32e..381f1eaf3 100644 --- a/internal/worker/module/module.go +++ b/internal/worker/module/module.go @@ -18,7 +18,7 @@ import ( "slices" workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" - "go.datum.net/galactic/internal/edge/ownership" + "go.datum.net/galactic/internal/ownership" ) // Scope is one VPC on this node: the unit of ownership and reconciliation. @@ -122,7 +122,7 @@ type Capability struct { // Module programs the host for one delivery mode. // // Plan and Read must not write. Apply runs inside the scope's fence commit -// (internal/edge/ownership): it must finish every write before it returns, +// (internal/ownership): it must finish every write before it returns, // must not call the fence, and must leave the host matching desired even // when the host held rows from an earlier incarnation or a partial commit, // because Worker calls it with the complete desired state of the scope. diff --git a/internal/worker/reconcile.go b/internal/worker/reconcile.go index 04119295d..5d10e8912 100644 --- a/internal/worker/reconcile.go +++ b/internal/worker/reconcile.go @@ -17,7 +17,7 @@ import ( "google.golang.org/protobuf/proto" workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" - "go.datum.net/galactic/internal/edge/ownership" + "go.datum.net/galactic/internal/ownership" "go.datum.net/galactic/internal/worker/module" "go.datum.net/galactic/internal/worker/store" ) @@ -298,38 +298,33 @@ func modeName(m workerv1alpha1.Mode) string { } // acquire returns a ticket for this incarnation's next commit to scope: -// Begin for a scope it owns, Claim for an unowned one, and Fence then Adopt -// for one an earlier Worker incarnation owned or this one fenced. +// Begin for a scope it owns, Claim for an unowned one, and Takeover for one +// an earlier Worker incarnation owned or left transitioning. A scope another +// writer owns, or is moving to, is errOwnedElsewhere or ErrTransitioning. func (w *Worker) acquire(scope module.Scope) (ownership.Ticket, error) { name := scope.Name() rec, err := w.d.Fence.Read(name) if err != nil { return ownership.Ticket{}, err } - w.mu.Lock() - fenced := w.scope(scope.VPC).fenced - w.mu.Unlock() - workerOwned := rec.Owner == "" || strings.HasPrefix(rec.Owner, ownerPrefix) switch { - case rec.Transitioning && fenced == rec.Generation: - case rec.Transitioning && workerOwned, !rec.Transitioning && rec.Owner != w.owner && rec.Owner != "" && workerOwned: - if fenced, err = w.d.Fence.Fence(name); err != nil { - return ownership.Ticket{}, err - } + case !rec.Transitioning && rec.Owner == w.owner: + return w.d.Fence.Begin(name, w.owner) + case !rec.Transitioning && rec.Owner == "": + return w.d.Fence.Claim(name, w.owner) + case rec.Owner == "" || ownership.WorkerOwned(rec.Owner): + return w.d.Fence.Takeover(name, w.owner, workerOrUnowned) case rec.Transitioning: return ownership.Ticket{}, fmt.Errorf("%w: %s is moving to another owner", ownership.ErrTransitioning, name) - case rec.Owner == "": - return w.d.Fence.Claim(name, w.owner) - case rec.Owner == w.owner: - return w.d.Fence.Begin(name, w.owner) default: return ownership.Ticket{}, fmt.Errorf("%w: %q owns %s", errOwnedElsewhere, rec.Owner, name) } - t, err := w.d.Fence.Adopt(name, w.owner, fenced) - w.mu.Lock() - w.scope(scope.VPC).fenced = 0 - w.mu.Unlock() - return t, err +} + +// workerOrUnowned accepts a scope no writer owns or a Worker incarnation +// owns, as it stands under the fence's lock. +func workerOrUnowned(r ownership.Record) bool { + return r.Owner == "" || ownership.WorkerOwned(r.Owner) } // reportFailure sets Programmed=False with reason on every live claim of diff --git a/internal/worker/recovery_test.go b/internal/worker/recovery_test.go index f4ae7157d..e69e8d02a 100644 --- a/internal/worker/recovery_test.go +++ b/internal/worker/recovery_test.go @@ -309,8 +309,9 @@ func TestWithdrawalDuringRepair(t *testing.T) { } fenced, err := e.fence.Read(module.Scope{VPC: vpcA}.Name()) e.mustOK(err) - if !fenced.Transitioning || fenced.Generation <= before.Generation { - t.Fatalf("release during repair did not fence the scope: %+v", fenced) + if fenced.Transitioning || !fenced.Pending || fenced.Generation <= before.Generation || + fenced.Owner != e.w.Owner() { + t.Fatalf("release during repair did not take the scope over again: %+v", fenced) } e.local.ResumePlan() diff --git a/internal/worker/worker.go b/internal/worker/worker.go index bfc5f8e93..45cd93157 100644 --- a/internal/worker/worker.go +++ b/internal/worker/worker.go @@ -9,7 +9,7 @@ // sources and recorded durably before it returns; nothing is programmed on // the call path. Reconciliation runs per VPC scope from a deduplicating // queue. It resolves every destination, takes the scope's ticket from the -// ownership fence (internal/edge/ownership), commits the complete desired +// ownership fence (internal/ownership), commits the complete desired // state of the scope through the delivery modules and reads the host back to // report programmed and observed state. Desired state is durable; // programmed and observed state are rebuilt from the host after a restart. @@ -30,7 +30,7 @@ import ( "k8s.io/client-go/util/workqueue" workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" - "go.datum.net/galactic/internal/edge/ownership" + "go.datum.net/galactic/internal/ownership" "go.datum.net/galactic/internal/worker/auth" "go.datum.net/galactic/internal/worker/module" "go.datum.net/galactic/internal/worker/sources" @@ -54,11 +54,6 @@ const ( DefaultResyncInterval = 30 * time.Second ) -// ownerPrefix starts the fence owner ID of every Worker incarnation, so a -// new incarnation can tell a scope an earlier one owned from a scope another -// writer owns. -const ownerPrefix = "galactic-worker:" - // Authenticator authenticates the caller of a call. type Authenticator interface { Authenticate(ctx context.Context) (auth.Caller, error) @@ -132,9 +127,6 @@ type scopeState struct { resyncing bool // repair is set after a failed commit, until a commit succeeds. repair bool - // fenced is the fence generation of a Fence this incarnation started and - // has not yet adopted. - fenced uint64 } // Worker is the reconciliation core and gRPC service. @@ -183,7 +175,7 @@ func New(cfg Config, d Deps) (*Worker, error) { w := &Worker{ cfg: cfg, d: d, - owner: ownerPrefix + cfg.Node + ":" + hex.EncodeToString(id), + owner: ownership.WorkerOwnerPrefix + cfg.Node + ":" + hex.EncodeToString(id), log: d.Log.With("component", "galactic-worker"), claims: map[module.ClaimKey]*claim{}, scopes: map[string]*scopeState{}, From 11fa74824d0d9e2fdc168b5f558b90bdef3b61d4 Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Wed, 18 Nov 2026 18:43:33 -0500 Subject: [PATCH 2/5] feat(cni): Write shared VPC state under the VPC write fence CNI, the galactic-cni installer and galactic-router GC wrote a VPC's shared host state without the node's VPC write fence. Worker cannot be the fenced authority for the node and VPC scope (docs/worker ownership.md, D12) while any of those writes bypass the fence. This change makes each of them an Exclusive write. Exclusive needs no owner, so workload attach still never waits for Worker. - galactic-tap and galactic-veth ADD create the VRF and enslave the host link inside an Exclusive write (internal/cni/vpcfence). - galactic-bgp runs each publish attempt (Argument claim, BGPVRFInstance, vrf_table registration and BGPAdvertisement), its rollback and the DEL advertisement release under the fence. - The installer's sidecar return install and advertisement reap run under the endpoint's VPC fence. The vrf_table repair in SweepEBPFVRFTable writes through a VPCWriter, and the installer's writer refuses a scope a galactic-worker incarnation owns. - maintenance.Guard.Reclaim checks the scope's owner under the lock with ExclusiveIf. galactic-router GC therefore keeps the VRF and BGPVRFInstance of a Worker-owned VPC, which Worker reclaims itself. This is the first step of the two-step GC transfer in ownership.md. ExclusiveWait retries ErrTransitioning until its context ends, and CNI bounds that wait at 30 s. The galactic-cni DaemonSet mounts the fence directory, as galactic-router already did. Without the mount, the installer's writes would fence against a private directory. --- config/galactic-cni/README.md | 13 + config/galactic-cni/daemonset.yaml | 10 + config/galactic-router/base/daemonset.yaml | 10 +- internal/cni/cni_test.go | 14 +- internal/cni/ops_add.go | 21 +- internal/cni/vpcfence/vpcfence.go | 62 ++++ internal/cni/vpcfence/vpcfence_test.go | 104 +++++++ internal/cnibgp/bgp.go | 274 +++++++++--------- internal/cnibgp/main_test.go | 8 +- internal/cnibgp/ops_del.go | 8 +- internal/cnibgp/resource.go | 18 ++ internal/cnitap/cnitap_test.go | 14 +- internal/cnitap/ops_add.go | 21 +- internal/edge/maintenance/maintenance.go | 11 +- internal/edge/maintenance/maintenance_test.go | 25 ++ internal/gc/gc.go | 70 ++++- internal/gc/gc_ebpf_test.go | 14 +- internal/gc/vrftables_root_test.go | 4 +- internal/installer/hostlinkrepair_test.go | 16 +- internal/installer/installer.go | 4 +- internal/installer/sidecarreap.go | 5 +- internal/installer/sidecarreturn.go | 19 +- internal/ownership/ownership.go | 37 +++ 23 files changed, 600 insertions(+), 182 deletions(-) create mode 100644 internal/cni/vpcfence/vpcfence.go create mode 100644 internal/cni/vpcfence/vpcfence_test.go diff --git a/config/galactic-cni/README.md b/config/galactic-cni/README.md index f72451f7a..dc7b98788 100644 --- a/config/galactic-cni/README.md +++ b/config/galactic-cni/README.md @@ -165,6 +165,19 @@ lock path is a constant every one of those processes shares. Without the mount, the repair would lock a file only it can see, and could re-enslave a link at the same moment an ADD or GC acts on its VRF. +## The `vpc-ownership` mount + +`/var/lib/galactic-edge/ownership` holds the node's VPC write fence +(`internal/ownership`). Host-executed CNI writes a VPC's shared host state +under it: the VRF, the SID Argument and BGPVRFInstance, `vrf_table` rows and +the attachment's BGPAdvertisement. `credential-refresh` writes under it too, +for the ingress sidecar's return path, its reaper and the `vrf_table` repair. +The repair leaves a VPC whose scope galactic-worker owns to that Worker. +galactic-router, galactic-edge and galactic-worker share the same directory. +Like `vrf-lock-dir`, it is mounted at its own host-absolute path, because +every writer must lock the same files. On a missing mount the writes fence +against a private, empty directory and no longer exclude the other writers. + ## The `cni-net-dir` mount `credential-refresh` mounts the host's `/etc/cni/net.d` writable, as the diff --git a/config/galactic-cni/daemonset.yaml b/config/galactic-cni/daemonset.yaml index 3debee5c3..6873654a5 100644 --- a/config/galactic-cni/daemonset.yaml +++ b/config/galactic-cni/daemonset.yaml @@ -171,6 +171,12 @@ spec: # stranded tap or veth under it. See config/galactic-cni/README.md. - name: vrf-lock-dir mountPath: /var/lib/cni/galactic-vrf + # The node's VPC write fence (internal/ownership), shared with + # host-executed CNI, galactic-router, galactic-edge and + # galactic-worker. The sidecar return path, its reaper and the + # vrf_table repair write a VPC's shared state only under it. + - name: vpc-ownership + mountPath: /var/lib/galactic-edge/ownership # BPF map creation under load (image pull + verifier work on a # slow node) can outrun readinessProbe/livenessProbe's short # initialDelaySeconds, risking a liveness-triggered restart loop @@ -225,6 +231,10 @@ spec: hostPath: path: /var/lib/cni/galactic-vrf type: DirectoryOrCreate + - name: vpc-ownership + hostPath: + path: /var/lib/galactic-edge/ownership + type: DirectoryOrCreate - name: bpf-fs # Host bpffs for pinned-map continuity; `Directory` (not # DirectoryOrCreate) and its SECURITY NOTE are documented in diff --git a/config/galactic-router/base/daemonset.yaml b/config/galactic-router/base/daemonset.yaml index 05f53f8de..b26468a71 100644 --- a/config/galactic-router/base/daemonset.yaml +++ b/config/galactic-router/base/daemonset.yaml @@ -164,11 +164,11 @@ spec: # DaemonSet regardless of this mount's presence. - name: cni-state mountPath: /var/lib/cni - # The node's write fence, shared with galactic-edge - # (internal/ownership). GC deletes a VPC's VRF and - # BGPVRFInstance only under that VPC's fence, after checking no - # edge attachment uses it. Without this mount GC reclaims - # neither. + # The node's VPC write fence (internal/ownership), shared with + # CNI, galactic-edge and galactic-worker. GC deletes a VPC's VRF + # and BGPVRFInstance only under that VPC's fence, after checking + # no edge attachment uses it and no galactic-worker owns it. + # Without this mount GC reclaims neither. - name: edge-ownership mountPath: /var/lib/galactic-edge/ownership - name: gobgp-api diff --git a/internal/cni/cni_test.go b/internal/cni/cni_test.go index 1edf9e451..0b820f417 100644 --- a/internal/cni/cni_test.go +++ b/internal/cni/cni_test.go @@ -15,15 +15,27 @@ import ( "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types" + "go.datum.net/galactic/internal/cni/vpcfence" "go.datum.net/galactic/internal/cniipam" "go.datum.net/galactic/internal/cnimaster" "go.datum.net/galactic/internal/cnitestutil" ) +// TestMain points the VPC write fence at a temporary directory, so tests +// neither need root to create it nor contend with a real node's. func TestMain(m *testing.M) { _ = os.Setenv("GALACTIC_CNI_NODE_NAME", "test-node") InitCNIConfig() - os.Exit(m.Run()) + os.Exit(func() int { + dir, err := os.MkdirTemp("", "cni-vpcfence-") + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + defer func() { _ = os.RemoveAll(dir) }() + vpcfence.Dir = dir + return m.Run() + }()) } const ( diff --git a/internal/cni/ops_add.go b/internal/cni/ops_add.go index 56ff82bd6..7d6111355 100644 --- a/internal/cni/ops_add.go +++ b/internal/cni/ops_add.go @@ -15,6 +15,7 @@ import ( "github.com/vishvananda/netlink" "go.datum.net/galactic/internal/cni/veth" + "go.datum.net/galactic/internal/cni/vpcfence" "go.datum.net/galactic/internal/cnimaster" "go.datum.net/galactic/internal/hostconf" "go.datum.net/galactic/internal/nadpatch" @@ -98,15 +99,19 @@ func cmdAdd(args *skel.CmdArgs) (err error) { }() // The VRF is created and the veth enslaved to it under one lock, so - // garbage collection never finds this VPC's VRF empty in between. + // garbage collection never finds this VPC's VRF empty in between. The VRF + // is shared state of the VPC on the node, so the write runs under the + // VPC's write fence; it never waits for the fence's owner to run. var addRes veth.AddResult - if err := vrf.EnsureAndEnslave(pluginConf.VPC, func(*netlink.Vrf) error { - var addErr error - addRes, addErr = veth.Add(pluginConf.VPC, pluginConf.VPCAttachment, args.ContainerID, pluginConf.MTU) - if addErr != nil { - return fmt.Errorf("add veth: %w", addErr) - } - return nil + if err := vpcfence.Write(context.Background(), pluginConf.VPC, func() error { + return vrf.EnsureAndEnslave(pluginConf.VPC, func(*netlink.Vrf) error { + var addErr error + addRes, addErr = veth.Add(pluginConf.VPC, pluginConf.VPCAttachment, args.ContainerID, pluginConf.MTU) + if addErr != nil { + return fmt.Errorf("add veth: %w", addErr) + } + return nil + }) }); err != nil { return err } diff --git a/internal/cni/vpcfence/vpcfence.go b/internal/cni/vpcfence/vpcfence.go new file mode 100644 index 000000000..3b143735e --- /dev/null +++ b/internal/cni/vpcfence/vpcfence.go @@ -0,0 +1,62 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package vpcfence runs CNI's writes to a VPC's shared host state under the +// VPC's write fence (internal/ownership), as a writer that never owns the +// scope. The shared state is the canonical host VRF and its table, the SID +// Argument and BGPVRFInstance, vrf_table rows and the BGPAdvertisement of an +// attachment on the node. Whichever process owns the scope, such as +// galactic-worker, sees each write as an Exclusive one and prepares its next +// change from the state it left. No write waits for that owner to run. +package vpcfence + +import ( + "context" + "fmt" + "time" + + "go.datum.net/galactic/internal/ownership" +) + +// Dir is the fence's state directory, shared with every other writer on the +// node. Tests point it elsewhere. +var Dir = ownership.DefaultDir + +// Wait bounds how long Write waits while ownership of the scope moves, which +// takes one locked file write. +const Wait = 30 * time.Second + +// Write runs mutate under vpc's write fence, waiting up to Wait, or until ctx +// is done, while ownership of the scope moves. It creates the fence's +// directory when the node has none yet. mutate must finish every write before +// it returns and must not call the fence. +func Write(ctx context.Context, vpc string, mutate func() error) error { + auth, err := ownership.New(Dir) + if err != nil { + return fmt.Errorf("open VPC write fence: %w", err) + } + ctx, cancel := context.WithTimeout(ctx, Wait) + defer cancel() + if err := auth.ExclusiveWait(ctx, ownership.VPCScope(vpc), func(ownership.Record) error { return nil }, + mutate); err != nil { + return fmt.Errorf("write VPC %s under its fence: %w", vpc, err) + } + return nil +} + +// WriteUnlessWorker is Write for maintenance a galactic-worker incarnation +// does itself once it owns the VPC's scope: it returns an error wrapping +// ownership.ErrWorkerOwned, and runs nothing, when one does. +func WriteUnlessWorker(ctx context.Context, vpc string, mutate func() error) error { + auth, err := ownership.New(Dir) + if err != nil { + return fmt.Errorf("open VPC write fence: %w", err) + } + ctx, cancel := context.WithTimeout(ctx, Wait) + defer cancel() + if err := auth.ExclusiveWait(ctx, ownership.VPCScope(vpc), ownership.RefuseWorkerOwned, mutate); err != nil { + return fmt.Errorf("write VPC %s under its fence: %w", vpc, err) + } + return nil +} diff --git a/internal/cni/vpcfence/vpcfence_test.go b/internal/cni/vpcfence/vpcfence_test.go new file mode 100644 index 000000000..66533f7bc --- /dev/null +++ b/internal/cni/vpcfence/vpcfence_test.go @@ -0,0 +1,104 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package vpcfence + +import ( + "context" + "errors" + "testing" + "time" + + "go.datum.net/galactic/internal/ownership" +) + +const vpc = "00000000e1" + +func authority(t *testing.T) *ownership.Authority { + t.Helper() + Dir = t.TempDir() + a, err := ownership.New(Dir) + if err != nil { + t.Fatal(err) + } + return a +} + +// TestWriteStalesOwnerTickets runs a write on a scope another process owns, +// without waiting for that owner, and leaves the owner's earlier ticket stale. +func TestWriteStalesOwnerTickets(t *testing.T) { + a := authority(t) + tk, err := a.Claim(ownership.VPCScope(vpc), ownership.WorkerOwnerPrefix+"n:1") + if err != nil { + t.Fatal(err) + } + ran := false + if err := Write(context.Background(), vpc, func() error { ran = true; return nil }); err != nil || !ran { + t.Fatalf("Write = %v (ran %t), want it run", err, ran) + } + if _, err := a.CommitSnapshot(tk, func() error { return nil }); !errors.Is(err, ownership.ErrStale) { + t.Fatalf("owner commit from a ticket issued before the write = %v, want ErrStale", err) + } +} + +// TestWriteWaitsForAnOpenTransfer retries while ownership moves and writes +// once the transfer completes. +func TestWriteWaitsForAnOpenTransfer(t *testing.T) { + a := authority(t) + scope := ownership.VPCScope(vpc) + gen, err := a.Fence(scope) + if err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- Write(context.Background(), vpc, func() error { return nil }) }() + select { + case err := <-done: + t.Fatalf("Write returned %v during the transfer", err) + case <-time.After(150 * time.Millisecond): + } + if _, err := a.Adopt(scope, ownership.WorkerOwnerPrefix+"n:1", gen); err != nil { + t.Fatal(err) + } + if err := <-done; err != nil { + t.Fatalf("Write after the transfer = %v", err) + } +} + +// TestWriteGivesUpWithItsContext returns ErrTransitioning once ctx ends. +func TestWriteGivesUpWithItsContext(t *testing.T) { + a := authority(t) + if _, err := a.Fence(ownership.VPCScope(vpc)); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + err := Write(ctx, vpc, func() error { + t.Error("write ran during the transfer") + return nil + }) + if !errors.Is(err, ownership.ErrTransitioning) || !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("Write = %v, want ErrTransitioning and the deadline", err) + } +} + +// TestWriteUnlessWorker runs nothing on a scope a Worker owns and writes one +// no Worker owns. +func TestWriteUnlessWorker(t *testing.T) { + a := authority(t) + if _, err := a.Claim(ownership.VPCScope(vpc), ownership.WorkerOwnerPrefix+"n:1"); err != nil { + t.Fatal(err) + } + err := WriteUnlessWorker(context.Background(), vpc, func() error { + t.Error("write ran on a Worker-owned scope") + return nil + }) + if !errors.Is(err, ownership.ErrWorkerOwned) { + t.Fatalf("WriteUnlessWorker = %v, want ErrWorkerOwned", err) + } + ran := false + if err := WriteUnlessWorker(context.Background(), "e2", func() error { ran = true; return nil }); err != nil || !ran { + t.Fatalf("WriteUnlessWorker on an unowned scope = %v (ran %t)", err, ran) + } +} diff --git a/internal/cnibgp/bgp.go b/internal/cnibgp/bgp.go index 6e96d269f..18d8ad776 100644 --- a/internal/cnibgp/bgp.go +++ b/internal/cnibgp/bgp.go @@ -41,6 +41,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" bgpv1alpha1 "go.datum.net/galactic/api/network/v1alpha1" + "go.datum.net/galactic/internal/cni/vpcfence" "go.datum.net/galactic/internal/cniipam" "go.datum.net/galactic/internal/crdnames" "go.datum.net/galactic/internal/gc" @@ -296,145 +297,158 @@ func publishBGPState( vpcHex string, k8s client.Client, ) (publishResult, error) { var result publishResult + // Every write publishFenced makes is shared state of the VPC on this + // node, so each attempt runs under the VPC's write fence. err := retryK8sOps(cniTimeout, func(ctx context.Context) error { - bgp, err := lookupBGPRouter(ctx, k8s, nodeName, namespace) - if err != nil { - return err - } + return vpcfence.Write(ctx, cfg.vpc, func() error { + return publishFenced(ctx, args, cfg, nodeName, namespace, ipamResult, vpcHex, k8s, &result) + }) + }) + return result, err +} - // Keyed by (vpc, node) rather than (vpc, attachment): the kernel VRF - // is shared by every attachment on this VPC on this node, so they all - // converge on one CRD and one Argument. - vrfName := crdnames.BGPVRFInstanceName(cfg.vpc, nodeName) - rtValue, err := routeTarget(int64(bgp.asNumber), vpcHex) - if err != nil { - return fmt.Errorf("compute route target: %w", err) - } +// publishFenced is one attempt of publishBGPState, run under the VPC's write +// fence. It records what it created in result. +func publishFenced( + ctx context.Context, args *skel.CmdArgs, cfg publishConfig, nodeName, namespace string, + ipamResult *cniipam.IPAMResult, vpcHex string, k8s client.Client, result *publishResult, +) error { + bgp, err := lookupBGPRouter(ctx, k8s, nodeName, namespace) + if err != nil { + return err + } - // Allocated, written and checked for a collision under the node's - // Argument lock; see claimArgument. - siblings := func() ([]string, error) { return siblingHostLinks(cfg.vpc, cfg.vpcAttachment) } - release := func(argument int32) error { - return attachreg.ReleaseArgument(ebpfPinDir, - attachreg.Node{Locator: bgp.srv6Locator, NodeID: bgp.nodeID}, cfg.vpc, uint16(argument)) - } - vrfID, created, err := claimArgument(ctx, k8s, namespace, bgp.routerName, vrfName, - result.vrfInstanceCreated, siblings, func(vrfID int32) bgpv1alpha1.BGPVRFInstanceSpec { - return buildVRFInstanceSpec(bgp.routerName, rtValue, vrfID) - }, release) - if created { - result.vrfInstanceCreated = true - } - if err != nil { - return err - } - slog.Debug("BGP: BGPVRFInstance applied", "resource_name", vrfName, "resource_namespace", namespace, - "vrfID", vrfID, "routeTarget", rtValue, "router", bgp.routerName, "created", created) - - // Advertise guest prefixes only. Gateway addresses stay host-local - // and are included separately in the eBPF pass-through routes. - prefixes, ipv6Subnet, ipv4Addr := ipamAdvertisementPrefixes(ipamResult) - - // Nothing to publish when this node's router has no SRv6 locator or - // node ID, so leave result.sid invalid rather than compute a SID for - // an attachment that has no SRv6 endpoint. - if bgp.srv6Locator != "" && bgp.nodeID != 0 { - sid, err := srv6.ComputeSID(bgp.srv6Locator, bgp.nodeID, vrfID, bgpv1alpha1.SRv6FunctionEndDT46) - if err != nil { - return fmt.Errorf("compute SRv6 uSID: %w", err) - } - result.sid = sid - } + // Keyed by (vpc, node) rather than (vpc, attachment): the kernel VRF + // is shared by every attachment on this VPC on this node, so they all + // converge on one CRD and one Argument. + vrfName := crdnames.BGPVRFInstanceName(cfg.vpc, nodeName) + rtValue, err := routeTarget(int64(bgp.asNumber), vpcHex) + if err != nil { + return fmt.Errorf("compute route target: %w", err) + } - // Not tracked for rollback: the vrf_table entry is shared by every - // attachment on this VPC and node, like the BGPVRFInstance above. - registered, err := registerEBPFDatapath( - bgp, cfg.vpc, cfg.vpcAttachment, cfg.ifaceType, uint16(vrfID), ebpfPinDir, localEgressPrefixes(ipamResult), - ) - if err != nil { - return fmt.Errorf("register eBPF uSID datapath: %w", err) - } - if registered { - registerTenantGateway(ebpfPinDir, cfg.vpc, cfg.vpcAttachment, ipamResult) - } - err = activateTrafficPolicy(ebpfPinDir, cfg.vpc, cfg.vpcAttachment, registered, ipamResult, cfg.guestHWAddr) + // Allocated, written and checked for a collision under the node's + // Argument lock; see claimArgument. + siblings := func() ([]string, error) { return siblingHostLinks(cfg.vpc, cfg.vpcAttachment) } + release := func(argument int32) error { + return attachreg.ReleaseArgument(ebpfPinDir, + attachreg.Node{Locator: bgp.srv6Locator, NodeID: bgp.nodeID}, cfg.vpc, uint16(argument)) + } + vrfID, created, err := claimArgument(ctx, k8s, namespace, bgp.routerName, vrfName, + result.vrfInstanceCreated, siblings, func(vrfID int32) bgpv1alpha1.BGPVRFInstanceSpec { + return buildVRFInstanceSpec(bgp.routerName, rtValue, vrfID) + }, release) + if created { + result.vrfInstanceCreated = true + } + if err != nil { + return err + } + slog.Debug("BGP: BGPVRFInstance applied", "resource_name", vrfName, "resource_namespace", namespace, + "vrfID", vrfID, "routeTarget", rtValue, "router", bgp.routerName, "created", created) + + // Advertise guest prefixes only. Gateway addresses stay host-local + // and are included separately in the eBPF pass-through routes. + prefixes, ipv6Subnet, ipv4Addr := ipamAdvertisementPrefixes(ipamResult) + + // Nothing to publish when this node's router has no SRv6 locator or + // node ID, so leave result.sid invalid rather than compute a SID for + // an attachment that has no SRv6 endpoint. + if bgp.srv6Locator != "" && bgp.nodeID != 0 { + sid, err := srv6.ComputeSID(bgp.srv6Locator, bgp.nodeID, vrfID, bgpv1alpha1.SRv6FunctionEndDT46) if err != nil { - return fmt.Errorf("activate traffic policy: %w", err) + return fmt.Errorf("compute SRv6 uSID: %w", err) } + result.sid = sid + } - adv := &bgpv1alpha1.BGPAdvertisement{ - ObjectMeta: metav1.ObjectMeta{ - Name: crdnames.BGPAdvertisementName(cfg.vpc, cfg.vpcAttachment, nodeName), - Namespace: namespace, - }, - } - var mergedPrefixes []string - var advOp controllerutil.OperationResult - // A predecessor's DEL on this node can update or delete the same - // advertisement between CreateOrUpdate's read and write; reread and - // write again rather than fail the ADD. - err = retry.OnError(retry.DefaultRetry, func(err error) bool { - return apierrors.IsConflict(err) || apierrors.IsNotFound(err) - }, func() error { - *adv = bgpv1alpha1.BGPAdvertisement{ObjectMeta: metav1.ObjectMeta{Name: adv.Name, Namespace: adv.Namespace}} - var err error - advOp, err = controllerutil.CreateOrUpdate(ctx, k8s, adv, func() error { - if adv.Annotations == nil { - adv.Annotations = make(map[string]string) - } - adv.Annotations[crdnames.NetNSKey(args.ContainerID)] = args.Netns - if ipv6Subnet != "" { - adv.Annotations[crdnames.SubnetKeyIPv6(args.ContainerID)] = ipv6Subnet - } - if ipv4Addr != "" { - adv.Annotations[crdnames.SubnetKeyIPv4(args.ContainerID)] = ipv4Addr - } - // The attachment's config carries no "ipam" block, such as a tap - // workload managing its own addressing. Mark the advertisement so - // its empty spec.prefixes reads as intentional rather than as - // addressing that failed to arrive. Cleared once any ADD for this - // attachment does carry an allocation. - if ipamResult == nil { - adv.Annotations[crdnames.AnnotationNoAddressing] = crdnames.AnnotationNoAddressingValue - } else { - delete(adv.Annotations, crdnames.AnnotationNoAddressing) - } - // Prune dead siblings first, so a replaced pod's stale annotation - // cannot collide with this ADD's usually identical prefix. - pruneDeadContainerAnnotations(adv.Annotations) - mergedPrefixes = allAdvertisedPrefixes(adv.Annotations) - adv.Spec = buildAdvertisementSpec(bgp.routerName, rtValue, mergedPrefixes, vrfID) - return nil - }) - return err - }) - if err != nil { - return fmt.Errorf("apply BGPAdvertisement: %w", err) - } - // Gated on a genuine create, like vrfInstanceCreated above. A - // BGPAdvertisement is reused across pod churn on the same attachment, - // so marking it created on a mere update would let rollback delete one - // still backing a live container's route. - if advOp == controllerutil.OperationResultCreated { - result.advertisementCreated = true - } - // The fallback for an instance that moved here from a node whose DEL - // never ran: its advertisement would otherwise stay until that - // node's GC, and the attachment's status would flip between the two. - // Best effort, since failing the ADD over it would roll back this - // node's own advertisement. - if err := deleteOtherNodeAdvertisements(ctx, k8s, namespace, cfg.vpc, cfg.vpcAttachment, nodeName, adv); err != nil { - slog.Warn("ADD: could not delete other nodes' advertisements for this attachment", - "err", err, "vpc", cfg.vpc, "vpcAttachment", cfg.vpcAttachment) - } - slog.Debug("BGP: BGPAdvertisement applied", "resource_name", adv.Name, "resource_namespace", namespace, - "prefixes", mergedPrefixes, "addedPrefixes", prefixes, "containerID", args.ContainerID, "operation", advOp) + // Not tracked for rollback: the vrf_table entry is shared by every + // attachment on this VPC and node, like the BGPVRFInstance above. + registered, err := registerEBPFDatapath( + bgp, cfg.vpc, cfg.vpcAttachment, cfg.ifaceType, uint16(vrfID), ebpfPinDir, localEgressPrefixes(ipamResult), + ) + if err != nil { + return fmt.Errorf("register eBPF uSID datapath: %w", err) + } + if registered { + registerTenantGateway(ebpfPinDir, cfg.vpc, cfg.vpcAttachment, ipamResult) + } + err = activateTrafficPolicy(ebpfPinDir, cfg.vpc, cfg.vpcAttachment, registered, ipamResult, cfg.guestHWAddr) + if err != nil { + return fmt.Errorf("activate traffic policy: %w", err) + } - slog.Info("ADD: BGP state published", "containerID", args.ContainerID, - "vpc", cfg.vpc, "vpcAttachment", cfg.vpcAttachment) - return nil + adv := &bgpv1alpha1.BGPAdvertisement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crdnames.BGPAdvertisementName(cfg.vpc, cfg.vpcAttachment, nodeName), + Namespace: namespace, + }, + } + var mergedPrefixes []string + var advOp controllerutil.OperationResult + // A predecessor's DEL on this node can update or delete the same + // advertisement between CreateOrUpdate's read and write; reread and + // write again rather than fail the ADD. + err = retry.OnError(retry.DefaultRetry, func(err error) bool { + return apierrors.IsConflict(err) || apierrors.IsNotFound(err) + }, func() error { + *adv = bgpv1alpha1.BGPAdvertisement{ObjectMeta: metav1.ObjectMeta{Name: adv.Name, Namespace: adv.Namespace}} + var err error + advOp, err = controllerutil.CreateOrUpdate(ctx, k8s, adv, func() error { + if adv.Annotations == nil { + adv.Annotations = make(map[string]string) + } + adv.Annotations[crdnames.NetNSKey(args.ContainerID)] = args.Netns + if ipv6Subnet != "" { + adv.Annotations[crdnames.SubnetKeyIPv6(args.ContainerID)] = ipv6Subnet + } + if ipv4Addr != "" { + adv.Annotations[crdnames.SubnetKeyIPv4(args.ContainerID)] = ipv4Addr + } + // The attachment's config carries no "ipam" block, such as a tap + // workload managing its own addressing. Mark the advertisement so + // its empty spec.prefixes reads as intentional rather than as + // addressing that failed to arrive. Cleared once any ADD for this + // attachment does carry an allocation. + if ipamResult == nil { + adv.Annotations[crdnames.AnnotationNoAddressing] = crdnames.AnnotationNoAddressingValue + } else { + delete(adv.Annotations, crdnames.AnnotationNoAddressing) + } + // Prune dead siblings first, so a replaced pod's stale annotation + // cannot collide with this ADD's usually identical prefix. + pruneDeadContainerAnnotations(adv.Annotations) + mergedPrefixes = allAdvertisedPrefixes(adv.Annotations) + adv.Spec = buildAdvertisementSpec(bgp.routerName, rtValue, mergedPrefixes, vrfID) + return nil + }) + return err }) - return result, err + if err != nil { + return fmt.Errorf("apply BGPAdvertisement: %w", err) + } + // Gated on a genuine create, like vrfInstanceCreated above. A + // BGPAdvertisement is reused across pod churn on the same attachment, + // so marking it created on a mere update would let rollback delete one + // still backing a live container's route. + if advOp == controllerutil.OperationResultCreated { + result.advertisementCreated = true + } + // The fallback for an instance that moved here from a node whose DEL + // never ran: its advertisement would otherwise stay until that + // node's GC, and the attachment's status would flip between the two. + // Best effort, since failing the ADD over it would roll back this + // node's own advertisement. + if err := deleteOtherNodeAdvertisements(ctx, k8s, namespace, cfg.vpc, cfg.vpcAttachment, nodeName, adv); err != nil { + slog.Warn("ADD: could not delete other nodes' advertisements for this attachment", + "err", err, "vpc", cfg.vpc, "vpcAttachment", cfg.vpcAttachment) + } + slog.Debug("BGP: BGPAdvertisement applied", "resource_name", adv.Name, "resource_namespace", namespace, + "prefixes", mergedPrefixes, "addedPrefixes", prefixes, "containerID", args.ContainerID, "operation", advOp) + + slog.Info("ADD: BGP state published", "containerID", args.ContainerID, + "vpc", cfg.vpc, "vpcAttachment", cfg.vpcAttachment) + return nil } // deleteOtherNodeAdvertisements deletes every other node's advertisement for diff --git a/internal/cnibgp/main_test.go b/internal/cnibgp/main_test.go index 86d8c513a..9241bc004 100644 --- a/internal/cnibgp/main_test.go +++ b/internal/cnibgp/main_test.go @@ -7,13 +7,16 @@ package cnibgp import ( "fmt" "os" + "path/filepath" "testing" + "go.datum.net/galactic/internal/cni/vpcfence" "go.datum.net/galactic/internal/nodevpc" ) -// TestMain points the node-wide Argument lock at a temporary directory, so -// tests neither need root to create it nor contend with a real node's lock, +// TestMain points the node-wide Argument lock and the VPC write fence at a +// temporary directory, so tests neither need root to create them nor contend +// with a real node's, // and runs unprivileged tests without the VRF lock, whose directory only // root can create. func TestMain(m *testing.M) { @@ -25,6 +28,7 @@ func TestMain(m *testing.M) { } defer func() { _ = os.RemoveAll(dir) }() nodevpc.ArgumentLockDir = dir + vpcfence.Dir = filepath.Join(dir, "ownership") if os.Geteuid() != 0 { vrfLockedFn = func(fn func() error) error { return fn() } } diff --git a/internal/cnibgp/ops_del.go b/internal/cnibgp/ops_del.go index 3bcf157d9..e6ad1e1be 100644 --- a/internal/cnibgp/ops_del.go +++ b/internal/cnibgp/ops_del.go @@ -12,6 +12,7 @@ import ( "github.com/containernetworking/cni/pkg/types" type100 "github.com/containernetworking/cni/pkg/types/100" + "go.datum.net/galactic/internal/cni/vpcfence" "go.datum.net/galactic/internal/nadpatch" "go.datum.net/galactic/internal/podslice" ) @@ -106,8 +107,11 @@ func releaseAdvertisementBestEffort(args *skel.CmdArgs, namespace, vpc, vpcAttac } ctx, cancel := context.WithTimeout(context.Background(), cniTimeout) defer cancel() - if err := releaseAdvertisement(ctx, k8sClient, namespace, vpc, vpcAttachment, cniConfig.NodeName, - args.ContainerID); err != nil { + // The advertisement is shared state of the VPC on the node. + if err := vpcfence.Write(ctx, vpc, func() error { + return releaseAdvertisement(ctx, k8sClient, namespace, vpc, vpcAttachment, cniConfig.NodeName, + args.ContainerID) + }); err != nil { slog.Error("DEL: failed to release advertisement, cleanup deferred to GC", "err", err, "containerID", args.ContainerID, "vpc", vpc, "vpcAttachment", vpcAttachment) return diff --git a/internal/cnibgp/resource.go b/internal/cnibgp/resource.go index 554f7e77d..d5f6ab3ec 100644 --- a/internal/cnibgp/resource.go +++ b/internal/cnibgp/resource.go @@ -18,6 +18,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" bgpv1alpha1 "go.datum.net/galactic/api/network/v1alpha1" + "go.datum.net/galactic/internal/cni/vpcfence" "go.datum.net/galactic/internal/crdnames" "go.datum.net/galactic/internal/plumbing/intf" "go.datum.net/galactic/internal/plumbing/nldump" @@ -83,9 +84,26 @@ type resourceTracker struct { // sibling writes anything of its own. Its host link is already enslaved to // the VPC's VRF by then, so the instance is kept while any link other than // this attachment's own is enslaved there. +// +// Both are shared state of the VPC on the node, so the rollback runs under +// the VPC's write fence. A rollback the fence refuses leaves both for GC. func (rt *resourceTracker) cleanup(ctx context.Context) { slog.Info("Selective rollback: cleaning up resources created during failed ADD", "vpc", rt.vpc, "vpcAttachment", rt.vpcAttachment) + if !rt.advertisementCreated && !rt.vrfInstanceCreated { + return + } + if err := vpcfence.Write(ctx, rt.vpc, func() error { + rt.cleanupFenced(ctx) + return nil + }); err != nil { + slog.Error("Rollback: kept created resources, could not take the VPC's write fence; GC reclaims them", + "err", err, "vpc", rt.vpc, "vpcAttachment", rt.vpcAttachment) + } +} + +// cleanupFenced is cleanup's work, run under the VPC's write fence. +func (rt *resourceTracker) cleanupFenced(ctx context.Context) { if rt.advertisementCreated && rt.k8s != nil { adv := &bgpv1alpha1.BGPAdvertisement{ diff --git a/internal/cnitap/cnitap_test.go b/internal/cnitap/cnitap_test.go index 1ec9aa372..8d0e3cead 100644 --- a/internal/cnitap/cnitap_test.go +++ b/internal/cnitap/cnitap_test.go @@ -15,6 +15,7 @@ import ( "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types" + "go.datum.net/galactic/internal/cni/vpcfence" "go.datum.net/galactic/internal/cniipam" "go.datum.net/galactic/internal/cnimaster" "go.datum.net/galactic/internal/cnitestutil" @@ -34,10 +35,21 @@ const ( testHostTap = "G0abc123def" ) +// TestMain points the VPC write fence at a temporary directory, so tests +// neither need root to create it nor contend with a real node's. func TestMain(m *testing.M) { _ = os.Setenv("GALACTIC_CNI_NODE_NAME", "test-node") InitCNIConfig() - os.Exit(m.Run()) + os.Exit(func() int { + dir, err := os.MkdirTemp("", "cni-vpcfence-") + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + defer func() { _ = os.RemoveAll(dir) }() + vpcfence.Dir = dir + return m.Run() + }()) } // movedKeyConf builds an otherwise-valid galactic-tap config carrying diff --git a/internal/cnitap/ops_add.go b/internal/cnitap/ops_add.go index 5261b75fa..a21c861e6 100644 --- a/internal/cnitap/ops_add.go +++ b/internal/cnitap/ops_add.go @@ -18,6 +18,7 @@ import ( "github.com/vishvananda/netlink" "go.datum.net/galactic/internal/cni/tap" + "go.datum.net/galactic/internal/cni/vpcfence" "go.datum.net/galactic/internal/cniipam" "go.datum.net/galactic/internal/cnimaster" "go.datum.net/galactic/internal/hostconf" @@ -108,15 +109,19 @@ func cmdAdd(args *skel.CmdArgs) (err error) { }() // The VRF is created and the tap enslaved to it under one lock, so - // garbage collection never finds this VPC's VRF empty in between. + // garbage collection never finds this VPC's VRF empty in between. The VRF + // is shared state of the VPC on the node, so the write runs under the + // VPC's write fence; it never waits for the fence's owner to run. var addRes tap.AddResult - if err := vrf.EnsureAndEnslave(pluginConf.VPC, func(*netlink.Vrf) error { - var addErr error - addRes, addErr = tap.Add(pluginConf.VPC, pluginConf.VPCAttachment, args.ContainerID, pluginConf.MTU) - if addErr != nil { - return fmt.Errorf("add tap: %w", addErr) - } - return nil + if err := vpcfence.Write(context.Background(), pluginConf.VPC, func() error { + return vrf.EnsureAndEnslave(pluginConf.VPC, func(*netlink.Vrf) error { + var addErr error + addRes, addErr = tap.Add(pluginConf.VPC, pluginConf.VPCAttachment, args.ContainerID, pluginConf.MTU) + if addErr != nil { + return fmt.Errorf("add tap: %w", addErr) + } + return nil + }) }); err != nil { return err } diff --git a/internal/edge/maintenance/maintenance.go b/internal/edge/maintenance/maintenance.go index b4e2c134b..0b8efcefc 100644 --- a/internal/edge/maintenance/maintenance.go +++ b/internal/edge/maintenance/maintenance.go @@ -123,11 +123,16 @@ func canonical(vpc string) string { // Reclaim runs reclaim, which removes vpc's shared state, with vpc's write // fence held, after confirming under the fence that no edge attachment uses // vpc. It returns an error wrapping ErrInUse, and runs nothing, when one does, -// and runs nothing while ownership of vpc moves between galactic-edge -// processes. The write makes every ticket galactic-edge holds for vpc stale, +// and runs nothing while ownership of vpc moves between processes. It also +// runs nothing, returning an error wrapping ownership.ErrWorkerOwned, when a +// galactic-worker incarnation owns vpc's scope, since that Worker reclaims +// the VPC's state itself. The write makes every ticket galactic-edge holds for vpc stale, // so a change it prepared before the reclaim is refused and prepared again. func (g *Guard) Reclaim(ctx context.Context, vpc string, reclaim func() error) error { - return g.Ownership.Exclusive(ownership.VPCScope(vpc), func() error { + return g.Ownership.ExclusiveIf(ownership.VPCScope(vpc), func(r ownership.Record) error { + if err := ownership.RefuseWorkerOwned(r); err != nil { + return err + } ctx, cancel := context.WithTimeout(ctx, listTimeout) defer cancel() vpcs, err := g.VPCs(ctx) diff --git a/internal/edge/maintenance/maintenance_test.go b/internal/edge/maintenance/maintenance_test.go index b60c842c7..81bcd88d0 100644 --- a/internal/edge/maintenance/maintenance_test.go +++ b/internal/edge/maintenance/maintenance_test.go @@ -267,3 +267,28 @@ func TestUnavailable(t *testing.T) { t.Error("Reclaim succeeded") } } + +// TestReclaimLeavesWorkerOwnedScope refuses, under the fence, a VPC whose +// scope a galactic-worker incarnation owns, and runs nothing; a scope another +// writer owns is still reclaimed. +func TestReclaimLeavesWorkerOwnedScope(t *testing.T) { + c := fake.NewClientBuilder().WithScheme(scheme(t)).Build() + g := newGuard(t, c) + for vpc, owner := range map[string]string{"a": ownership.WorkerOwnerPrefix + node + ":1", "b": "galactic-edge:1"} { + tk, err := g.Ownership.Claim(ownership.VPCScope(vpc), owner) + if err != nil { + t.Fatal(err) + } + if _, err := g.Ownership.CommitSnapshot(tk, func() error { return nil }); err != nil { + t.Fatal(err) + } + } + ran := false + err := g.Reclaim(context.Background(), "a", func() error { ran = true; return nil }) + if !errors.Is(err, ownership.ErrWorkerOwned) || ran { + t.Fatalf("Reclaim of a Worker-owned VPC = %v (ran %t), want ErrWorkerOwned and nothing run", err, ran) + } + if err := g.Reclaim(context.Background(), "b", func() error { ran = true; return nil }); err != nil || !ran { + t.Fatalf("Reclaim of an edge-owned VPC = %v (ran %t), want it reclaimed", err, ran) + } +} diff --git a/internal/gc/gc.go b/internal/gc/gc.go index 1900bf700..48b876c55 100644 --- a/internal/gc/gc.go +++ b/internal/gc/gc.go @@ -25,6 +25,7 @@ import ( "go.datum.net/galactic/internal/crdnames" "go.datum.net/galactic/internal/edge/hostattach" "go.datum.net/galactic/internal/edge/maintenance" + "go.datum.net/galactic/internal/ownership" "go.datum.net/galactic/internal/plumbing/ebpf/ifindexvrfmap" "go.datum.net/galactic/internal/plumbing/ebpf/nptv6map" "go.datum.net/galactic/internal/plumbing/ebpf/uformat" @@ -68,7 +69,8 @@ type OrphanedCRD struct { } // EdgeUsers keeps garbage collection off the VPCs Galactic Edge attachments -// use on this node (internal/edge/maintenance). An edge attachment joins its +// use on this node (internal/edge/maintenance), and off the VPCs whose scope a +// galactic-worker incarnation owns, which that Worker reclaims itself. An edge attachment joins its // VPC's canonical host VRF, so that VRF, its BGPVRFInstance and its uSID rows // must outlive the VPC's last workload while an edge attachment remains. // @@ -78,8 +80,9 @@ type EdgeUsers interface { // means the answer is unknown. VPCs(ctx context.Context) (map[string]struct{}, error) // Reclaim runs reclaim under vpc's write fence, after confirming under - // the fence that no edge attachment uses vpc; otherwise it returns an - // error wrapping maintenance.ErrInUse and runs nothing. + // the fence that no edge attachment uses vpc and no galactic-worker owns + // its scope; otherwise it returns an error wrapping maintenance.ErrInUse + // or ownership.ErrWorkerOwned and runs nothing. Reclaim(ctx context.Context, vpc string, reclaim func() error) error } @@ -477,6 +480,11 @@ func RemoveOrphanedCRDs(ctx context.Context, k8s client.Client, orphans []Orphan "resource_name", o.Name, "resource_namespace", o.Namespace, "vpc", o.VPC) continue } + if errors.Is(err, ownership.ErrWorkerOwned) { + slog.Info("GC: left BGPVRFInstance to the galactic-worker that owns its VPC", + "resource_name", o.Name, "resource_namespace", o.Namespace, "vpc", o.VPC) + continue + } if errors.Is(err, vrf.ErrInUse) { slog.Info("GC: kept BGPVRFInstance whose VRF a link joined since it was collected", "resource_name", o.Name, "resource_namespace", o.Namespace, "reason", err.Error()) @@ -708,6 +716,8 @@ func RemoveOrphanedVRFs(ctx context.Context, vrfNames []string, edge EdgeUsers) switch { case errors.Is(err, maintenance.ErrInUse): slog.Info("GC: kept VRF an edge attachment uses", "name", name, "vpc", vpc) + case errors.Is(err, ownership.ErrWorkerOwned): + slog.Info("GC: left VRF to the galactic-worker that owns its VPC", "name", name, "vpc", vpc) case errors.Is(err, vrf.ErrInUse): slog.Info("GC: kept VRF that a link joined since it was collected", "name", name, "vpc", vpc, "reason", err.Error()) @@ -746,7 +756,13 @@ func removeVRF(name string) (removed bool, err error) { // because the datapath has not finished loading or merely inaccessible, means // there is no reachable datapath this tick. That is treated as nothing to do, // not an error. -func SweepEBPFVRFTable(ctx context.Context, k8s client.Client, namespace, nodeName, pinDir string) CleanupResult { +// +// Each repair is shared state of the instance's VPC, and runs through write. +// A repair write refuses with ownership.ErrWorkerOwned is skipped: the +// galactic-worker incarnation that owns the VPC's scope keeps its rows. +func SweepEBPFVRFTable( + ctx context.Context, k8s client.Client, namespace, nodeName, pinDir string, write VPCWriter, +) CleanupResult { result := CleanupResult{} if _, statErr := os.Stat(pinDir); statErr != nil { @@ -875,7 +891,7 @@ func SweepEBPFVRFTable(ctx context.Context, k8s client.Client, namespace, nodeNa live[key] = struct{}{} } - repairVRFRows(reg.VRF, nodeName, liveCRDNames, existingRows, &result) + repairVRFRows(ctx, reg.VRF, nodeName, liveCRDNames, existingRows, write, &result) removed, err := reg.VRF.Reconcile(live, cutoff) for _, e := range removed { @@ -924,8 +940,9 @@ func SweepEBPFVRFTable(ctx context.Context, k8s client.Client, namespace, nodeNa // was recreated on another table with no CNI ADD to re-register it, and the // datapath would otherwise route the VPC through a table another VPC can be // given (#1270). The egress kind the CNI wrote is kept. -func repairVRFRows(table *usidmap.VRFTable, nodeName string, liveCRDNames map[usidmap.VRFKey]string, - existingRows map[usidmap.VRFKey]usidmap.VRFEntry, result *CleanupResult) { +func repairVRFRows(ctx context.Context, table *usidmap.VRFTable, nodeName string, + liveCRDNames map[usidmap.VRFKey]string, existingRows map[usidmap.VRFKey]usidmap.VRFEntry, write VPCWriter, + result *CleanupResult) { if len(liveCRDNames) == 0 { return } @@ -946,7 +963,15 @@ func repairVRFRows(table *usidmap.VRFTable, nodeName string, liveCRDNames map[us row, present := existingRows[key] switch { case !present: - if err := table.Register(key.Block, key.Argument, vrfTableID, egressKind); err != nil { + err := writeVPC(ctx, write, instanceNameVPC(instName), func() error { + return table.Register(key.Block, key.Argument, vrfTableID, egressKind) + }) + if errors.Is(err, ownership.ErrWorkerOwned) { + slog.Debug("GC: left a missing eBPF vrf_table entry to the VPC's galactic-worker", + "vrfInstance", instName, "block", key.Block, "argument", key.Argument) + continue + } + if err != nil { slog.Error("GC: failed to repair missing eBPF vrf_table entry", "vrfInstance", instName, "block", key.Block, "argument", key.Argument, "err", err) result.Errors++ @@ -957,7 +982,15 @@ func repairVRFRows(table *usidmap.VRFTable, nodeName string, liveCRDNames map[us "vrfTableID", vrfTableID, "egressKind", egressKind) result.EBPFVRFEntriesRegistered++ case row.VRFTableID != vrfTableID: - if err := table.Register(key.Block, key.Argument, vrfTableID, row.EgressKind); err != nil { + err := writeVPC(ctx, write, instanceNameVPC(instName), func() error { + return table.Register(key.Block, key.Argument, vrfTableID, row.EgressKind) + }) + if errors.Is(err, ownership.ErrWorkerOwned) { + slog.Debug("GC: left an eBPF vrf_table entry with a stale table ID to the VPC's galactic-worker", + "vrfInstance", instName, "block", key.Block, "argument", key.Argument) + continue + } + if err != nil { slog.Error("GC: failed to rewrite eBPF vrf_table entry with a stale table ID", "vrfInstance", instName, "block", key.Block, "argument", key.Argument, "err", err) result.Errors++ @@ -971,6 +1004,25 @@ func repairVRFRows(table *usidmap.VRFTable, nodeName string, liveCRDNames map[us } } +// VPCWriter runs write as a write to vpc's shared state on the node, under +// whatever coordination the caller's process needs. An error wrapping +// ownership.ErrWorkerOwned means write did not run, because a galactic-worker +// incarnation owns the VPC's scope. +type VPCWriter func(ctx context.Context, vpc string, write func() error) error + +// Unfenced is a VPCWriter that runs write directly, for a process that is +// the node's only writer, such as a test. +func Unfenced(_ context.Context, _ string, write func() error) error { return write() } + +// writeVPC runs write through w for vpc, or directly when vpc is "", which +// names no fence. +func writeVPC(ctx context.Context, w VPCWriter, vpc string, write func() error) error { + if vpc == "" { + return write() + } + return w(ctx, vpc, write) +} + // instanceVPC returns the VPC of the kernel VRF among links that the // BGPVRFInstance named instName serves on nodeName, in either CRD name form, // or "" when none does. diff --git a/internal/gc/gc_ebpf_test.go b/internal/gc/gc_ebpf_test.go index 8de400745..eed147201 100644 --- a/internal/gc/gc_ebpf_test.go +++ b/internal/gc/gc_ebpf_test.go @@ -67,7 +67,7 @@ func gcTestScheme(t *testing.T) *runtime.Scheme { // that as an error. func TestSweepEBPFVRFTable_MissingPinDirIsNoOp(t *testing.T) { k8s := fake.NewClientBuilder().WithScheme(gcTestScheme(t)).Build() - result := SweepEBPFVRFTable(context.Background(), k8s, "default", "node-a", "/sys/fs/bpf/galactic-does-not-exist") + result := SweepEBPFVRFTable(context.Background(), k8s, "default", "node-a", "/sys/fs/bpf/galactic-does-not-exist", Unfenced) if result.Errors != 0 || result.EBPFVRFEntriesRemoved != 0 { t.Errorf("result = %+v, want a zero-value result (no-op)", result) } @@ -144,7 +144,7 @@ func TestSweepEBPFVRFTable_RemovesStaleKeepsLive(t *testing.T) { t.Fatalf("seed stale entry: %v", err) } - result := SweepEBPFVRFTable(context.Background(), k8s, namespace, nodeName, pinDir) + result := SweepEBPFVRFTable(context.Background(), k8s, namespace, nodeName, pinDir, Unfenced) if result.Errors != 0 { t.Errorf("result.Errors = %d, want 0", result.Errors) } @@ -222,7 +222,7 @@ func TestSweepEBPFVRFTable_PreservesIngressSidecarReservedBlock(t *testing.T) { t.Fatalf("seed ordinary stale entry: %v", err) } - result := SweepEBPFVRFTable(context.Background(), k8s, namespace, nodeName, pinDir) + result := SweepEBPFVRFTable(context.Background(), k8s, namespace, nodeName, pinDir, Unfenced) if result.Errors != 0 { t.Errorf("result.Errors = %d, want 0", result.Errors) } @@ -291,7 +291,7 @@ func TestSweepEBPFVRFTable_NoRoutersForNodeSkipsSweep(t *testing.T) { t.Fatalf("seed entry: %v", err) } - result := SweepEBPFVRFTable(context.Background(), k8s, namespace, nodeName, pinDir) + result := SweepEBPFVRFTable(context.Background(), k8s, namespace, nodeName, pinDir, Unfenced) if result.Errors != 0 { t.Errorf("result.Errors = %d, want 0 (no-router is a skip, not an error)", result.Errors) } @@ -489,19 +489,19 @@ func TestSweepEBPFVRFTable_KeepsEdgeAttachmentEntry(t *testing.T) { } setLinks(nil, errors.New("netlink")) - if res := SweepEBPFVRFTable(context.Background(), k8s, namespace, nodeName, pinDir); res.Errors != 1 || + if res := SweepEBPFVRFTable(context.Background(), k8s, namespace, nodeName, pinDir, Unfenced); res.Errors != 1 || res.EBPFVRFEntriesRemoved != 0 || !present(edgeVRFID) || !present(otherVRFID) { t.Fatalf("sweep without the edge links = %+v; want an error and nothing removed", res) } setLinks([]hostattach.ObservedAttachment{{ID: hostattach.LinkID("a", "uid"), HostIndex: edgeIndex}}, nil) - res := SweepEBPFVRFTable(context.Background(), k8s, namespace, nodeName, pinDir) + res := SweepEBPFVRFTable(context.Background(), k8s, namespace, nodeName, pinDir, Unfenced) if res.Errors != 0 || res.EBPFVRFEntriesRemoved != 1 || !present(edgeVRFID) || present(otherVRFID) { t.Fatalf("sweep with an edge link = %+v; want only the unused entry removed", res) } setLinks(nil, nil) - res = SweepEBPFVRFTable(context.Background(), k8s, namespace, nodeName, pinDir) + res = SweepEBPFVRFTable(context.Background(), k8s, namespace, nodeName, pinDir, Unfenced) if res.Errors != 0 || res.EBPFVRFEntriesRemoved != 1 || present(edgeVRFID) { t.Fatalf("sweep after the edge link is released = %+v; want its entry removed", res) } diff --git a/internal/gc/vrftables_root_test.go b/internal/gc/vrftables_root_test.go index 7e4bd6545..a38c8f52c 100644 --- a/internal/gc/vrftables_root_test.go +++ b/internal/gc/vrftables_root_test.go @@ -96,7 +96,7 @@ func TestSweepEBPFVRFTable_RewritesStaleTableID(t *testing.T) { t.Fatalf("seed entry: %v", err) } - result := SweepEBPFVRFTable(context.Background(), k8s, "default", vrfTablesNode, pinDir) + result := SweepEBPFVRFTable(context.Background(), k8s, "default", vrfTablesNode, pinDir, Unfenced) if result.Errors != 0 || result.EBPFVRFEntriesRewritten != 1 { t.Errorf("result = %+v, want 1 entry rewritten and no errors", result) } @@ -180,7 +180,7 @@ func TestRecreatedVRFKeepsItsTableWhileAnotherVPCIsCreated(t *testing.T) { t.Fatalf("recreated A's table = %d, %v; want its own table %d", got, err, tableA) } - result := SweepEBPFVRFTable(context.Background(), k8s, "default", vrfTablesNode, pinDir) + result := SweepEBPFVRFTable(context.Background(), k8s, "default", vrfTablesNode, pinDir, Unfenced) if result.Errors != 0 { t.Errorf("result.Errors = %d, want 0", result.Errors) } diff --git a/internal/installer/hostlinkrepair_test.go b/internal/installer/hostlinkrepair_test.go index f4532ace8..74cdc907a 100644 --- a/internal/installer/hostlinkrepair_test.go +++ b/internal/installer/hostlinkrepair_test.go @@ -6,18 +6,32 @@ package installer import ( "errors" + "fmt" "os" "testing" + "go.datum.net/galactic/internal/cni/vpcfence" "go.datum.net/galactic/internal/hostgw" "go.datum.net/galactic/internal/plumbing/ebpf/metrics" ) // TestMain keeps Run's host link repair, which every test calling Run // starts, off the test host's real links and VRF lock. +// TestMain stubs the host link re-enslavement and points the VPC write fence +// at a temporary directory, so tests neither need root to create it nor +// contend with a real node's. func TestMain(m *testing.M) { reenslaveHostLinksFn = func() (hostgw.MasterlessResult, error) { return hostgw.MasterlessResult{}, nil } - os.Exit(m.Run()) + os.Exit(func() int { + dir, err := os.MkdirTemp("", "installer-vpcfence-") + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + defer func() { _ = os.RemoveAll(dir) }() + vpcfence.Dir = dir + return m.Run() + }()) } // hostLinkMetric returns the value of the unlabeled host link series name. diff --git a/internal/installer/installer.go b/internal/installer/installer.go index 695f480e8..42f123bbb 100644 --- a/internal/installer/installer.go +++ b/internal/installer/installer.go @@ -33,6 +33,7 @@ import ( cloudv1alpha1 "go.datum.net/galactic/api/cloud/v1alpha1" bgpv1alpha1 "go.datum.net/galactic/api/network/v1alpha1" + "go.datum.net/galactic/internal/cni/vpcfence" "go.datum.net/galactic/internal/config" "go.datum.net/galactic/internal/gc" "go.datum.net/galactic/internal/hostconf" @@ -1352,7 +1353,8 @@ func Run(ctx context.Context, grpcHealthPort, metricsPort int) error { // router's GC no longer sees them, and this daemon owns their // eBPF rows and the tap teardown. startStaleTapReclaim(ctx, staleTapSem, staleTaps, ebpfState, m.HostLinks) - result := gc.SweepEBPFVRFTable(ctx, ebpfState.k8sClient, ebpfState.namespace, ebpfState.nodeName, attach.PinDir) + result := gc.SweepEBPFVRFTable(ctx, ebpfState.k8sClient, ebpfState.namespace, ebpfState.nodeName, attach.PinDir, + vpcfence.WriteUnlessWorker) logEBPFVRFSweepResult(result) // nptv6_table's sweep is its map's only writer, so it registers diff --git a/internal/installer/sidecarreap.go b/internal/installer/sidecarreap.go index c6bc9c5eb..8ea634399 100644 --- a/internal/installer/sidecarreap.go +++ b/internal/installer/sidecarreap.go @@ -170,7 +170,10 @@ func (r *sidecarReaper) reap(ctx context.Context, k8s client.Client, namespace, var errs []error for _, a := range gone { - if err := deleteIngressAdvertisement(ctx, k8s, namespace, a); err != nil { + // The advertisement is shared state of its VPC on the node. + if err := underAdvertisementFence(ctx, a.name, func() error { + return deleteIngressAdvertisement(ctx, k8s, namespace, a) + }); err != nil { errs = append(errs, err) } } diff --git a/internal/installer/sidecarreturn.go b/internal/installer/sidecarreturn.go index 1448d1cb8..31ec5e47a 100644 --- a/internal/installer/sidecarreturn.go +++ b/internal/installer/sidecarreturn.go @@ -22,6 +22,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" bgpv1alpha1 "go.datum.net/galactic/api/network/v1alpha1" + "go.datum.net/galactic/internal/cni/vpcfence" "go.datum.net/galactic/internal/crdnames" "go.datum.net/galactic/internal/plumbing/ebpf/attach" "go.datum.net/galactic/internal/plumbing/ebpf/ifindexvrfmap" @@ -29,6 +30,7 @@ import ( "go.datum.net/galactic/internal/plumbing/ebpf/tpmap" "go.datum.net/galactic/internal/plumbing/ebpf/uformat" "go.datum.net/galactic/internal/plumbing/ebpf/usidmap" + "go.datum.net/galactic/internal/plumbing/intf" "go.datum.net/galactic/internal/plumbing/nldump" ) @@ -221,7 +223,9 @@ func ensureSidecarReturnPath( var errs []error applied := 0 for _, e := range endpoints { - err := dp.install(block, e) + // The vrf_table row the install writes is shared state of the + // endpoint's VPC on the node. + err := underAdvertisementFence(ctx, e.advName, func() error { return dp.install(block, e) }) rec.Operation(metrics.SidecarReturnOpInstall, err) if err != nil { // Per-endpoint and never fatal to the sweep: one Envoy pod still @@ -651,3 +655,16 @@ func galacticRouterNamesForNode( } return names, nil } + +// underAdvertisementFence runs write under the write fence of the VPC whose +// encoded segment leads the BGPAdvertisement name advName. A name whose +// segment does not decode, which no current writer produces, names no fence, +// and write runs without one. +func underAdvertisementFence(ctx context.Context, advName string, write func() error) error { + segment, _, _ := strings.Cut(advName, "-") + vpc, err := intf.HexToBase62(segment) + if err != nil { + return write() + } + return vpcfence.Write(ctx, vpc, write) +} diff --git a/internal/ownership/ownership.go b/internal/ownership/ownership.go index 69eae449b..4db8c9b07 100644 --- a/internal/ownership/ownership.go +++ b/internal/ownership/ownership.go @@ -46,6 +46,7 @@ package ownership import ( + "context" "encoding/json" "errors" "fmt" @@ -53,6 +54,7 @@ import ( "path/filepath" "regexp" "strings" + "time" "golang.org/x/sys/unix" ) @@ -259,6 +261,28 @@ func (a *Authority) Exclusive(scope string, check, mutate func() error) error { return a.ExclusiveIf(scope, func(Record) error { return check() }, mutate) } +// transitionPoll is how often ExclusiveWait retries a scope whose ownership +// is moving. +const transitionPoll = 50 * time.Millisecond + +// ExclusiveWait is ExclusiveIf, retried while ownership of the scope moves +// until ctx is done, when it returns the last ErrTransitioning joined with +// ctx's error. +func (a *Authority) ExclusiveWait(ctx context.Context, scope string, check func(Record) error, + mutate func() error) error { + for { + err := a.ExclusiveIf(scope, check, mutate) + if !errors.Is(err, ErrTransitioning) { + return err + } + select { + case <-ctx.Done(): + return errors.Join(err, ctx.Err()) + case <-time.After(transitionPoll): + } + } +} + // ExclusiveIf is Exclusive with a check that also reads the scope's record as // it stands under the lock, so a writer can refuse a scope by its owner // without a window in which the owner changes. @@ -353,6 +377,19 @@ const WorkerOwnerPrefix = "galactic-worker:" // WorkerOwned reports whether owner is a galactic-worker incarnation. func WorkerOwned(owner string) bool { return strings.HasPrefix(owner, WorkerOwnerPrefix) } +// ErrWorkerOwned is returned by a writer that leaves a scope alone because a +// galactic-worker incarnation owns it and does that writer's work there. +var ErrWorkerOwned = errors.New("ownership: a galactic-worker incarnation owns the scope") + +// RefuseWorkerOwned is an ExclusiveIf check that refuses a scope a +// galactic-worker incarnation owns with ErrWorkerOwned. +func RefuseWorkerOwned(r Record) error { + if WorkerOwned(r.Owner) { + return fmt.Errorf("%w: %q", ErrWorkerOwned, r.Owner) + } + return nil +} + // Read returns the scope's record. func (a *Authority) Read(scope string) (Record, error) { var out Record From 2e4733ceb44984e985593dca370b2be4916016b2 Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Wed, 18 Nov 2026 19:01:58 -0500 Subject: [PATCH 3/5] feat(worker): Own each node and VPC scope's host state Make Worker the fenced authority for the canonical per-node, per-VPC host state that CNI attachments and service claims share: the host VRF and its table, the SID Argument and BGPVRFInstance, and the vrf_table row (docs/worker ownership.md, D12). - internal/worker/hostscope is Worker's view of that state. Plan reads it without writing. Ensure (nodevpc.Ensure) and Reclaim run inside Worker's commit. AttachmentsOf reads the links enslaved to a VPC's VRF, with their current ifindex and neighbors, at the time of the call. - Worker discovers each VPC from its host VRF and from the fence records a Worker incarnation owns. It takes every scope no other writer owns with Claim or Takeover. A snapshot for a VPC with no claims writes nothing, so adopting baseline node state leaves it unchanged. - A VPC with claims has its canonical state ensured before the delivery modules run. nodevpc.InPlace checks it without locks, so a VPC whose state is in place commits nothing. - A VPC stays in use while Worker holds a claim in it, a link is enslaved to its VRF, or an edge attachment uses it. Once it has been unused for --host-grace (60 s), Worker reclaims the VRF and the BGPVRFInstance as scope owner. gc.ReclaimVPC runs RunGC's evidence for that VPC with the fence already held. The grace keeps the state across an attachment's replacement. - A scope left transitioning by an earlier incarnation is completed even when its VRF does not exist yet. Otherwise the CNI ADD that would create the VRF would wait on it forever. - galactic-worker subscribes to link events, so a CNI attachment joining or leaving a VPC's VRF requeues that VPC. The privileged tests run under task test:unit-root. In TestHostScopeTwoVPCsSameGuestIP, CNI ADDs and Envoy-style socket claims interleave for two VPCs whose guests share IPv4 and IPv6 addresses. After each step the test checks every VPC's table, guest routes, SID and a live VRF-bound socket. The steps are: replace a tap; remove the last tap while a socket claim references the VPC; restart Worker; and replay a stale DEL. The test then runs one router GC pass, which leaves the Worker-owned VPC and reclaims an unowned control VRF. Finally, Worker reclaims the VPC once the claim is released. TestAttachWithWorkerDown attaches a workload while no Worker runs, and also into a scope left between Fence and Adopt. The Worker that then starts converges on what CNI wrote. TestLegacyAdoptionUnchanged builds node state the way the baseline CNI did and compares links, routes, rules, every pinned map row and the BGPVRFInstances before and after Worker adopts them. --- cmd/galactic-worker/root.go | 33 +- cmd/galactic-worker/scrape_test.go | 3 +- docs/cni/gc-cmd-sequence.md | 5 + docs/worker/README.md | 3 + docs/worker/contract.md | 10 +- docs/worker/ownership.md | 39 +- internal/gc/gc_ebpf_test.go | 3 +- internal/gc/reclaimvpc.go | 86 +++ internal/nodevpc/ensure.go | 85 ++- internal/ownership/ownership.go | 15 + internal/ownership/takeover_test.go | 22 +- internal/worker/harness_test.go | 6 +- internal/worker/host_test.go | 207 ++++++ internal/worker/hostscope/hostscope.go | 246 +++++++ internal/worker/hostscope/watch.go | 111 ++++ internal/worker/hostscope_root_test.go | 852 +++++++++++++++++++++++++ internal/worker/reconcile.go | 160 ++++- internal/worker/worker.go | 74 ++- 18 files changed, 1919 insertions(+), 41 deletions(-) create mode 100644 internal/gc/reclaimvpc.go create mode 100644 internal/worker/host_test.go create mode 100644 internal/worker/hostscope/hostscope.go create mode 100644 internal/worker/hostscope/watch.go create mode 100644 internal/worker/hostscope_root_test.go diff --git a/cmd/galactic-worker/root.go b/cmd/galactic-worker/root.go index 29e8c28eb..7d804e3bf 100644 --- a/cmd/galactic-worker/root.go +++ b/cmd/galactic-worker/root.go @@ -27,11 +27,15 @@ import ( workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" "go.datum.net/galactic/internal/config" + "go.datum.net/galactic/internal/edge/maintenance" + "go.datum.net/galactic/internal/gc" "go.datum.net/galactic/internal/logging" "go.datum.net/galactic/internal/metadata" "go.datum.net/galactic/internal/ownership" + "go.datum.net/galactic/internal/plumbing/ebpf/attach" "go.datum.net/galactic/internal/worker" "go.datum.net/galactic/internal/worker/auth" + "go.datum.net/galactic/internal/worker/hostscope" "go.datum.net/galactic/internal/worker/module" "go.datum.net/galactic/internal/worker/sources" "go.datum.net/galactic/internal/worker/store" @@ -66,6 +70,8 @@ type options struct { renewInterval time.Duration claimLimit int resyncInterval time.Duration + pinDir string + hostGrace time.Duration } func (o *options) validate() error { @@ -76,6 +82,9 @@ func (o *options) validate() error { if o.allowlist == "" { errs = append(errs, errors.New("--allowlist is required")) } + if o.hostGrace <= 0 { + errs = append(errs, errors.New("--host-grace must be positive")) + } if o.claimLimit < 1 { errs = append(errs, errors.New("--claim-limit must be at least 1")) } @@ -149,14 +158,26 @@ func run(o *options) error { if err != nil { return fmt.Errorf("create API client: %w", err) } + // Edge attachments keep their VPC's host state; without the guard Worker + // knows of none, so it reclaims nothing (gc.EdgeUsers). + var edge gc.EdgeUsers + if guard, err := maintenance.NewGuard(direct, o.nodeName, o.ownershipDir); err != nil { + slog.Error("edge attachment guard unavailable: Worker will reclaim no VPC host state", "error", err) + edge = maintenance.Unavailable{Err: err} + } else { + edge = guard + } w, err := worker.New(worker.Config{ Node: o.nodeName, Lease: o.lease, RenewInterval: o.renewInterval, - ClaimLimit: o.claimLimit, ResyncInterval: o.resyncInterval, + ClaimLimit: o.claimLimit, ResyncInterval: o.resyncInterval, HostGrace: o.hostGrace, }, worker.Deps{ Store: st, Fence: fence, Sources: sources.Kube{Reader: mgr.GetClient(), Namespace: o.namespace}, Modules: reg, + Host: &hostscope.Kernel{ + K8s: direct, Namespace: o.namespace, Node: o.nodeName, PinDir: o.pinDir, Edge: edge, + }, Auth: &auth.Authenticator{ Reviewer: auth.KubeTokenReviewer{Client: direct}, Pods: auth.CgroupPodResolver{ProcRoot: o.procRoot}, @@ -171,6 +192,13 @@ func run(o *options) error { if err := sources.Watch(ctx, mgr.GetCache(), w.Changed); err != nil { return err } + // A CNI attachment's link joining or leaving a VPC's VRF requeues that + // VPC; the periodic resync covers a lost subscription. + if err := hostscope.Watch(ctx, w.HostChanged, func(err error) { + slog.Error("link event subscription failed; relying on periodic resync", "error", err) + }); err != nil { + return err + } mgrErr := make(chan error, 1) go func() { mgrErr <- mgr.Start(ctx) }() @@ -258,6 +286,9 @@ func newRootCommand() *cobra.Command { f.IntVar(&o.claimLimit, "claim-limit", worker.DefaultClaimLimit, "Claims one caller may hold") f.DurationVar(&o.resyncInterval, "resync-interval", worker.DefaultResyncInterval, "How often every VPC scope is reconciled again") + f.StringVar(&o.pinDir, "pin-dir", attach.PinDir, "bpffs directory of the pinned uSID maps") + f.DurationVar(&o.hostGrace, "host-grace", worker.DefaultHostGrace, + "How long a VPC's host state must go unused before Worker reclaims it") f.Bool("build-info", false, "Print build information and exit") f.BoolP("version", "V", false, "Print version and exit") logOpts.BindFlags(cmd.PersistentFlags()) diff --git a/cmd/galactic-worker/scrape_test.go b/cmd/galactic-worker/scrape_test.go index 35cc96c8a..a4f1ca251 100644 --- a/cmd/galactic-worker/scrape_test.go +++ b/cmd/galactic-worker/scrape_test.go @@ -23,7 +23,7 @@ func TestMetricsScrape(t *testing.T) { func TestOptionsValidate(t *testing.T) { o := &options{nodeName: "node-a", allowlist: "a.yaml", claimLimit: 1, - lease: worker.DefaultLease, renewInterval: worker.DefaultRenewInterval} + lease: worker.DefaultLease, renewInterval: worker.DefaultRenewInterval, hostGrace: worker.DefaultHostGrace} if err := o.validate(); err != nil { t.Fatalf("validate() = %v", err) } @@ -32,6 +32,7 @@ func TestOptionsValidate(t *testing.T) { "no allowlist": func(o *options) { o.allowlist = "" }, "no claims": func(o *options) { o.claimLimit = 0 }, "lease too short": func(o *options) { o.lease = o.renewInterval }, + "no host grace": func(o *options) { o.hostGrace = 0 }, } { bad := *o mutate(&bad) diff --git a/docs/cni/gc-cmd-sequence.md b/docs/cni/gc-cmd-sequence.md index cc8a40616..a80dc4981 100644 --- a/docs/cni/gc-cmd-sequence.md +++ b/docs/cni/gc-cmd-sequence.md @@ -294,6 +294,11 @@ sequenceDiagram If the guard cannot answer, because the fence directory is not mounted, `edgeattachments` cannot be listed or the links cannot be read, GC deletes no VRF or `BGPVRFInstance` that pass and counts an error. +- **A Worker-owned VPC is left to its Worker.** `Reclaim` reads the scope's + owner under the same lock and refuses a VPC whose scope a `galactic-worker` + incarnation owns (`ownership.ErrWorkerOwned`). That Worker reclaims the VPC + itself once it has gone unused, on the same evidence + ([host state ownership](../worker/ownership.md#node-and-vpc-scope)). - **The very first pass waits for the cache to sync.** `root.go` calls `mgr.GetCache().WaitForCacheSync(ctx)` before the initial `RunGC` — an empty, not-yet-populated `BGPAdvertisement`/`BGPRouter` list would diff --git a/docs/worker/README.md b/docs/worker/README.md index 8221f42da..7ece77132 100644 --- a/docs/worker/README.md +++ b/docs/worker/README.md @@ -13,6 +13,9 @@ is built in yet: Local ([#1435](https://github.com/datum-cloud/galactic/issues/1 and Fabric ([#1436](https://github.com/datum-cloud/galactic/issues/1436)) add them, so every capability reports `ModuleNotRunning` until then. `examples/worker-client` is an independent client of the published API. +[#1434](https://github.com/datum-cloud/galactic/issues/1434) makes Worker the +owner of each node and VPC scope, with CNI writing that state under the fence +([ownership](ownership.md#node-and-vpc-scope)). | Page | Contents | | -------------------------------------- | ----------------------------------------------------------------------------------------- | diff --git a/docs/worker/contract.md b/docs/worker/contract.md index 1d4ca3ee8..5662461e6 100644 --- a/docs/worker/contract.md +++ b/docs/worker/contract.md @@ -96,11 +96,11 @@ every client on the branch has moved to it (D7). Worker keeps three states for each claim, and never reports one as another (D2). -| State | What it holds | Durable | -| ---------- | ------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------- | -| Desired | The accepted generation and spec, and the owner. | Yes, under `/var/lib/galactic-worker` | -| Programmed | What Worker wrote for each destination: mode, VRF table, SID, map rows, routes, and the read-back of each write. | Rebuilt from the host after a restart | -| Observed | Readiness evidence Worker did not write: the TAP or veth exists and is up, the neighbor resolves, CNI has attached. | Rebuilt from the host and from CNI attachment events | +| State | What it holds | Durable | +| ---------- | ------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------- | +| Desired | The accepted generation and spec, and the owner. | Yes, under `/var/lib/galactic-worker` | +| Programmed | What Worker wrote for each destination: mode, VRF table, SID, map rows, routes, and the read-back of each write. | Rebuilt from the host after a restart | +| Observed | Readiness evidence Worker did not write: the TAP or veth exists and is up, the neighbor resolves, CNI has attached. | Rebuilt from the host; link events in a VPC's VRF requeue its scope | Status carries `observed_generation` and these conditions, matching the EdgeAttachment condition names where they mean the same thing: diff --git a/docs/worker/ownership.md b/docs/worker/ownership.md index 0ef2c5b04..752764bb6 100644 --- a/docs/worker/ownership.md +++ b/docs/worker/ownership.md @@ -72,7 +72,8 @@ Each step uses the fence operations that exist today (D13). fences and adopts under one lock, so a Worker that dies while taking a scope over cannot block CNI until it returns. 2. **Adoption of preexisting state.** When Worker starts, it takes each scope it - finds (from the VRF devices and the table reservations in `tables.json`). + finds: every host VRF device, and every fence record a Worker incarnation + owns, which covers a transfer left open before CNI created the VRF. It uses `Claim` for an unowned scope, or `Takeover` for one an earlier Worker incarnation owned, including one an older incarnation left between `Fence` and `Adopt`. Both return a ticket that requires `CommitSnapshot`, @@ -96,6 +97,42 @@ Each step uses the fence operations that exist today (D13). create a new authority, as `internal/ownership` requires. Never restore the directory while writers run. +## Node and VPC scope + +[#1434](https://github.com/datum-cloud/galactic/issues/1434) implements the +fenced authority for the node and VPC scope and the first transfer step for +CNI and garbage collection. + +- **CNI writes with `Exclusive`.** galactic-tap and galactic-veth create the VRF + and enslave the host link with `Exclusive` on the VPC's scope. galactic-bgp + does the same for each publish attempt (Argument claim, BGPVRFInstance, + `vrf_table` registration and BGPAdvertisement), its rollback and the DEL + advertisement release (`internal/cni/vpcfence`). The installer's sidecar + return install and advertisement reap use the endpoint's VPC scope. CNI waits + at most 30 s for an open transfer and never waits for Worker. +- **Worker owns the scope.** Worker takes every VPC scope it discovers that no + other writer owns. A VPC with claims gets its canonical state from + `nodevpc.Ensure` inside Worker's commit, before the delivery modules run. + Adopting a scope with no claims writes nothing. +- **References.** A VPC's host state is in use while Worker holds a claim in + it, a link is enslaved to its VRF, or a Galactic Edge attachment uses it. + Worker reclaims the VRF and BGPVRFInstance once the VPC has gone unused for + `--host-grace` (60 s), on the same evidence galactic-router's GC uses + (`gc.ReclaimVPC`). The grace keeps the state across an attachment's + replacement. +- **GC transfer.** galactic-router's GC (`maintenance.Guard.Reclaim`) and the + installer's `vrf_table` repair refuse a scope a Worker incarnation owns + (`ownership.ErrWorkerOwned`) and keep doing their work everywhere else. A + Worker-owned scope is reclaimed only by a running Worker. +- **Observation.** Worker subscribes to link events. A link joining, leaving or + changing in a VPC's VRF requeues that VPC, and `hostscope.AttachmentsOf` + reads each attachment's current ifindex and neighbors when a module asks. + Worker never creates or deletes an attachment's link. + +The router's `egress_route_table` writers (#1436), the galactic-vrf sidecar's +in-pod BGPVRFInstance and backend writes (#1439) and edge host links (#1440) +move in their own Tasks. + ## Defects this transfer removes Two writers bypass any cross-process lock at the branch start. Both become diff --git a/internal/gc/gc_ebpf_test.go b/internal/gc/gc_ebpf_test.go index eed147201..6edeea7fa 100644 --- a/internal/gc/gc_ebpf_test.go +++ b/internal/gc/gc_ebpf_test.go @@ -67,7 +67,8 @@ func gcTestScheme(t *testing.T) *runtime.Scheme { // that as an error. func TestSweepEBPFVRFTable_MissingPinDirIsNoOp(t *testing.T) { k8s := fake.NewClientBuilder().WithScheme(gcTestScheme(t)).Build() - result := SweepEBPFVRFTable(context.Background(), k8s, "default", "node-a", "/sys/fs/bpf/galactic-does-not-exist", Unfenced) + result := SweepEBPFVRFTable(context.Background(), k8s, "default", "node-a", "/sys/fs/bpf/galactic-does-not-exist", + Unfenced) if result.Errors != 0 || result.EBPFVRFEntriesRemoved != 0 { t.Errorf("result = %+v, want a zero-value result (no-op)", result) } diff --git a/internal/gc/reclaimvpc.go b/internal/gc/reclaimvpc.go new file mode 100644 index 000000000..67e091eb8 --- /dev/null +++ b/internal/gc/reclaimvpc.go @@ -0,0 +1,86 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package gc + +import ( + "context" + "errors" + "fmt" + + "sigs.k8s.io/controller-runtime/pkg/client" + + "go.datum.net/galactic/internal/edge/maintenance" +) + +// ReclaimVPC is RunGC's reclamation of one VPC's orphaned BGPVRFInstance and +// host VRF on nodeName, for the process that owns the VPC's write fence and +// calls it with the fence held, inside its own commit. It judges the VPC on +// the same evidence RunGC does: live or unknown BGPAdvertisements, sidecar +// EndpointSlices, edge attachments and links enslaved to the VRF. It deletes +// under the VRF lock, so a link enslaved since collection keeps both. It +// never takes the fence itself. BGPAdvertisements are left to RunGC. +func ReclaimVPC(ctx context.Context, k8s client.Client, namespace, nodeName, vpc string, edge EdgeUsers) ( + CleanupResult, error) { + var result CleanupResult + // Collection reads edge use once for the pass; each delete reads it + // again, as maintenance.Guard does under the fence. + held := &passEdgeUsers{EdgeUsers: heldFence{EdgeUsers: edge}} + want := map[string]struct{}{} + addVPC(want, vpc) + + orphans, err := CollectOrphanedCRDs(ctx, k8s, namespace, nodeName, held) + if err != nil { + return result, fmt.Errorf("collect orphaned BGPVRFInstances: %w", err) + } + var instances []OrphanedCRD + for _, o := range orphans { + if o.Kind == kindBGPVRFInstance && vpcInSet(want, vpcFromName(o.Name)) { + instances = append(instances, o) + } + } + r := RemoveOrphanedCRDs(ctx, k8s, instances, held) + result.OrphanedCRDsRemoved, result.Errors = r.OrphanedCRDsRemoved, r.Errors + + names, err := CollectOrphanedVRFs(ctx, k8s, namespace, nodeName, held) + if err != nil { + return result, fmt.Errorf("collect orphaned VRFs: %w", err) + } + var vrfs []string + for _, name := range names { + if v, ok := vpcFromVRFName(name); ok && vpcInSet(want, v) { + vrfs = append(vrfs, name) + } + } + r = RemoveOrphanedVRFs(ctx, vrfs, held) + result.OrphanedVRFsRemoved, result.Errors = r.OrphanedVRFsRemoved, result.Errors+r.Errors + if result.Errors > 0 { + return result, errors.New("reclaim VPC: some deletions failed") + } + return result, nil +} + +// heldFence is an EdgeUsers for a caller that already holds the VPC's write +// fence: Reclaim checks edge use again and runs reclaim directly, since +// taking the fence again would deadlock on its lock. +type heldFence struct { + EdgeUsers +} + +// Reclaim runs reclaim unless an edge attachment uses vpc, when it returns an +// error wrapping maintenance.ErrInUse. +func (h heldFence) Reclaim(ctx context.Context, vpc string, reclaim func() error) error { + vpcs, err := h.VPCs(ctx) + if err != nil { + return err + } + if vpcInSet(vpcs, vpc) { + return fmt.Errorf("%w: VPC %s", maintenance.ErrInUse, vpc) + } + return reclaim() +} + +// VPCFromVRFName returns the base62 VPC a Galactic host VRF's interface name +// carries, without zero padding, and whether name is one. +func VPCFromVRFName(name string) (string, bool) { return vpcFromVRFName(name) } diff --git a/internal/nodevpc/ensure.go b/internal/nodevpc/ensure.go index aca8c9e6f..b135e58e7 100644 --- a/internal/nodevpc/ensure.go +++ b/internal/nodevpc/ensure.go @@ -70,35 +70,11 @@ type State struct { // Ensure returns ErrNoSRv6Identity, before creating anything, for a node whose // router has no SRv6 identity. func Ensure(ctx context.Context, cfg Config, vpc string) (State, error) { - router, err := LookupRouter(ctx, cfg.K8s, cfg.Node, cfg.Namespace) + p, err := prepare(ctx, cfg, vpc) if err != nil { return State{}, err } - node := attachreg.Node{Locator: router.SRv6Locator, NodeID: router.NodeID} - if !node.Configured() { - return State{}, fmt.Errorf("%w: router %s", ErrNoSRv6Identity, router.Name) - } - block, err := node.Block() - if err != nil { - return State{}, err - } - vpcHex, err := intf.Base62ToHex(vpc) - if err != nil { - return State{}, fmt.Errorf("convert VPC %q to hex: %w", vpc, err) - } - rtValue, err := RouteTarget(int64(router.ASN), vpcHex) - if err != nil { - return State{}, fmt.Errorf("compute route target: %w", err) - } - // The number vpc_attribution_table names a VPC by holds 48 bits, which - // a base62 identifier can exceed. CNI ADD then leaves attribution out - // and forwards as usual; Ensure does too. - owner := attributionOwner{} - if n, err := strconv.ParseUint(vpcHex, 16, 48); err == nil { - owner = attributionOwner{vpc: n, known: true} - } - name := crdnames.BGPVRFInstanceName(vpc, cfg.Node) - want := func(vrfID int32) bgpv1alpha1.BGPVRFInstanceSpec { return InstanceSpec(router.Name, rtValue, vrfID) } + router, node, name, want, block, owner := p.router, p.node, p.name, p.want, p.block, p.owner // A caller that runs Ensure on every pass finds everything in place // almost always, so that is read first, without the node's locks, which @@ -134,6 +110,63 @@ func Ensure(ctx context.Context, cfg Config, vpc string) (State, error) { return state, nil } +// InPlace reports whether all of vpc's state Ensure creates is already on the +// node, reading it without the node's locks and writing nothing. It returns +// ErrNoSRv6Identity for a node whose router has no SRv6 identity. +func InPlace(ctx context.Context, cfg Config, vpc string) (bool, error) { + p, err := prepare(ctx, cfg, vpc) + if err != nil { + return false, err + } + _, ok := current(ctx, cfg, vpc, p.name, p.want, p.block, p.owner) + return ok, nil +} + +// prepared is what Ensure and InPlace derive from the node's router before +// reading or writing vpc's state. +type prepared struct { + router Router + node attachreg.Node + name string + want func(int32) bgpv1alpha1.BGPVRFInstanceSpec + block uint64 + owner attributionOwner +} + +func prepare(ctx context.Context, cfg Config, vpc string) (prepared, error) { + router, err := LookupRouter(ctx, cfg.K8s, cfg.Node, cfg.Namespace) + if err != nil { + return prepared{}, err + } + node := attachreg.Node{Locator: router.SRv6Locator, NodeID: router.NodeID} + if !node.Configured() { + return prepared{}, fmt.Errorf("%w: router %s", ErrNoSRv6Identity, router.Name) + } + block, err := node.Block() + if err != nil { + return prepared{}, err + } + vpcHex, err := intf.Base62ToHex(vpc) + if err != nil { + return prepared{}, fmt.Errorf("convert VPC %q to hex: %w", vpc, err) + } + rtValue, err := RouteTarget(int64(router.ASN), vpcHex) + if err != nil { + return prepared{}, fmt.Errorf("compute route target: %w", err) + } + // The number vpc_attribution_table names a VPC by holds 48 bits, which + // a base62 identifier can exceed. CNI ADD then leaves attribution out + // and forwards as usual; Ensure does too. + owner := attributionOwner{} + if n, err := strconv.ParseUint(vpcHex, 16, 48); err == nil { + owner = attributionOwner{vpc: n, known: true} + } + return prepared{ + router: router, node: node, name: crdnames.BGPVRFInstanceName(vpc, cfg.Node), block: block, owner: owner, + want: func(vrfID int32) bgpv1alpha1.BGPVRFInstanceSpec { return InstanceSpec(router.Name, rtValue, vrfID) }, + }, nil +} + // ensureVRFRow makes the vrf_table row for s's (Block, Argument) map to s's // table. A missing row is written with the veth egress kind. A row that maps // to another table is rewritten with its egress kind kept, as GC's repair diff --git a/internal/ownership/ownership.go b/internal/ownership/ownership.go index 4db8c9b07..5ff6708bd 100644 --- a/internal/ownership/ownership.go +++ b/internal/ownership/ownership.go @@ -390,6 +390,21 @@ func RefuseWorkerOwned(r Record) error { return nil } +// Scopes returns the name of every scope with a record. +func (a *Authority) Scopes() ([]string, error) { + entries, err := os.ReadDir(a.dir) + if err != nil { + return nil, fmt.Errorf("ownership: list %s: %w", a.dir, err) + } + var out []string + for _, e := range entries { + if name, ok := strings.CutSuffix(e.Name(), ".json"); ok && scopeName.MatchString(name) { + out = append(out, name) + } + } + return out, nil +} + // Read returns the scope's record. func (a *Authority) Read(scope string) (Record, error) { var out Record diff --git a/internal/ownership/takeover_test.go b/internal/ownership/takeover_test.go index c45e741bf..f59ea682c 100644 --- a/internal/ownership/takeover_test.go +++ b/internal/ownership/takeover_test.go @@ -6,6 +6,7 @@ package ownership import ( "errors" + "slices" "testing" ) @@ -106,7 +107,7 @@ func TestExclusiveIfSeesTheRecord(t *testing.T) { errOwned := errors.New("owned") before, _ := a.Read(scope) err := a.ExclusiveIf(scope, func(r Record) error { - if r.Owner == "cni" { + if r.Owner != "" { return errOwned } return nil @@ -136,3 +137,22 @@ func TestWorkerOwned(t *testing.T) { }) } } + +// TestScopesListsRecords names every scope with a record, and nothing else +// in the directory. +func TestScopesListsRecords(t *testing.T) { + a := newAuthority(t) + for _, s := range []string{"vpc-a", "vpc-b"} { + if err := a.Exclusive(s, noCheck, func() error { return nil }); err != nil { + t.Fatal(err) + } + } + got, err := a.Scopes() + if err != nil { + t.Fatal(err) + } + slices.Sort(got) + if !slices.Equal(got, []string{"vpc-a", "vpc-b"}) { + t.Fatalf("Scopes = %v, want [vpc-a vpc-b]", got) + } +} diff --git a/internal/worker/harness_test.go b/internal/worker/harness_test.go index 0aef9952b..591ecd470 100644 --- a/internal/worker/harness_test.go +++ b/internal/worker/harness_test.go @@ -28,6 +28,7 @@ import ( workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" "go.datum.net/galactic/internal/ownership" "go.datum.net/galactic/internal/worker/auth" + "go.datum.net/galactic/internal/worker/hostscope" "go.datum.net/galactic/internal/worker/module" "go.datum.net/galactic/internal/worker/module/fake" "go.datum.net/galactic/internal/worker/refusal" @@ -115,6 +116,9 @@ type env struct { cfg Config clock *clock modules func(*fake.Host) []module.Module + // hostState, when set, is the VPCs' canonical host state each + // incarnation reconciles. + hostState hostscope.Host // pauseOnStart pauses the Local module's Plan before Start, so the // first reconcile of a restart holds until the test resumes it. pauseOnStart bool @@ -207,7 +211,7 @@ func (e *env) start() { e.t.Fatal(err) } w, err := New(e.cfg, Deps{Store: e.store, Fence: e.fence, Sources: e.src, Modules: reg, Auth: metaAuth{}, - Now: e.clock.Now}) + Host: e.hostState, Now: e.clock.Now}) if err != nil { e.t.Fatal(err) } diff --git a/internal/worker/host_test.go b/internal/worker/host_test.go new file mode 100644 index 000000000..de4176db8 --- /dev/null +++ b/internal/worker/host_test.go @@ -0,0 +1,207 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package worker + +import ( + "context" + "slices" + "sync" + "testing" + "time" + + "go.datum.net/galactic/internal/ownership" + "go.datum.net/galactic/internal/worker/hostscope" + "go.datum.net/galactic/internal/worker/module" +) + +// fakeHostState is an in-memory hostscope.Host: a VPC is present while it +// has a VRF, and referenced while it has links. +type fakeHostState struct { + mu sync.Mutex + vrfs map[string]bool + links map[string]int + inPlace map[string]bool + ensures map[string]int + reclaims map[string]int + reclaimed []string +} + +func newFakeHostState() *fakeHostState { + return &fakeHostState{vrfs: map[string]bool{}, links: map[string]int{}, inPlace: map[string]bool{}, + ensures: map[string]int{}, reclaims: map[string]int{}} +} + +func (f *fakeHostState) Discover(context.Context) ([]string, error) { + f.mu.Lock() + defer f.mu.Unlock() + out := make([]string, 0, len(f.vrfs)) + for v := range f.vrfs { + out = append(out, v) + } + slices.Sort(out) + return out, nil +} + +func (f *fakeHostState) Plan(_ context.Context, vpc string, claimed bool) (hostscope.Plan, error) { + f.mu.Lock() + defer f.mu.Unlock() + p := hostscope.Plan{Present: f.vrfs[vpc], Ensure: claimed && !f.inPlace[vpc]} + for i := range f.links[vpc] { + p.Links = append(p.Links, vpc+"-link"+string(rune('0'+i))) + } + return p, nil +} + +func (f *fakeHostState) Ensure(_ context.Context, vpc string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.ensures[vpc]++ + f.vrfs[vpc], f.inPlace[vpc] = true, true + return nil +} + +func (f *fakeHostState) Reclaim(_ context.Context, vpc string) (bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.reclaims[vpc]++ + if f.links[vpc] > 0 { + return false, nil + } + delete(f.vrfs, vpc) + delete(f.inPlace, vpc) + f.reclaimed = append(f.reclaimed, vpc) + return true, nil +} + +func (f *fakeHostState) Attachments(context.Context, string) ([]hostscope.Attachment, error) { + return nil, nil +} + +func (f *fakeHostState) set(vpc string, links int) { + f.mu.Lock() + defer f.mu.Unlock() + f.vrfs[vpc], f.links[vpc], f.inPlace[vpc] = true, links, true +} + +func (f *fakeHostState) counts(vpc string) (ensures, reclaims int, present bool) { + f.mu.Lock() + defer f.mu.Unlock() + return f.ensures[vpc], f.reclaims[vpc], f.vrfs[vpc] +} + +func hostEnv(t *testing.T) (*env, *fakeHostState) { + t.Helper() + e := newEnv(t) + h := newFakeHostState() + e.hostState = h + e.cfg.HostGrace = 50 * time.Millisecond + return e, h +} + +func (e *env) fenceOwner(vpc string) ownership.Record { + e.t.Helper() + rec, err := e.fence.Read(module.Scope{VPC: vpc}.Name()) + if err != nil { + e.t.Fatal(err) + } + return rec +} + +// TestHostAdoptsDiscoveredScopes takes every VPC scope found on the host with +// a snapshot that writes nothing, leaving a scope another writer owns alone. +func TestHostAdoptsDiscoveredScopes(t *testing.T) { + e, h := hostEnv(t) + h.set(vpcA, 1) + h.set(vpcB, 1) + tk, err := e.fence.Claim(module.Scope{VPC: vpcB}.Name(), "galactic-edge:node-a:1") + e.mustOK(err) + _, err = e.fence.CommitSnapshot(tk, func() error { return nil }) + e.mustOK(err) + + e.start() + e.eventually("vpcA adopted", func() bool { + r := e.fenceOwner(vpcA) + return r.Owner == e.w.Owner() && !r.Pending + }) + e.idle() + if r := e.fenceOwner(vpcB); r.Owner != "galactic-edge:node-a:1" { + t.Fatalf("scope another writer owns = %+v", r) + } + for _, v := range []string{vpcA, vpcB} { + if ens, rec, present := h.counts(v); ens != 0 || rec != 0 || !present { + t.Fatalf("%s: ensures %d, reclaims %d, present %t; adoption must write nothing", v, ens, rec, present) + } + } + + // A restart takes the scope over from the earlier incarnation. + before := e.fenceOwner(vpcA) + e.crash() + e.start() + e.eventually("vpcA taken over", func() bool { + r := e.fenceOwner(vpcA) + return r.Owner == e.w.Owner() && !r.Pending && r.Generation > before.Generation + }) +} + +// TestHostReclaimsAfterGrace reclaims a VPC's host state only once it has +// gone unused for the grace, and never while a link or claim uses it. +func TestHostReclaimsAfterGrace(t *testing.T) { + e, h := hostEnv(t) + h.set(vpcA, 1) + e.start() + e.idle() + + // A claim keeps the state after the last link goes. + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1"))) + e.waitReady(mkey(vpcA, "svc-a"), 1) + h.set(vpcA, 0) + e.w.HostChanged(vpcA) + e.clock.Advance(time.Hour) + time.Sleep(3 * e.cfg.HostGrace) + e.idle() + if _, rec, present := h.counts(vpcA); rec != 0 || !present { + t.Fatalf("reclaims %d, present %t while a claim uses the VPC", rec, present) + } + + // Released: unused from now, reclaimed once the grace passes. + e.mustOK(e.release(natSA, key(vpcA, "svc-a"), 1)) + e.waitGone(mkey(vpcA, "svc-a")) + e.w.HostChanged(vpcA) + e.idle() + if _, rec, _ := h.counts(vpcA); rec != 0 { + t.Fatalf("reclaimed before the grace") + } + e.clock.Advance(time.Minute) + e.eventually("vpcA reclaimed", func() bool { + _, _, present := h.counts(vpcA) + return !present + }) + e.idle() + e.w.mu.Lock() + _, tracked := e.w.scopes[vpcA] + e.w.mu.Unlock() + if tracked { + t.Fatal("scope still tracked after its VRF was reclaimed") + } +} + +// TestHostEnsuredForClaims ensures the VPC's canonical state for a claim when +// it is missing, and not again while it is in place. +func TestHostEnsuredForClaims(t *testing.T) { + e, h := hostEnv(t) + e.start() + e.mustOK(e.request(natSA, key(vpcB, "svc-b"), 1, veth("guest-b1"))) + e.waitReady(mkey(vpcB, "svc-b"), 1) + e.idle() + ens, _, present := h.counts(vpcB) + if ens == 0 || !present { + t.Fatalf("ensures %d, present %t; want the state ensured for the claim", ens, present) + } + e.w.Changed(vpcB) + e.idle() + if again, _, _ := h.counts(vpcB); again != ens { + t.Fatalf("ensured again (%d → %d) with the state in place", ens, again) + } +} diff --git a/internal/worker/hostscope/hostscope.go b/internal/worker/hostscope/hostscope.go new file mode 100644 index 000000000..67509f84c --- /dev/null +++ b/internal/worker/hostscope/hostscope.go @@ -0,0 +1,246 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package hostscope is Worker's view of one VPC's canonical host state on +// the node: the host VRF and its table, the SID Argument and BGPVRFInstance, +// and the VPC's vrf_table row. CNI attachments and service claims of the VPC +// share that state. Worker owns the VPC's write fence (internal/ownership) +// and writes the state only inside its commits. CNI writes the same state +// with Exclusive whether or not Worker runs. +// +// The links enslaved to the VRF are the attachments CNI created. Worker +// observes them, reading each one's current ifindex and neighbors when asked, +// and never creates or deletes them. +package hostscope + +import ( + "context" + "errors" + "fmt" + "net" + "net/netip" + "sort" + + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" + "sigs.k8s.io/controller-runtime/pkg/client" + + "go.datum.net/galactic/internal/gc" + "go.datum.net/galactic/internal/nodevpc" + "go.datum.net/galactic/internal/plumbing/intf" + "go.datum.net/galactic/internal/plumbing/nldump" + "go.datum.net/galactic/internal/plumbing/vrf" +) + +// Plan is what Worker read of a VPC's host state before a commit, without +// writing. +type Plan struct { + // Present is true when the VPC's host VRF exists. + Present bool + // Links are the names of the links enslaved to the VRF. + Links []string + // EdgeUsed is true when a Galactic Edge attachment uses the VPC. + EdgeUsed bool + // Ensure is true when the VPC has claims and some of its canonical state + // is missing or wrong. + Ensure bool +} + +// Referenced reports whether anything on the node other than Worker's own +// claims uses the VPC's host state. +func (p Plan) Referenced() bool { return len(p.Links) > 0 || p.EdgeUsed } + +// Neighbor is one neighbor entry on an attachment's link. +type Neighbor struct { + Addr netip.Addr + MAC net.HardwareAddr + State int +} + +// Attachment is one link enslaved to a VPC's host VRF, as read from the +// kernel at the time of the call. +type Attachment struct { + Name string + // Index is the link's current ifindex, which changes when CNI replaces + // the attachment. + Index int + // Kind is the link type, such as "tuntap" or "veth". + Kind string + Up bool + Neighbors []Neighbor +} + +// Host reads and writes VPCs' canonical host state. +type Host interface { + // Discover returns every VPC with a host VRF on the node. + Discover(ctx context.Context) ([]string, error) + // Plan reads vpc's host state. claimed tells it whether Worker holds + // claims in vpc, which need the canonical state ensured. + Plan(ctx context.Context, vpc string, claimed bool) (Plan, error) + // Ensure creates whatever is missing of vpc's canonical state and + // changes nothing already in place. Worker calls it inside its commit. + Ensure(ctx context.Context, vpc string) error + // Reclaim removes vpc's VRF and BGPVRFInstance when nothing on the node + // uses them, judged again under the commit, and reports whether the VRF + // is gone. Worker calls it inside its commit, only for a VPC it holds no + // claims in. + Reclaim(ctx context.Context, vpc string) (bool, error) + // Attachments returns the links enslaved to vpc's VRF, read now. + Attachments(ctx context.Context, vpc string) ([]Attachment, error) +} + +// Kernel is the Host of the calling process's network namespace, which must +// be the host's. +type Kernel struct { + // K8s reads and writes BGPRouters and BGPVRFInstances. It should read + // the API server rather than a cache, so an Argument claim sees every + // instance. + K8s client.Client + // Namespace holds the node's BGPRouter and BGPVRFInstances. + Namespace string + // Node is this node. + Node string + // PinDir is the bpffs directory of the pinned uSID maps. + PinDir string + // Edge reports the VPCs Galactic Edge attachments use. + Edge gc.EdgeUsers +} + +var _ Host = (*Kernel)(nil) + +func (k *Kernel) cfg() nodevpc.Config { + return nodevpc.Config{K8s: k.K8s, Namespace: k.Namespace, Node: k.Node, PinDir: k.PinDir} +} + +// Discover returns the VPC of every Galactic host VRF. +func (k *Kernel) Discover(context.Context) ([]string, error) { + links, err := vrf.ListVRFLinks() + if err != nil { + return nil, fmt.Errorf("list VRF links: %w", err) + } + var vpcs []string + for _, l := range links { + if vpc, ok := gc.VPCFromVRFName(l.Name); ok { + vpcs = append(vpcs, vpc) + } + } + sort.Strings(vpcs) + return vpcs, nil +} + +// Plan reads vpc's VRF, its enslaved links and edge use, and, when claimed, +// whether its canonical state is in place. +func (k *Kernel) Plan(ctx context.Context, vpc string, claimed bool) (Plan, error) { + var p Plan + links, err := Enslaved(vpc) + switch { + case errors.Is(err, vrf.ErrNotFound): + case err != nil: + return p, err + default: + p.Present = true + for _, l := range links { + p.Links = append(p.Links, l.Attrs().Name) + } + } + if k.Edge != nil { + vpcs, err := k.Edge.VPCs(ctx) + if err != nil { + return p, fmt.Errorf("find VPCs edge attachments use: %w", err) + } + _, p.EdgeUsed = vpcs[vpc] + } + if claimed { + ok, err := nodevpc.InPlace(ctx, k.cfg(), vpc) + if err != nil { + return p, err + } + p.Ensure = !ok + } + return p, nil +} + +// Ensure runs nodevpc.Ensure for vpc. +func (k *Kernel) Ensure(ctx context.Context, vpc string) error { + _, err := nodevpc.Ensure(ctx, k.cfg(), vpc) + return err +} + +// Reclaim runs gc.ReclaimVPC for vpc and reports whether its VRF is gone. +func (k *Kernel) Reclaim(ctx context.Context, vpc string) (bool, error) { + if _, err := gc.ReclaimVPC(ctx, k.K8s, k.Namespace, k.Node, vpc, k.Edge); err != nil { + return false, err + } + _, err := vrf.TableID(vpc) + switch { + case errors.Is(err, vrf.ErrNotFound): + return true, nil + case err != nil: + return false, err + } + return false, nil +} + +// Attachments returns AttachmentsOf(vpc). +func (k *Kernel) Attachments(_ context.Context, vpc string) ([]Attachment, error) { + return AttachmentsOf(vpc) +} + +// Enslaved returns the links whose master is vpc's host VRF, in the calling +// process's network namespace. The error wraps vrf.ErrNotFound when the VRF +// does not exist. +func Enslaved(vpc string) ([]netlink.Link, error) { + name := intf.GenerateInterfaceNameVRF(vpc) + master, err := netlink.LinkByName(name) + if err != nil { + var nf netlink.LinkNotFoundError + if errors.As(err, &nf) { + return nil, fmt.Errorf("VRF %s: %w", name, vrf.ErrNotFound) + } + return nil, fmt.Errorf("look up VRF %s: %w", name, err) + } + links, err := nldump.LinkList() + if err != nil { + return nil, fmt.Errorf("list links: %w", err) + } + var out []netlink.Link + for _, l := range links { + if l.Attrs().MasterIndex == master.Attrs().Index { + out = append(out, l) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Attrs().Name < out[j].Attrs().Name }) + return out, nil +} + +// AttachmentsOf returns the links enslaved to vpc's host VRF with their +// current ifindex and neighbors, in the calling process's network namespace, +// or none when the VRF does not exist. +func AttachmentsOf(vpc string) ([]Attachment, error) { + links, err := Enslaved(vpc) + if errors.Is(err, vrf.ErrNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + out := make([]Attachment, 0, len(links)) + for _, l := range links { + a := l.Attrs() + att := Attachment{Name: a.Name, Index: a.Index, Kind: l.Type(), Up: a.Flags&net.FlagUp != 0} + neighs, err := netlink.NeighList(a.Index, unix.AF_UNSPEC) + if err != nil { + return nil, fmt.Errorf("list neighbors of %s: %w", a.Name, err) + } + for _, n := range neighs { + addr, ok := netip.AddrFromSlice(n.IP) + if !ok { + continue + } + att.Neighbors = append(att.Neighbors, Neighbor{Addr: addr.Unmap(), MAC: n.HardwareAddr, State: n.State}) + } + out = append(out, att) + } + return out, nil +} diff --git a/internal/worker/hostscope/watch.go b/internal/worker/hostscope/watch.go new file mode 100644 index 000000000..5f37d0c6d --- /dev/null +++ b/internal/worker/hostscope/watch.go @@ -0,0 +1,111 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package hostscope + +import ( + "context" + "fmt" + "sync" + + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" + + "go.datum.net/galactic/internal/gc" +) + +// Watch calls changed with a VPC whenever a link event in the calling +// process's network namespace touches that VPC's host VRF: the VRF itself +// appears or goes, or a link joins, leaves or changes while enslaved to it. +// That is how Worker observes a CNI attachment. It returns once subscribed and +// stops when ctx is done. A lost subscription is reported through errs, after +// which the caller's periodic resync still covers every VPC. +func Watch(ctx context.Context, changed func(vpc string), errs func(error)) error { + updates := make(chan netlink.LinkUpdate, 256) + done := make(chan struct{}) + if err := netlink.LinkSubscribeWithOptions(updates, done, netlink.LinkSubscribeOptions{ + ErrorCallback: errs, + }); err != nil { + return fmt.Errorf("subscribe to link events: %w", err) + } + w := &watcher{vrfs: map[int]string{}, links: map[int]string{}} + if err := w.seed(); err != nil { + close(done) + return err + } + go func() { + defer close(done) + for { + select { + case <-ctx.Done(): + return + case u, ok := <-updates: + if !ok { + return + } + for _, vpc := range w.update(u) { + changed(vpc) + } + } + } + }() + return nil +} + +// watcher maps link events to VPCs. It remembers which VPC each VRF and each +// enslaved link belonged to, so an event for a link that left or was deleted +// still names the VPC it left. +type watcher struct { + mu sync.Mutex + // vrfs maps a host VRF's ifindex to its VPC. + vrfs map[int]string + // links maps an enslaved link's ifindex to its VRF's VPC. + links map[int]string +} + +func (w *watcher) seed() error { + links, err := netlink.LinkList() + if err != nil { + return fmt.Errorf("list links: %w", err) + } + for _, l := range links { + if vpc, ok := gc.VPCFromVRFName(l.Attrs().Name); ok && l.Type() == "vrf" { + w.vrfs[l.Attrs().Index] = vpc + } + } + for _, l := range links { + if vpc, ok := w.vrfs[l.Attrs().MasterIndex]; ok { + w.links[l.Attrs().Index] = vpc + } + } + return nil +} + +// update records u and returns the VPCs it touched. +func (w *watcher) update(u netlink.LinkUpdate) []string { + w.mu.Lock() + defer w.mu.Unlock() + a := u.Attrs() + deleted := u.Header.Type == unix.RTM_DELLINK + var out []string + if vpc, ok := gc.VPCFromVRFName(a.Name); ok && u.Type() == "vrf" { + if deleted { + delete(w.vrfs, a.Index) + } else { + w.vrfs[a.Index] = vpc + } + return append(out, vpc) + } + if prev, ok := w.links[a.Index]; ok { + out = append(out, prev) + delete(w.links, a.Index) + } + if vpc, ok := w.vrfs[a.MasterIndex]; ok && !deleted { + w.links[a.Index] = vpc + if len(out) == 0 || out[0] != vpc { + out = append(out, vpc) + } + } + return out +} diff --git a/internal/worker/hostscope_root_test.go b/internal/worker/hostscope_root_test.go new file mode 100644 index 000000000..2c09416ae --- /dev/null +++ b/internal/worker/hostscope_root_test.go @@ -0,0 +1,852 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package worker + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "net" + "net/netip" + "os" + "path/filepath" + "slices" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/rlimit" + "github.com/containernetworking/plugins/pkg/ns" + "github.com/containernetworking/plugins/pkg/testutils" + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" + discoveryv1 "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sruntime "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + bgpv1alpha1 "go.datum.net/galactic/api/network/v1alpha1" + galacticv1alpha1 "go.datum.net/galactic/api/v1alpha1" + "go.datum.net/galactic/internal/cni/tap" + "go.datum.net/galactic/internal/cni/vpcfence" + "go.datum.net/galactic/internal/cniipam" + "go.datum.net/galactic/internal/crdnames" + "go.datum.net/galactic/internal/edge/hostattach" + "go.datum.net/galactic/internal/edge/maintenance" + "go.datum.net/galactic/internal/gc" + "go.datum.net/galactic/internal/hostgw" + "go.datum.net/galactic/internal/nodevpc" + "go.datum.net/galactic/internal/ownership" + "go.datum.net/galactic/internal/plumbing/ebpf/attach" + "go.datum.net/galactic/internal/plumbing/ebpf/attachreg" + "go.datum.net/galactic/internal/plumbing/ebpf/ifindexvrfmap" + "go.datum.net/galactic/internal/plumbing/ebpf/usidmap" + "go.datum.net/galactic/internal/plumbing/intf" + "go.datum.net/galactic/internal/plumbing/srv6" + "go.datum.net/galactic/internal/plumbing/vrf" + "go.datum.net/galactic/internal/worker/hostscope" + "go.datum.net/galactic/internal/worker/module" +) + +const ( + hsNamespace = "galactic-system" + hsRouter = "node-a-router" + hsLocator = "2001:db8:10::/48" + hsNodeID = 7 + hsASN = 65000 + // hsMTU is the tap MTU the test's CNI ADDs ask for. + hsMTU = 1400 + // envoySA holds Envoy-style socket claims, a service attachment that + // references a VPC's host VRF with no tap of its own. + envoySA = "galactic-system/envoy" + // sockPort is the port of the VRF-bound listener standing in for the + // service's live socket. + sockPort = 9000 +) + +// svcAtt is the service attachment of each VPC's Envoy-style claim, and +// guestAtt the guest it delivers to. +var ( + svcAtt = map[string]string{vpcA: svcA, vpcB: svcB} + guestAtt = map[string]string{vpcA: "guest-a1", vpcB: "guest-b1"} +) + +const ( + svcA = "svc-a" + svcB = "svc-b" +) + +// tapAtt is the VPCAttachment of each VPC's guest tap. Both VPCs give their +// guest the same addresses (guestIPAM). +var tapAtt = map[string]string{vpcA: "a1", vpcB: "b1"} + +// sockAddr is the address every VPC's live socket listens on; both VPCs use +// it, so only the VRF a packet travels in tells the listeners apart. +var sockAddr = netip.MustParseAddr("2001:db8:ee::1") + +var guestIPAM = &cniipam.IPAMResult{ + IPv6Subnet: &net.IPNet{IP: net.ParseIP("2001:db8:a::"), Mask: net.CIDRMask(64, 128)}, + IPv6Gateway: net.ParseIP("2001:db8:a::1"), + IPv4Address: net.ParseIP("198.51.100.2").To4(), + IPv4Gateway: net.ParseIP("198.51.100.1").To4(), +} + +func init() { + testCaps[envoySA] = []string{workerv1alpha1.CapabilityLocalSocket} +} + +// nsHost runs another Host's calls inside a test network namespace, as the +// production Host runs in the host's, whichever goroutine Worker calls it +// from. +type nsHost struct { + netns ns.NetNS + inner hostscope.Host +} + +func (h nsHost) do(fn func() error) error { + var inner error + if err := h.netns.Do(func(ns.NetNS) error { inner = fn(); return nil }); err != nil { + return err + } + return inner +} + +func (h nsHost) Discover(ctx context.Context) (out []string, err error) { + err = h.do(func() error { out, err = h.inner.Discover(ctx); return err }) + return out, err +} + +func (h nsHost) Plan(ctx context.Context, vpc string, claimed bool) (p hostscope.Plan, err error) { + err = h.do(func() error { p, err = h.inner.Plan(ctx, vpc, claimed); return err }) + return p, err +} + +func (h nsHost) Ensure(ctx context.Context, vpc string) error { + return h.do(func() error { return h.inner.Ensure(ctx, vpc) }) +} + +func (h nsHost) Reclaim(ctx context.Context, vpc string) (gone bool, err error) { + err = h.do(func() error { gone, err = h.inner.Reclaim(ctx, vpc); return err }) + return gone, err +} + +func (h nsHost) Attachments(ctx context.Context, vpc string) (out []hostscope.Attachment, err error) { + err = h.do(func() error { out, err = h.inner.Attachments(ctx, vpc); return err }) + return out, err +} + +// hostFixture is one node's kernel, pinned maps and API objects, shared by a +// Worker under test and by the CNI and GC writes the test makes. +type hostFixture struct { + t *testing.T + e *env + netns ns.NetNS + pinDir string + k8s client.Client + guard *maintenance.Guard + // socks are the live VRF-bound listeners, by VPC. + socks map[string]int + node attachreg.Node +} + +func newHostFixture(t *testing.T) *hostFixture { + t.Helper() + requireRoot(t) + if err := rlimit.RemoveMemlock(); err != nil { + t.Fatal(err) + } + netns, err := testutils.NewNS() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = netns.Close() + _ = testutils.UnmountNS(netns) + }) + if err := netns.Do(func(ns.NetNS) error { + lo, err := netlink.LinkByName("lo") + if err != nil { + return err + } + return netlink.LinkSetUp(lo) + }); err != nil { + t.Fatal(err) + } + pinDir := fmt.Sprintf("/sys/fs/bpf/galactic-worker-hostscope-%d", os.Getpid()) + t.Cleanup(func() { _ = os.RemoveAll(pinDir) }) + objs, err := attach.Load(pinDir) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = objs.Close() }) + + scheme := k8sruntime.NewScheme() + utilruntime.Must(bgpv1alpha1.AddToScheme(scheme)) + utilruntime.Must(galacticv1alpha1.AddToScheme(scheme)) + utilruntime.Must(discoveryv1.AddToScheme(scheme)) + k8s := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&bgpv1alpha1.BGPRouter{ + ObjectMeta: metav1.ObjectMeta{Name: hsRouter, Namespace: hsNamespace}, + Spec: bgpv1alpha1.BGPRouterSpec{ + TargetRef: bgpv1alpha1.TargetRef{Kind: "Node", Name: node}, + LocalASN: hsASN, RouterID: "192.0.2.1", SRv6Locator: hsLocator, NodeID: hsNodeID, + AddressFamilies: []bgpv1alpha1.AddressFamily{{AFI: bgpv1alpha1.AFIL2VPN, SAFI: bgpv1alpha1.SAFIEVPN}}, + }, + }).Build() + + nodevpc.ArgumentLockDir = t.TempDir() + e := newEnv(t) + origDir := vpcfence.Dir + vpcfence.Dir = e.fenceDir + t.Cleanup(func() { vpcfence.Dir = origDir }) + e.cfg.HostGrace = 200 * time.Millisecond + guard := &maintenance.Guard{Reader: k8s, NodeName: node, Ownership: e.fence, + HostLinks: func() ([]hostattach.ObservedAttachment, error) { return nil, nil }} + e.hostState = nsHost{netns: netns, inner: &hostscope.Kernel{ + K8s: k8s, Namespace: hsNamespace, Node: node, PinDir: pinDir, Edge: guard, + }} + f := &hostFixture{t: t, e: e, netns: netns, pinDir: pinDir, k8s: k8s, guard: guard, + socks: map[string]int{}, node: attachreg.Node{Locator: hsLocator, NodeID: hsNodeID}} + t.Cleanup(func() { + for _, fd := range f.socks { + _ = unix.Close(fd) + } + _ = netns.Do(func(ns.NetNS) error { + for _, v := range []string{vpcA, vpcB, "vpcC"} { + _ = vrf.Delete(v) + } + return nil + }) + }) + return f +} + +func (f *hostFixture) in(fn func() error) { + f.t.Helper() + if err := f.netns.Do(func(ns.NetNS) error { return fn() }); err != nil { + f.t.Fatal(err) + } +} + +// cniAdd runs a tap-mode CNI ADD for vpc's guest as container cid: the tap +// and its VRF, the guest's host gateway, then galactic-bgp's Argument claim +// and datapath registration. fenced writes the VPC's shared state under its +// write fence, as CNI does now; unfenced is the CNI of the baseline, which +// wrote it directly. +func (f *hostFixture) cniAdd(vpc, cid string, fenced bool) error { + write := func(fn func() error) error { + if fenced { + return vpcfence.Write(context.Background(), vpc, fn) + } + return fn() + } + att := tapAtt[vpc] + return f.netns.Do(func(ns.NetNS) error { + if err := write(func() error { + return vrf.EnsureAndEnslave(vpc, func(*netlink.Vrf) error { + _, err := tap.Add(vpc, att, cid, hsMTU) + return err + }) + }); err != nil { + return fmt.Errorf("tap ADD: %w", err) + } + if err := hostgw.ConfigureHostGateway(vpc, att, guestIPAM, nil); err != nil { + return fmt.Errorf("host gateway: %w", err) + } + return write(func() error { + vpcHex, err := intf.Base62ToHex(vpc) + if err != nil { + return err + } + rt, err := nodevpc.RouteTarget(hsASN, vpcHex) + if err != nil { + return err + } + id, _, err := nodevpc.ClaimArgument(context.Background(), f.k8s, hsNamespace, hsRouter, + crdnames.BGPVRFInstanceName(vpc, node), false, + func() ([]string, error) { return nil, nil }, + func(id int32) bgpv1alpha1.BGPVRFInstanceSpec { return nodevpc.InstanceSpec(hsRouter, rt, id) }, + func(arg int32) error { return attachreg.ReleaseArgument(f.pinDir, f.node, vpc, uint16(arg)) }) + if err != nil { + return fmt.Errorf("claim Argument: %w", err) + } + _, err = attachreg.RegisterDatapath(f.pinDir, attachreg.Registration{ + Node: f.node, VPC: vpc, VPCAttachment: att, InterfaceType: attachreg.InterfaceTypeTap, + Argument: uint16(id), LocalPrefixes: []string{"198.51.100.2/32", "2001:db8:a::/64"}, + }) + return err + }) + }) +} + +// cniDel runs a tap-mode CNI DEL for vpc's guest as container cid and +// reports whether cid still owned the tap. +func (f *hostFixture) cniDel(vpc, cid string) (owned bool) { + f.in(func() error { + var err error + owned, err = tap.Release(vpc, tapAtt[vpc], cid, func(ifindex uint32) error { + return attachreg.UnregisterInterface(f.pinDir, ifindex) + }) + return err + }) + return owned +} + +// listen opens vpc's live socket: a UDP listener bound to the VPC's host VRF +// on sockAddr, which it adds to the VRF device. +func (f *hostFixture) listen(vpc string) { + f.t.Helper() + f.in(func() error { + name := intf.GenerateInterfaceNameVRF(vpc) + link, err := netlink.LinkByName(name) + if err != nil { + return err + } + addr := &netlink.Addr{IPNet: &net.IPNet{IP: sockAddr.AsSlice(), Mask: net.CIDRMask(128, 128)}, + Flags: unix.IFA_F_NODAD} + if err := netlink.AddrReplace(link, addr); err != nil { + return fmt.Errorf("address %s: %w", name, err) + } + fd, err := vrfSocket(name) + if err != nil { + return err + } + if err := unix.Bind(fd, &unix.SockaddrInet6{Port: sockPort, Addr: sockAddr.As16()}); err != nil { + _ = unix.Close(fd) + return fmt.Errorf("bind in %s: %w", name, err) + } + f.socks[vpc] = fd + return nil + }) +} + +func vrfSocket(vrfName string) (int, error) { + fd, err := unix.Socket(unix.AF_INET6, unix.SOCK_DGRAM|unix.SOCK_CLOEXEC, 0) + if err != nil { + return 0, err + } + if err := unix.SetsockoptString(fd, unix.SOL_SOCKET, unix.SO_BINDTODEVICE, vrfName); err != nil { + _ = unix.Close(fd) + return 0, fmt.Errorf("bind to %s: %w", vrfName, err) + } + return fd, nil +} + +// probeSockets sends a datagram to sockAddr inside each VPC's VRF and +// requires the VPC's own live socket, opened before every step, to receive +// it and no other listener to. +func (f *hostFixture) probeSockets(step string) { + f.t.Helper() + for vpc := range f.socks { + msg := []byte(step + "/" + vpc) + f.in(func() error { + fd, err := vrfSocket(intf.GenerateInterfaceNameVRF(vpc)) + if err != nil { + return err + } + defer func() { _ = unix.Close(fd) }() + return unix.Sendto(fd, msg, 0, &unix.SockaddrInet6{Port: sockPort, Addr: sockAddr.As16()}) + }) + buf := make([]byte, 128) + deadline := time.Now().Add(2 * time.Second) + for { + n, _, err := unix.Recvfrom(f.socks[vpc], buf, unix.MSG_DONTWAIT) + if err == nil { + if string(buf[:n]) != string(msg) { + f.t.Fatalf("%s: VPC %s socket received %q, want %q", step, vpc, buf[:n], msg) + } + break + } + if !errors.Is(err, unix.EAGAIN) || time.Now().After(deadline) { + f.t.Fatalf("%s: VPC %s live socket received nothing: %v", step, vpc, err) + } + time.Sleep(5 * time.Millisecond) + } + for other, fd := range f.socks { + if other == vpc { + continue + } + if n, _, err := unix.Recvfrom(fd, buf, unix.MSG_DONTWAIT); err == nil { + f.t.Fatalf("%s: VPC %s's datagram %q reached VPC %s's socket", step, vpc, buf[:n], other) + } + } + } +} + +// vpcState is what the test checks of one VPC after each step. +type vpcState struct { + table uint32 + argument uint16 + sid netip.Addr + // tapIndex is the guest tap's ifindex, or 0 with no tap. + tapIndex int + // guestRoutes are the links the VRF table routes the guest addresses + // through. + guestRoutes []int +} + +func (f *hostFixture) state(vpc string) vpcState { + f.t.Helper() + var s vpcState + f.in(func() error { + table, err := vrf.TableID(vpc) + if err != nil { + return err + } + s.table = table + if l, err := netlink.LinkByName(intf.GenerateInterfaceNameHost(vpc, tapAtt[vpc])); err == nil { + s.tapIndex = l.Attrs().Index + } + for _, fam := range []int{netlink.FAMILY_V4, netlink.FAMILY_V6} { + routes, err := netlink.RouteListFiltered(fam, &netlink.Route{Table: int(table)}, netlink.RT_FILTER_TABLE) + if err != nil { + return err + } + for _, r := range routes { + if r.Dst != nil && (r.Dst.Contains(guestIPAM.IPv4Address) || r.Dst.IP.Equal(guestIPAM.IPv6Subnet.IP)) && + r.Type == unix.RTN_UNICAST { + s.guestRoutes = append(s.guestRoutes, r.LinkIndex) + } + } + } + return nil + }) + var inst bgpv1alpha1.BGPVRFInstance + if err := f.k8s.Get(context.Background(), client.ObjectKey{Namespace: hsNamespace, + Name: crdnames.BGPVRFInstanceName(vpc, node)}, &inst); err != nil { + f.t.Fatalf("BGPVRFInstance of %s: %v", vpc, err) + } + s.argument = uint16(inst.Spec.VRFID) + sid, err := srv6.ComputeSID(hsLocator, hsNodeID, inst.Spec.VRFID, bgpv1alpha1.SRv6FunctionEndDT46) + if err != nil { + f.t.Fatal(err) + } + s.sid = sid + block, err := f.node.Block() + if err != nil { + f.t.Fatal(err) + } + reg, closer, err := usidmap.OpenPinnedRegistry(f.pinDir) + if err != nil { + f.t.Fatal(err) + } + defer func() { _ = closer.Close() }() + row, ok, err := reg.VRF.Get(block, s.argument) + if err != nil || !ok || row.VRFTableID != s.table { + f.t.Fatalf("VPC %s: vrf_table row for Argument %d = %+v (present %t, %v), want table %d", + vpc, s.argument, row, ok, err, s.table) + } + return s +} + +// instances counts the BGPVRFInstances of vpc on the node. +func (f *hostFixture) instances(vpc string) int { + f.t.Helper() + list := &bgpv1alpha1.BGPVRFInstanceList{} + if err := f.k8s.List(context.Background(), list, client.InNamespace(hsNamespace)); err != nil { + f.t.Fatal(err) + } + n := 0 + for _, i := range list.Items { + if i.Name == crdnames.BGPVRFInstanceName(vpc, node) { + n++ + } + } + return n +} + +// checkStep verifies, after a step, the table, route, SID and live socket of +// both VPCs, which use the same guest addresses: each VPC keeps the table and +// SID it had, the two never share either, each guest route goes through the +// VPC's own current tap, and each live socket still receives in its own VRF. +func (f *hostFixture) checkStep(step string, want map[string]vpcState) map[string]vpcState { + f.t.Helper() + got := map[string]vpcState{} + for _, vpc := range []string{vpcA, vpcB} { + s := f.state(vpc) + if n := f.instances(vpc); n != 1 { + f.t.Fatalf("%s: VPC %s has %d BGPVRFInstances, want 1", step, vpc, n) + } + if w, ok := want[vpc]; ok && (w.table != s.table || w.argument != s.argument || w.sid != s.sid) { + f.t.Fatalf("%s: VPC %s moved: table %d → %d, Argument %d → %d, SID %s → %s", + step, vpc, w.table, s.table, w.argument, s.argument, w.sid, s.sid) + } + for _, idx := range s.guestRoutes { + if idx != s.tapIndex { + f.t.Fatalf("%s: VPC %s routes its guest through ifindex %d, not its tap %d", step, vpc, idx, s.tapIndex) + } + } + if (s.tapIndex != 0) != (len(s.guestRoutes) == 2) { + f.t.Fatalf("%s: VPC %s tap %d with guest routes %v, want both families through a live tap or none", + step, vpc, s.tapIndex, s.guestRoutes) + } + got[vpc] = s + } + if got[vpcA].table == got[vpcB].table || got[vpcA].argument == got[vpcB].argument { + f.t.Fatalf("%s: VPCs share host state: %+v and %+v", step, got[vpcA], got[vpcB]) + } + f.probeSockets(step) + return got +} + +// observedIndex returns the ifindex Worker's host reads for vpc's tap now. +func (f *hostFixture) observedIndex(vpc string) int { + f.t.Helper() + atts, err := f.e.hostState.Attachments(context.Background(), vpc) + if err != nil { + f.t.Fatal(err) + } + for _, a := range atts { + if a.Name == intf.GenerateInterfaceNameHost(vpc, tapAtt[vpc]) { + return a.Index + } + } + return 0 +} + +func socketSpec(uid string) *workerv1alpha1.ClaimSpec { + return spec(workerv1alpha1.Handoff_HANDOFF_SOCKET, uid) +} + +// TestHostScopeTwoVPCsSameGuestIP is #1434's acceptance sequence. CNI ADDs +// and Envoy-style socket claims interleave for two VPCs whose guests use the +// same addresses, and every step checks the table, route, SID and live socket +// of both: +// +// 1. Replace one tap. +// 2. Remove the last tap while an Envoy claim references the VPC. +// 3. Restart Worker. +// 4. Replay a stale DEL. +// +// A last step releases the claim and lets Worker, the scope's owner, reclaim +// the unused VPC. +func TestHostScopeTwoVPCsSameGuestIP(t *testing.T) { + f := newHostFixture(t) + e := f.e + e.start() + + var wg sync.WaitGroup + errs := make(chan error, 4) + for _, v := range []string{vpcA, vpcB} { + wg.Add(2) + go func() { defer wg.Done(); errs <- f.cniAdd(v, "c1-"+v, true) }() + go func() { + defer wg.Done() + errs <- e.request(envoySA, key(v, svcAtt[v]), 1, socketSpec(guestAtt[v])) + }() + } + wg.Wait() + close(errs) + for err := range errs { + e.mustOK(err) + } + e.waitReady(mkey(vpcA, svcA), 1) + e.waitReady(mkey(vpcB, svcB), 1) + e.idle() + f.listen(vpcA) + f.listen(vpcB) + base := f.checkStep("interleaved ADDs and claims", nil) + for _, v := range []string{vpcA, vpcB} { + if got := f.observedIndex(v); got != base[v].tapIndex { + t.Fatalf("Worker observes %s's tap at %d, want %d", v, got, base[v].tapIndex) + } + if r := e.fenceOwner(v); r.Owner != e.w.Owner() { + t.Fatalf("VPC %s scope owner = %q, want this Worker", v, r.Owner) + } + } + + // 1. Replace VPC A's tap: the predecessor's DEL, then the successor's + // ADD, which gets a new ifindex. + if !f.cniDel(vpcA, "c1-"+vpcA) { + t.Fatal("DEL of the tap's own container found it owned by another") + } + e.mustOK(f.cniAdd(vpcA, "c2-"+vpcA, true)) + e.idle() + s1 := f.checkStep("replace a tap", base) + if s1[vpcA].tapIndex == base[vpcA].tapIndex { + t.Fatal("the replacement tap kept its predecessor's ifindex") + } + if got := f.observedIndex(vpcA); got != s1[vpcA].tapIndex { + t.Fatalf("Worker observes the replaced tap at %d, want the new ifindex %d", got, s1[vpcA].tapIndex) + } + + // 2. Remove VPC A's last tap while the Envoy claim references the VPC. + // Neither galactic-router's GC, which reclaims a VPC with no tap and no + // advertisement, nor Worker past its grace removes the VPC's state. A + // VRF no writer owns, in the same GC pass, is reclaimed: GC ran. + if !f.cniDel(vpcA, "c2-"+vpcA) { + t.Fatal("DEL of the tap's own container found it owned by another") + } + f.in(func() error { return vrf.Add("vpcC") }) + e.clock.Advance(time.Hour) + e.w.HostChanged(vpcA) + time.Sleep(2 * e.cfg.HostGrace) + e.idle() + f.in(func() error { + res := gc.RunGC(context.Background(), f.k8s, hsNamespace, node, f.guard) + if res.Errors != 0 { + return fmt.Errorf("GC pass reported %d errors", res.Errors) + } + return nil + }) + f.in(func() error { + if _, err := vrf.TableID("vpcC"); !errors.Is(err, vrf.ErrNotFound) { + return fmt.Errorf("GC left the unowned control VRF: %w", err) + } + return nil + }) + s2 := f.checkStep("remove the last tap with an Envoy reference", s1) + if s2[vpcA].tapIndex != 0 { + t.Fatal("VPC A's tap survived its DEL") + } + e.waitReady(mkey(vpcA, svcA), 1) + + // 3. Restart Worker. The new incarnation takes every scope over with a + // snapshot that changes nothing on the host. + before := hostSnapshot(t, f.netns, f.pinDir, f.k8s) + oldOwner := e.w.Owner() + e.crash() + e.start() + for _, v := range []string{vpcA, vpcB} { + e.eventually("scope "+v+" taken over", func() bool { + r := e.fenceOwner(v) + return r.Owner == e.w.Owner() && !r.Pending + }) + } + if e.w.Owner() == oldOwner { + t.Fatal("the restarted Worker reused its owner ID") + } + e.waitReady(mkey(vpcA, svcA), 1) + e.waitReady(mkey(vpcB, svcB), 1) + e.idle() + if after := hostSnapshot(t, f.netns, f.pinDir, f.k8s); !slices.Equal(before, after) { + t.Fatalf("restart changed host state:\n%s", diffLines(before, after)) + } + s3 := f.checkStep("restart Worker", s2) + + // 4. Replay a stale DEL. VPC B's successor container takes the tap over + // first; the predecessor's delayed DEL then leaves the tap, its ifindex + // rows and its route alone. + e.mustOK(f.cniAdd(vpcB, "c2-"+vpcB, true)) + if f.cniDel(vpcB, "c1-"+vpcB) { + t.Fatal("the stale DEL found the successor's tap owned by its predecessor") + } + e.idle() + s4 := f.checkStep("replay a stale DEL", s3) + if s4[vpcB].tapIndex != s3[vpcB].tapIndex { + t.Fatalf("VPC B's tap moved from %d to %d", s3[vpcB].tapIndex, s4[vpcB].tapIndex) + } + table, closer, err := ifindexvrfmap.OpenPinned(f.pinDir) + e.mustOK(err) + row, ok, err := table.Get(uint32(s4[vpcB].tapIndex)) + _ = closer.Close() + if err != nil || !ok || row.Argument != s4[vpcB].argument { + t.Fatalf("ifindex_vrf_table row of VPC B's tap = %+v (present %t, %v), want Argument %d", + row, ok, err, s4[vpcB].argument) + } + + // Release the Envoy claim: VPC A is now unused, and once the grace + // passes Worker reclaims its VRF and BGPVRFInstance as the scope owner. + e.mustOK(e.release(envoySA, key(vpcA, svcA), 1)) + e.waitGone(mkey(vpcA, svcA)) + e.w.HostChanged(vpcA) + e.idle() + e.clock.Advance(time.Hour) + e.eventually("VPC A reclaimed", func() bool { + var gone bool + _ = f.netns.Do(func(ns.NetNS) error { + _, err := vrf.TableID(vpcA) + gone = errors.Is(err, vrf.ErrNotFound) + return nil + }) + return gone && f.instances(vpcA) == 0 + }) + _ = unix.Close(f.socks[vpcA]) + delete(f.socks, vpcA) + if s := f.state(vpcB); s.table != s4[vpcB].table || s.argument != s4[vpcB].argument { + t.Fatalf("reclaiming VPC A changed VPC B: %+v → %+v", s4[vpcB], s) + } + f.probeSockets("reclaim the unused VPC") +} + +// TestAttachWithWorkerDown attaches a workload while no Worker runs, first on +// a scope an earlier incarnation owned and then on one it left between Fence +// and Adopt. Both ADDs succeed, the second once a Worker takes the scope +// over, and the started Worker converges on the state CNI wrote. +func TestAttachWithWorkerDown(t *testing.T) { + f := newHostFixture(t) + e := f.e + dead := ownership.WorkerOwnerPrefix + node + ":dead" + tk, err := e.fence.Claim(module.Scope{VPC: vpcA}.Name(), dead) + e.mustOK(err) + _, err = e.fence.CommitSnapshot(tk, func() error { return nil }) + e.mustOK(err) + tk, err = e.fence.Claim(module.Scope{VPC: vpcB}.Name(), dead) + e.mustOK(err) + _, err = e.fence.CommitSnapshot(tk, func() error { return nil }) + e.mustOK(err) + _, err = e.fence.Fence(module.Scope{VPC: vpcB}.Name()) + e.mustOK(err) + + e.mustOK(f.cniAdd(vpcA, "c1-"+vpcA, true)) + addB := make(chan error, 1) + go func() { addB <- f.cniAdd(vpcB, "c1-"+vpcB, true) }() + select { + case err := <-addB: + t.Fatalf("ADD into a transitioning scope returned %v before any Worker ran", err) + case <-time.After(300 * time.Millisecond): + } + + e.start() + select { + case err := <-addB: + e.mustOK(err) + case <-time.After(10 * time.Second): + t.Fatal("ADD still waiting after Worker took the scope over") + } + for _, v := range []string{vpcA, vpcB} { + e.eventually("scope "+v+" taken over", func() bool { + r := e.fenceOwner(v) + return r.Owner == e.w.Owner() && !r.Pending && !r.Transitioning + }) + } + e.idle() + f.listen(vpcA) + f.listen(vpcB) + base := f.checkStep("attach with Worker down", nil) + + // The claims Worker serves once it returns converge on the VRF and + // Argument CNI created. + e.mustOK(e.request(envoySA, key(vpcA, svcA), 1, socketSpec("guest-a1"))) + e.mustOK(e.request(envoySA, key(vpcB, svcB), 1, socketSpec("guest-b1"))) + e.waitReady(mkey(vpcA, svcA), 1) + e.waitReady(mkey(vpcB, svcB), 1) + e.idle() + f.checkStep("Worker returns", base) +} + +// TestLegacyAdoptionUnchanged builds node state the way the baseline CNI did, +// writing shared state with no fence, then starts Worker. Worker takes every +// scope, and links, routes, rules, every pinned map row and the +// BGPVRFInstances stay byte-for-byte as they were. +func TestLegacyAdoptionUnchanged(t *testing.T) { + f := newHostFixture(t) + e := f.e + for _, v := range []string{vpcA, vpcB} { + e.mustOK(f.cniAdd(v, "c1-"+v, false)) + } + before := hostSnapshot(t, f.netns, f.pinDir, f.k8s) + for _, v := range []string{vpcA, vpcB} { + if r := e.fenceOwner(v); r.Owner != "" || r.Revision != 0 { + t.Fatalf("baseline CNI touched the fence of %s: %+v", v, r) + } + } + + e.start() + for _, v := range []string{vpcA, vpcB} { + e.eventually("scope "+v+" adopted", func() bool { + r := e.fenceOwner(v) + return r.Owner == e.w.Owner() && !r.Pending + }) + } + e.idle() + if after := hostSnapshot(t, f.netns, f.pinDir, f.k8s); !slices.Equal(before, after) { + t.Fatalf("adoption changed host state:\n%s", diffLines(before, after)) + } +} + +// hostSnapshot renders every link, route, rule, pinned map row and +// BGPVRFInstance the test's node holds, sorted. Per-CPU maps, which hold +// only counters, are left out. +func hostSnapshot(t *testing.T, netns ns.NetNS, pinDir string, k8s client.Client) []string { + t.Helper() + var out []string + err := netns.Do(func(ns.NetNS) error { + links, err := netlink.LinkList() + if err != nil { + return err + } + for _, l := range links { + a := l.Attrs() + out = append(out, fmt.Sprintf("link %d %s %s master=%d mtu=%d flags=%s alias=%q", + a.Index, a.Name, l.Type(), a.MasterIndex, a.MTU, a.Flags, a.Alias)) + } + routes, err := netlink.RouteListFiltered(netlink.FAMILY_ALL, &netlink.Route{Table: unix.RT_TABLE_UNSPEC}, + netlink.RT_FILTER_TABLE) + if err != nil { + return err + } + for _, r := range routes { + out = append(out, fmt.Sprintf("route table=%d dst=%s dev=%d type=%d src=%s gw=%s", + r.Table, r.Dst, r.LinkIndex, r.Type, r.Src, r.Gw)) + } + rules, err := netlink.RuleList(netlink.FAMILY_ALL) + if err != nil { + return err + } + for _, r := range rules { + out = append(out, fmt.Sprintf("rule family=%d prio=%d table=%d", r.Family, r.Priority, r.Table)) + } + return nil + }) + if err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(pinDir) + if err != nil { + t.Fatal(err) + } + for _, ent := range entries { + m, err := ebpf.LoadPinnedMap(filepath.Join(pinDir, ent.Name()), nil) + if err != nil { + continue // a pinned program + } + switch m.Type() { + case ebpf.PerCPUHash, ebpf.PerCPUArray, ebpf.LRUCPUHash, ebpf.ProgramArray, ebpf.PerfEventArray, + ebpf.RingBuf: + _ = m.Close() + continue + } + var k, v []byte + it := m.Iterate() + for it.Next(&k, &v) { + out = append(out, fmt.Sprintf("map %s %s=%s", ent.Name(), hex.EncodeToString(k), hex.EncodeToString(v))) + } + if err := it.Err(); err != nil { + t.Fatalf("iterate %s: %v", ent.Name(), err) + } + _ = m.Close() + } + list := &bgpv1alpha1.BGPVRFInstanceList{} + if err := k8s.List(context.Background(), list); err != nil { + t.Fatal(err) + } + for _, i := range list.Items { + out = append(out, fmt.Sprintf("instance %s vrfID=%d rv=%s", i.Name, i.Spec.VRFID, i.ResourceVersion)) + } + sort.Strings(out) + return out +} + +func diffLines(before, after []string) string { + var out strings.Builder + for _, l := range before { + if !slices.Contains(after, l) { + out.WriteString("- " + l + "\n") + } + } + for _, l := range after { + if !slices.Contains(before, l) { + out.WriteString("+ " + l + "\n") + } + } + return out.String() +} diff --git a/internal/worker/reconcile.go b/internal/worker/reconcile.go index 5d10e8912..9d2ffb401 100644 --- a/internal/worker/reconcile.go +++ b/internal/worker/reconcile.go @@ -86,6 +86,9 @@ func (w *Worker) Reconcile(ctx context.Context, vpc string) error { scope := module.Scope{VPC: vpc} claims := w.snapshotScope(vpc) if len(claims) == 0 { + if w.d.Host != nil { + return w.reconcileHost(ctx, vpc) + } w.mu.Lock() delete(w.scopes, vpc) w.mu.Unlock() @@ -97,6 +100,12 @@ func (w *Worker) Reconcile(ctx context.Context, vpc string) error { return err } + ensure, err := w.planClaimedHost(ctx, vpc) + if err != nil { + w.reportFailure(evals, workerv1alpha1.ReasonProgrammingFailed, err) + return err + } + t, err := w.acquire(scope) if err != nil { reason := workerv1alpha1.ReasonProgrammingFailed @@ -120,8 +129,15 @@ func (w *Worker) Reconcile(ctx context.Context, vpc string) error { changes[i] = ch empty = empty && ch.Empty() } - if !empty || t.Resync { + if !empty || ensure || t.Resync { mutate := func() error { + // A snapshot reconciles the complete state, including the host + // state; Ensure changes nothing already in place. + if w.d.Host != nil && (ensure || t.Resync) { + if err := w.d.Host.Ensure(ctx, vpc); err != nil { + return fmt.Errorf("ensure host state: %w", err) + } + } for i, m := range mods { if changes[i].Empty() { continue @@ -169,6 +185,148 @@ func (w *Worker) Reconcile(ctx context.Context, vpc string) error { return w.report(evals, state) } +// planClaimedHost reports whether the canonical host state of vpc, which +// Worker holds claims in, must be ensured. The claims keep the state in use +// whatever else does. +func (w *Worker) planClaimedHost(ctx context.Context, vpc string) (bool, error) { + if w.d.Host == nil { + return false, nil + } + w.mu.Lock() + w.scope(vpc).unreferencedSince = time.Time{} + w.mu.Unlock() + hp, err := w.d.Host.Plan(ctx, vpc, true) + if err != nil { + return false, fmt.Errorf("plan host state: %w", err) + } + return hp.Ensure, nil +} + +// reconcileHost reconciles a scope Worker holds no claims in. It takes the +// scope, adopting with a snapshot commit that writes nothing whatever host +// state CNI or an earlier writer left, and reclaims the VPC's host state once +// nothing has used it for HostGrace. A scope another writer owns is left +// alone. A scope whose VRF is gone is forgotten. +func (w *Worker) reconcileHost(ctx context.Context, vpc string) error { + hp, err := w.d.Host.Plan(ctx, vpc, false) + if err != nil { + return fmt.Errorf("plan host state of VPC %s: %w", vpc, err) + } + if !hp.Present { + // A transfer an earlier incarnation left open refuses every CNI + // write, including the ADD that would create the VRF, so it is + // completed even with nothing on the host. + if err := w.completeTransfer(vpc); err != nil { + return err + } + w.forgetScope(vpc) + return nil + } + reclaim := w.unusedFor(vpc, hp.Referenced()) + + t, err := w.acquire(module.Scope{VPC: vpc}) + if errors.Is(err, errOwnedElsewhere) || errors.Is(err, ownership.ErrTransitioning) || + errors.Is(err, ownership.ErrNotOwner) { + w.log.Debug("leaving host state of a VPC another writer owns", "vpc", vpc, "reason", err) + return nil + } + if err != nil { + return err + } + if !reclaim && !t.Resync { + return nil + } + gone := false + mutate := func() error { + // A claim accepted since the plan keeps the state; its own + // reconcile follows this one. + if !reclaim || w.hasClaims(vpc) { + return nil + } + var err error + gone, err = w.d.Host.Reclaim(ctx, vpc) + return err + } + commit := w.d.Fence.Commit + if t.Resync { + commit = w.d.Fence.CommitSnapshot + } + if _, err := commit(t, mutate); err != nil { + return fmt.Errorf("commit host state of %s: %w", module.Scope{VPC: vpc}.Name(), err) + } + if gone { + w.log.Info("reclaimed unused VPC host state", "vpc", vpc) + w.forgetScope(vpc) + } + return nil +} + +// completeTransfer takes over vpc's scope when it is transitioning away from +// an earlier Worker incarnation, committing a snapshot that writes nothing. +func (w *Worker) completeTransfer(vpc string) error { + scope := module.Scope{VPC: vpc} + rec, err := w.d.Fence.Read(scope.Name()) + if err != nil || !rec.Transitioning || !workerOrUnowned(rec) { + return err + } + t, err := w.acquire(scope) + if err != nil { + return err + } + if _, err := w.d.Fence.CommitSnapshot(t, func() error { return nil }); err != nil { + return fmt.Errorf("commit %s: %w", scope.Name(), err) + } + return nil +} + +// unusedFor records whether vpc's host state is in use, and reports whether +// it has gone unused for HostGrace. While it waits it queues the scope again +// for when the grace ends. +func (w *Worker) unusedFor(vpc string, referenced bool) bool { + w.mu.Lock() + s := w.scope(vpc) + now := w.d.Now() + if referenced { + s.unreferencedSince = time.Time{} + w.mu.Unlock() + return false + } + if s.unreferencedSince.IsZero() { + s.unreferencedSince = now + } + left := w.cfg.HostGrace - now.Sub(s.unreferencedSince) + w.mu.Unlock() + if left > 0 { + w.queue.AddAfter(vpc, left) + return false + } + return true +} + +// hasClaims reports whether Worker holds a claim in vpc. +func (w *Worker) hasClaims(vpc string) bool { + w.mu.Lock() + defer w.mu.Unlock() + for k := range w.claims { + if k.VPC == vpc { + return true + } + } + return false +} + +// forgetScope stops tracking vpc's scope unless Worker holds claims in it. +func (w *Worker) forgetScope(vpc string) { + w.mu.Lock() + defer w.mu.Unlock() + for k := range w.claims { + if k.VPC == vpc { + return + } + } + delete(w.scopes, vpc) +} + // resolve evaluates every claim of a scope against trusted sources and // capability status, withdraws claims whose authority was revoked, and // returns the desired state of each mode. diff --git a/internal/worker/worker.go b/internal/worker/worker.go index 45cd93157..9a9aa6d1f 100644 --- a/internal/worker/worker.go +++ b/internal/worker/worker.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "log/slog" + "strings" "sync" "sync/atomic" "time" @@ -32,6 +33,7 @@ import ( workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" "go.datum.net/galactic/internal/ownership" "go.datum.net/galactic/internal/worker/auth" + "go.datum.net/galactic/internal/worker/hostscope" "go.datum.net/galactic/internal/worker/module" "go.datum.net/galactic/internal/worker/sources" "go.datum.net/galactic/internal/worker/store" @@ -52,6 +54,9 @@ const ( // DefaultResyncInterval is how often every scope is reconciled again to // refresh observed state. DefaultResyncInterval = 30 * time.Second + // DefaultHostGrace is how long a VPC's host state goes unused before + // Worker reclaims it. + DefaultHostGrace = 60 * time.Second ) // Authenticator authenticates the caller of a call. @@ -71,6 +76,10 @@ type Config struct { ResyncInterval time.Duration // Reconcilers is the number of scopes reconciled at once. Reconcilers int + // HostGrace is how long a VPC's host state must go unused, with no + // claim, attachment link or edge attachment, before Worker reclaims it. + // It keeps the state across an attachment's replacement. + HostGrace time.Duration } func (c *Config) defaults() { @@ -95,6 +104,9 @@ func (c *Config) defaults() { if c.Reconcilers == 0 { c.Reconcilers = 4 } + if c.HostGrace == 0 { + c.HostGrace = DefaultHostGrace + } } // Deps are a Worker's collaborators. @@ -103,8 +115,13 @@ type Deps struct { Fence *ownership.Authority Sources sources.Sources Modules *module.Registry - Auth Authenticator - Log *slog.Logger + // Host is the VPCs' canonical host state. With it, Worker owns every + // VPC scope on the node that no other writer owns, ensures the state for + // its claims and reclaims it once unused. Without it, Worker only runs + // its modules for scopes with claims. + Host hostscope.Host + Auth Authenticator + Log *slog.Logger // Now is the clock, time.Now when nil. Now func() time.Time } @@ -127,6 +144,9 @@ type scopeState struct { resyncing bool // repair is set after a failed commit, until a commit succeeds. repair bool + // unreferencedSince is when Worker first found the scope's host state + // unused, or zero while something uses it. + unreferencedSince time.Time } // Worker is the reconciliation core and gRPC service. @@ -224,6 +244,7 @@ func (w *Worker) Start(ctx context.Context) error { for _, s := range scopes { w.queue.Add(s) } + w.discover(ctx) for range w.cfg.Reconcilers { go w.runReconciler(ctx) @@ -233,7 +254,10 @@ func (w *Worker) Start(ctx context.Context) error { w.queue.ShutDown() }() go w.every(ctx, w.cfg.RenewInterval, w.ExpireLeases) - go w.every(ctx, w.cfg.ResyncInterval, func() { w.Changed("") }) + go w.every(ctx, w.cfg.ResyncInterval, func() { + w.discover(ctx) + w.Changed("") + }) return nil } @@ -257,6 +281,50 @@ func (w *Worker) Changed(vpc string) { } } +// HostChanged requeues vpc's scope after a change to its host state that +// Worker did not make, such as a CNI attachment's link joining or leaving +// the VPC's VRF. It starts tracking the scope when Worker did not know it. +func (w *Worker) HostChanged(vpc string) { + if w.d.Host == nil { + return + } + w.mu.Lock() + w.scope(vpc) + w.mu.Unlock() + w.queue.Add(vpc) +} + +// discover starts tracking, and queues, every VPC with host state on the +// node. +func (w *Worker) discover(ctx context.Context) { + if w.d.Host == nil { + return + } + vpcs, err := w.d.Host.Discover(ctx) + if err != nil { + w.log.Error("discover VPC host state", "error", err) + return + } + // A scope a Worker incarnation owns or left mid-transfer may have no + // VRF, such as one whose CNI ADD waits for the transfer to finish. + scopes, err := w.d.Fence.Scopes() + if err != nil { + w.log.Error("list VPC write fence scopes", "error", err) + } + for _, name := range scopes { + vpc, ok := strings.CutPrefix(name, "vpc-") + if !ok { + continue + } + if rec, err := w.d.Fence.Read(name); err == nil && ownership.WorkerOwned(rec.Owner) { + vpcs = append(vpcs, vpc) + } + } + for _, vpc := range vpcs { + w.HostChanged(vpc) + } +} + func (w *Worker) every(ctx context.Context, d time.Duration, fn func()) { t := time.NewTicker(d) defer t.Stop() From 91a706fc35788f98b278f34509f7e2bfbd557211 Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Wed, 18 Nov 2026 19:40:14 -0500 Subject: [PATCH 4/5] docs(cni): Say what the VPC fence wait does not bound vpcfence.Wait bounds only the retry while ownership of a scope moves. The scope's flock blocks without a deadline, so a lock holder that never finishes stalls a CNI ADD past the 30 s wait. Say so in the constant's comment. --- internal/cni/vpcfence/vpcfence.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/cni/vpcfence/vpcfence.go b/internal/cni/vpcfence/vpcfence.go index 3b143735e..9f72a2a2c 100644 --- a/internal/cni/vpcfence/vpcfence.go +++ b/internal/cni/vpcfence/vpcfence.go @@ -24,7 +24,9 @@ import ( var Dir = ownership.DefaultDir // Wait bounds how long Write waits while ownership of the scope moves, which -// takes one locked file write. +// takes one locked file write. It does not bound the wait for the scope's +// lock itself: a writer that holds the lock without finishing stalls Write +// until it does, as the fence requires of every bounded mutation. const Wait = 30 * time.Second // Write runs mutate under vpc's write fence, waiting up to Wait, or until ctx From c94697de5de25c0df4619d79cb7d6eb0a1ab7df6 Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Wed, 18 Nov 2026 20:02:29 -0500 Subject: [PATCH 5/5] test(worker): Wait for Worker goroutines when a test incarnation stops The test env's crash cancelled the Worker and returned while a reconcile could still be writing fence records. The test's temporary directory cleanup then raced that write and failed with "directory not empty". This failed TestClaimLimit in the privileged CI job on two consecutive runs, and a loaded local run reproduces it on the base commit. Worker.Wait returns once every goroutine Start started has returned. crash calls it after cancelling, so an incarnation's late writes land before the next incarnation starts or the test removes its state. --- internal/worker/harness_test.go | 7 ++++++- internal/worker/worker.go | 21 ++++++++++++++------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/internal/worker/harness_test.go b/internal/worker/harness_test.go index 591ecd470..53dd43143 100644 --- a/internal/worker/harness_test.go +++ b/internal/worker/harness_test.go @@ -238,7 +238,8 @@ func (e *env) start() { } // crash stops the incarnation without any cleanup, as a killed process -// would. Host, store and fence state stay. +// would, and waits for its goroutines to return. Host, store and fence state +// stay. func (e *env) crash() { if e.cancel == nil { return @@ -251,6 +252,10 @@ func (e *env) crash() { e.local.Resume() e.local.ResumePlan() } + // A killed process writes nothing more; waiting for the reconcile in + // progress keeps its late writes out of the next incarnation's run and + // the test's temporary directories. + e.w.Wait() } // ctx returns a call context for caller with a live session. diff --git a/internal/worker/worker.go b/internal/worker/worker.go index 9a9aa6d1f..1009f1e62 100644 --- a/internal/worker/worker.go +++ b/internal/worker/worker.go @@ -171,6 +171,7 @@ type Worker struct { shutting atomic.Bool started atomic.Bool + wg sync.WaitGroup } // New returns a Worker. Start loads desired state and starts reconciling. @@ -247,20 +248,26 @@ func (w *Worker) Start(ctx context.Context) error { w.discover(ctx) for range w.cfg.Reconcilers { - go w.runReconciler(ctx) + w.wg.Go(func() { w.runReconciler(ctx) }) } - go func() { + w.wg.Go(func() { <-ctx.Done() w.queue.ShutDown() - }() - go w.every(ctx, w.cfg.RenewInterval, w.ExpireLeases) - go w.every(ctx, w.cfg.ResyncInterval, func() { - w.discover(ctx) - w.Changed("") + }) + w.wg.Go(func() { w.every(ctx, w.cfg.RenewInterval, w.ExpireLeases) }) + w.wg.Go(func() { + w.every(ctx, w.cfg.ResyncInterval, func() { + w.discover(ctx) + w.Changed("") + }) }) return nil } +// Wait returns once every goroutine Start started has returned, which they do +// after Start's context is done and any reconcile in progress has finished. +func (w *Worker) Wait() { w.wg.Wait() } + // Shutdown makes every later mutating call fail with UNAVAILABLE and // ShuttingDown. Forwarding continues: the datapath does not call Worker. func (w *Worker) Shutdown() { w.shutting.Store(true) }