Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/galactic-edge/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,12 +42,12 @@ import (
"go.datum.net/galactic/internal/edge/agent"
"go.datum.net/galactic/internal/edge/diag"
"go.datum.net/galactic/internal/edge/hostattach"
"go.datum.net/galactic/internal/edge/ownership"
"go.datum.net/galactic/internal/edge/route"
"go.datum.net/galactic/internal/edge/timing"
"go.datum.net/galactic/internal/logging"
"go.datum.net/galactic/internal/metadata"
"go.datum.net/galactic/internal/nodevpc"
"go.datum.net/galactic/internal/ownership"
"go.datum.net/galactic/internal/plumbing/ebpf/attach"
"go.datum.net/galactic/internal/plumbing/sysctl"
)
Expand Down
2 changes: 1 addition & 1 deletion cmd/galactic-router/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,11 @@ import (
"go.datum.net/galactic/internal/config"
"go.datum.net/galactic/internal/controller"
"go.datum.net/galactic/internal/edge/maintenance"
"go.datum.net/galactic/internal/edge/ownership"
"go.datum.net/galactic/internal/gc"
"go.datum.net/galactic/internal/hash"
"go.datum.net/galactic/internal/logging"
"go.datum.net/galactic/internal/metadata"
"go.datum.net/galactic/internal/ownership"
"go.datum.net/galactic/internal/plumbing/ebpf/attach"
"go.datum.net/galactic/internal/plumbing/ebpf/vipxlatmap"
"go.datum.net/galactic/internal/plumbing/loaddr"
Expand Down
35 changes: 33 additions & 2 deletions cmd/galactic-worker/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,15 @@ import (

workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1"
"go.datum.net/galactic/internal/config"
"go.datum.net/galactic/internal/edge/ownership"
"go.datum.net/galactic/internal/edge/maintenance"
"go.datum.net/galactic/internal/gc"
"go.datum.net/galactic/internal/logging"
"go.datum.net/galactic/internal/metadata"
"go.datum.net/galactic/internal/ownership"
"go.datum.net/galactic/internal/plumbing/ebpf/attach"
"go.datum.net/galactic/internal/worker"
"go.datum.net/galactic/internal/worker/auth"
"go.datum.net/galactic/internal/worker/hostscope"
"go.datum.net/galactic/internal/worker/module"
"go.datum.net/galactic/internal/worker/sources"
"go.datum.net/galactic/internal/worker/store"
Expand Down Expand Up @@ -66,6 +70,8 @@ type options struct {
renewInterval time.Duration
claimLimit int
resyncInterval time.Duration
pinDir string
hostGrace time.Duration
}

func (o *options) validate() error {
Expand All @@ -76,6 +82,9 @@ func (o *options) validate() error {
if o.allowlist == "" {
errs = append(errs, errors.New("--allowlist is required"))
}
if o.hostGrace <= 0 {
errs = append(errs, errors.New("--host-grace must be positive"))
}
if o.claimLimit < 1 {
errs = append(errs, errors.New("--claim-limit must be at least 1"))
}
Expand Down Expand Up @@ -149,14 +158,26 @@ func run(o *options) error {
if err != nil {
return fmt.Errorf("create API client: %w", err)
}
// Edge attachments keep their VPC's host state; without the guard Worker
// knows of none, so it reclaims nothing (gc.EdgeUsers).
var edge gc.EdgeUsers
if guard, err := maintenance.NewGuard(direct, o.nodeName, o.ownershipDir); err != nil {
slog.Error("edge attachment guard unavailable: Worker will reclaim no VPC host state", "error", err)
edge = maintenance.Unavailable{Err: err}
} else {
edge = guard
}
w, err := worker.New(worker.Config{
Node: o.nodeName, Lease: o.lease, RenewInterval: o.renewInterval,
ClaimLimit: o.claimLimit, ResyncInterval: o.resyncInterval,
ClaimLimit: o.claimLimit, ResyncInterval: o.resyncInterval, HostGrace: o.hostGrace,
}, worker.Deps{
Store: st,
Fence: fence,
Sources: sources.Kube{Reader: mgr.GetClient(), Namespace: o.namespace},
Modules: reg,
Host: &hostscope.Kernel{
K8s: direct, Namespace: o.namespace, Node: o.nodeName, PinDir: o.pinDir, Edge: edge,
},
Auth: &auth.Authenticator{
Reviewer: auth.KubeTokenReviewer{Client: direct},
Pods: auth.CgroupPodResolver{ProcRoot: o.procRoot},
Expand All @@ -171,6 +192,13 @@ func run(o *options) error {
if err := sources.Watch(ctx, mgr.GetCache(), w.Changed); err != nil {
return err
}
// A CNI attachment's link joining or leaving a VPC's VRF requeues that
// VPC; the periodic resync covers a lost subscription.
if err := hostscope.Watch(ctx, w.HostChanged, func(err error) {
slog.Error("link event subscription failed; relying on periodic resync", "error", err)
}); err != nil {
return err
}

mgrErr := make(chan error, 1)
go func() { mgrErr <- mgr.Start(ctx) }()
Expand Down Expand Up @@ -258,6 +286,9 @@ func newRootCommand() *cobra.Command {
f.IntVar(&o.claimLimit, "claim-limit", worker.DefaultClaimLimit, "Claims one caller may hold")
f.DurationVar(&o.resyncInterval, "resync-interval", worker.DefaultResyncInterval,
"How often every VPC scope is reconciled again")
f.StringVar(&o.pinDir, "pin-dir", attach.PinDir, "bpffs directory of the pinned uSID maps")
f.DurationVar(&o.hostGrace, "host-grace", worker.DefaultHostGrace,
"How long a VPC's host state must go unused before Worker reclaims it")
f.Bool("build-info", false, "Print build information and exit")
f.BoolP("version", "V", false, "Print version and exit")
logOpts.BindFlags(cmd.PersistentFlags())
Expand Down
3 changes: 2 additions & 1 deletion cmd/galactic-worker/scrape_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ func TestMetricsScrape(t *testing.T) {

func TestOptionsValidate(t *testing.T) {
o := &options{nodeName: "node-a", allowlist: "a.yaml", claimLimit: 1,
lease: worker.DefaultLease, renewInterval: worker.DefaultRenewInterval}
lease: worker.DefaultLease, renewInterval: worker.DefaultRenewInterval, hostGrace: worker.DefaultHostGrace}
if err := o.validate(); err != nil {
t.Fatalf("validate() = %v", err)
}
Expand All @@ -32,6 +32,7 @@ func TestOptionsValidate(t *testing.T) {
"no allowlist": func(o *options) { o.allowlist = "" },
"no claims": func(o *options) { o.claimLimit = 0 },
"lease too short": func(o *options) { o.lease = o.renewInterval },
"no host grace": func(o *options) { o.hostGrace = 0 },
} {
bad := *o
mutate(&bad)
Expand Down
13 changes: 13 additions & 0 deletions config/galactic-cni/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,19 @@ lock path is a constant every one of those processes shares. Without the
mount, the repair would lock a file only it can see, and could re-enslave a
link at the same moment an ADD or GC acts on its VRF.

## The `vpc-ownership` mount

`/var/lib/galactic-edge/ownership` holds the node's VPC write fence
(`internal/ownership`). Host-executed CNI writes a VPC's shared host state
under it: the VRF, the SID Argument and BGPVRFInstance, `vrf_table` rows and
the attachment's BGPAdvertisement. `credential-refresh` writes under it too,
for the ingress sidecar's return path, its reaper and the `vrf_table` repair.
The repair leaves a VPC whose scope galactic-worker owns to that Worker.
galactic-router, galactic-edge and galactic-worker share the same directory.
Like `vrf-lock-dir`, it is mounted at its own host-absolute path, because
every writer must lock the same files. On a missing mount the writes fence
against a private, empty directory and no longer exclude the other writers.

## The `cni-net-dir` mount

`credential-refresh` mounts the host's `/etc/cni/net.d` writable, as the
Expand Down
10 changes: 10 additions & 0 deletions config/galactic-cni/daemonset.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,12 @@ spec:
# stranded tap or veth under it. See config/galactic-cni/README.md.
- name: vrf-lock-dir
mountPath: /var/lib/cni/galactic-vrf
# The node's VPC write fence (internal/ownership), shared with
# host-executed CNI, galactic-router, galactic-edge and
# galactic-worker. The sidecar return path, its reaper and the
# vrf_table repair write a VPC's shared state only under it.
- name: vpc-ownership
mountPath: /var/lib/galactic-edge/ownership
# BPF map creation under load (image pull + verifier work on a
# slow node) can outrun readinessProbe/livenessProbe's short
# initialDelaySeconds, risking a liveness-triggered restart loop
Expand Down Expand Up @@ -225,6 +231,10 @@ spec:
hostPath:
path: /var/lib/cni/galactic-vrf
type: DirectoryOrCreate
- name: vpc-ownership
hostPath:
path: /var/lib/galactic-edge/ownership
type: DirectoryOrCreate
- name: bpf-fs
# Host bpffs for pinned-map continuity; `Directory` (not
# DirectoryOrCreate) and its SECURITY NOTE are documented in
Expand Down
10 changes: 5 additions & 5 deletions config/galactic-router/base/daemonset.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -164,11 +164,11 @@ spec:
# DaemonSet regardless of this mount's presence.
- name: cni-state
mountPath: /var/lib/cni
# The node's write fence, shared with galactic-edge
# (internal/edge/ownership). GC deletes a VPC's VRF and
# BGPVRFInstance only under that VPC's fence, after checking no
# edge attachment uses it. Without this mount GC reclaims
# neither.
# The node's VPC write fence (internal/ownership), shared with
# CNI, galactic-edge and galactic-worker. GC deletes a VPC's VRF
# and BGPVRFInstance only under that VPC's fence, after checking
# no edge attachment uses it and no galactic-worker owns it.
# Without this mount GC reclaims neither.
- name: edge-ownership
mountPath: /var/lib/galactic-edge/ownership
- name: gobgp-api
Expand Down
7 changes: 6 additions & 1 deletion docs/cni/gc-cmd-sequence.md
Original file line number Diff line number Diff line change
Expand Up @@ -286,14 +286,19 @@ sequenceDiagram
counts a VPC as used while an `EdgeAttachment` on this node is not
`Released`, read uncached, or a host link stamped
`galactic-edge:hostattach:` for it remains. Each delete runs inside
`Reclaim`, which takes the VPC's write fence (`internal/edge/ownership`,
`Reclaim`, which takes the VPC's write fence (`internal/ownership`,
`/var/lib/galactic-edge/ownership`) that `galactic-edge` holds while it
programs, checks again and deletes only if nothing uses the VPC. An
attachment admitted after collection therefore keeps the VRF, and a change
`galactic-edge` prepared before the delete is refused and prepared again.
If the guard cannot answer, because the fence directory is not mounted,
`edgeattachments` cannot be listed or the links cannot be read, GC deletes
no VRF or `BGPVRFInstance` that pass and counts an error.
- **A Worker-owned VPC is left to its Worker.** `Reclaim` reads the scope's
owner under the same lock and refuses a VPC whose scope a `galactic-worker`
incarnation owns (`ownership.ErrWorkerOwned`). That Worker reclaims the VPC
itself once it has gone unused, on the same evidence
([host state ownership](../worker/ownership.md#node-and-vpc-scope)).
- **The very first pass waits for the cache to sync.** `root.go` calls
`mgr.GetCache().WaitForCacheSync(ctx)` before the initial `RunGC` — an
empty, not-yet-populated `BGPAdvertisement`/`BGPRouter` list would
Expand Down
8 changes: 4 additions & 4 deletions docs/edge/provider-contract.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ This document is the contract the [Galactic Edge foundation](../designs/galactic
| Node packaging | A new `galactic-edge` DaemonSet on nodes labeled `galactic.datumapis.com/node=edge` and `galactic.datumapis.com/edge=enabled`. The `edge=enabled` label is new, and [node labels](../operations/node-labels.md) gains it when the DaemonSet ships. `galactic-edge` is the only process that changes host state for an edge attachment. It writes only the CNI datapath's maps under `/sys/fs/bpf/galactic`, and never opens `/sys/fs/bpf/galactic-edge`, which is `galactic-gateway`'s pin directory despite the name (`internal/plumbing/ebpf/edgeattach/attach.go:38`). |
| Provider privilege | None on the host. A provider pod gets no host namespace, hostPath, `NET_ADMIN` or BPF access. It opens the packet device the node agent created in its own network namespace. |
| Claim store | The cell's API server, compare-and-swap on `resourceVersion` with a UID precondition, as `internal/edge/claim` requires. Claims never span cells. |
| Node-local write fence | `internal/edge/ownership`, with its state and lock files in the hostPath `/var/lib/galactic-edge/ownership`, shared by every process that writes a fenced scope on the node. |
| Node-local write fence | `internal/ownership`, with its state and lock files in the hostPath `/var/lib/galactic-edge/ownership`, shared by every process that writes a fenced scope on the node. |
| Forwarding gate | A new map of the CNI datapath, with one row per edge host link and per accepted prefix holding a `CLOCK_BOOTTIME` deadline. The datapath compares it with `bpf_ktime_get_boot_ns` on every packet, the pattern `serviceroutemap` already uses for authorization deadlines. |
| Allocation | Attachment addresses come from the service's client pool, allocated by the leader of `galactic-controller`. Delegated client prefixes are routes the provider requests inside that pool; the prefix conflict check makes them exclusive. An edge attachment joins the VPC's canonical host VRF and allocates no host table or uSID Argument of its own. |
| Node VPC state | When an attachment is its gVPC's only user on a node, `galactic-edge` creates the gVPC's canonical host VRF, the node's `BGPVRFInstance` for it and its `vrf_table` row, as CNI ADD does, through `nodevpc.Ensure` under the same node locks. `galactic-router` GC stays the only remover: it counts edge attachments as users and deletes the state once nothing on the node uses the gVPC. The CNI, the ingress sidecar and GC keep their roles, so no second reference count exists (#1288). |
Expand Down Expand Up @@ -124,7 +124,7 @@ A ValidatingAdmissionPolicy reads only its own parameter object, so the provider

- Every create uses a derived name. `AlreadyExists` is answered by reading the object and comparing its immutable fields; a mismatch is refused with `IdentityConflict`, never overwritten.
- Claim writes are compare-and-swap. A lost reply is recovered as `claim.Instance.Renew` does: the writer rereads, and validity always runs from the start of the latest call.
- Node mutations go through `ownership.Authority.Commit` (`internal/edge/ownership/ownership.go:162`), which refuses a ticket issued before another commit. A retry built from an old view cannot undo newer state.
- Node mutations go through `ownership.Authority.Commit` (`internal/ownership/ownership.go:162`), which refuses a ticket issued before another commit. A retry built from an old view cannot undo newer state.

### Readiness

Expand Down Expand Up @@ -300,7 +300,7 @@ A drain stops new routes and attachments for its scope at once. Forwarding conti
| Provider process restarts in its pod | New holder ID. It takes over its own instance lease after a release or `TakeoverWait`, then core re-accepts its routes at once (see [Acceptance](#acceptance)). |
| Provider pod replaced, same node | New pod UID and `EdgeInstance`. Routes move by takeover. The old attachment drains. |
| Replacement on another node | As above. The old node's gate closes at its own deadline even if that node cannot reach the API. |
| `galactic-edge` restarts | New ownership owner ID, `Fence`, `Adopt`, then a snapshot commit (`internal/edge/ownership/ownership.go:212`, `:226`, `:169`). It extends no deadline until it has read a renewal newer than the one applied. |
| `galactic-edge` restarts | New ownership owner ID, `Fence`, `Adopt`, then a snapshot commit (`internal/ownership/ownership.go:212`, `:226`, `:169`). It extends no deadline until it has read a renewal newer than the one applied. |
| `galactic-router` restarts | Advertises an edge prefix only while its gate deadline, read from the pinned map, is in the future. |
| Node reboots | Pinned maps and leases from the previous boot are void: the boot ID changed. Every claim on that node is reacquired through the takeover rules. |

Expand Down Expand Up @@ -352,7 +352,7 @@ The revocation bound works like this. `galactic-edge` extends a gate only after

## Foundation resources and their writers

Foundation-owned host state is what `galactic-edge` creates for an attachment, plus the shared state it writes into. Each gVPC on the node is one scope of `internal/edge/ownership`, which `galactic-edge` owns and writes only in snapshot commits. A writer that can remove shared state an edge attachment uses does so only through `maintenance.Guard.Reclaim` (`internal/edge/maintenance`): under the scope's lock (`ownership.Authority.Exclusive`), and never between `Fence` and `Adopt`, it checks again that no edge attachment uses the gVPC; a refused check writes nothing, and a write makes every ticket `galactic-edge` holds for the scope stale, so a change prepared before it is prepared again. A gVPC is in use while the API has an `EdgeAttachment` on the node for it that is not `Released`, which covers one still being prepared, or the host has a link stamped for it. Every other writer in the table below either writes only rows an edge attachment never uses, or only adds rows; the table gives the reason for each. No lab or end-to-end environment runs `galactic-edge` yet.
Foundation-owned host state is what `galactic-edge` creates for an attachment, plus the shared state it writes into. Each gVPC on the node is one scope of `internal/ownership`, which `galactic-edge` owns and writes only in snapshot commits. A writer that can remove shared state an edge attachment uses does so only through `maintenance.Guard.Reclaim` (`internal/edge/maintenance`): under the scope's lock (`ownership.Authority.Exclusive`), and never between `Fence` and `Adopt`, it checks again that no edge attachment uses the gVPC; a refused check writes nothing, and a write makes every ticket `galactic-edge` holds for the scope stale, so a change prepared before it is prepared again. A gVPC is in use while the API has an `EdgeAttachment` on the node for it that is not `Released`, which covers one still being prepared, or the host has a link stamped for it. Every other writer in the table below either writes only rows an edge attachment never uses, or only adds rows; the table gives the reason for each. No lab or end-to-end environment runs `galactic-edge` yet.

### Owned by `galactic-edge`

Expand Down
Loading
Loading