Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,14 @@ internal/plumbing/ebpf/xmitdrops/xmitdrops_bpfel.go
internal/plumbing/ebpf/xmitdrops/xmitdrops_bpfel.o
internal/plumbing/ebpf/xmitdrops/xmitdrops_bpfeb.go
internal/plumbing/ebpf/xmitdrops/xmitdrops_bpfeb.o
internal/plumbing/ebpf/localprog/local_bpfel.go
internal/plumbing/ebpf/localprog/local_bpfel.o
internal/plumbing/ebpf/localprog/local_bpfeb.go
internal/plumbing/ebpf/localprog/local_bpfeb.o
internal/plumbing/ebpf/localprog/localfixture/fixture_bpfel.go
internal/plumbing/ebpf/localprog/localfixture/fixture_bpfel.o
internal/plumbing/ebpf/localprog/localfixture/fixture_bpfeb.go
internal/plumbing/ebpf/localprog/localfixture/fixture_bpfeb.o
tests/handoffs/probe/handoff_bpfel.go
tests/handoffs/probe/handoff_bpfel.o
tests/handoffs/probe/handoff_bpfeb.go
Expand Down
3 changes: 2 additions & 1 deletion Taskfile.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ tasks:
cmds:
- python3 hack/docs/generate_api.py --check
build:ebpf:
desc: Regenerate the eBPF uSID, edge gateway DSR, NAT egress, XDP dispatcher, XDP transmit drop and Local handoff probe programs (requires clang)
desc: Regenerate the eBPF uSID, edge gateway DSR, NAT egress, XDP dispatcher, XDP transmit drop, Local handoff and Local handoff probe programs (requires clang)
run: once
deps: [require-clang]
cmds:
Expand All @@ -131,6 +131,7 @@ tasks:
- go generate ./internal/plumbing/ebpf/natprog/...
- go generate ./internal/plumbing/ebpf/xdpdispatch/...
- go generate ./internal/plumbing/ebpf/xmitdrops/...
- go generate ./internal/plumbing/ebpf/localprog/...
- go generate ./tests/handoffs/probe/...
require-clang:
internal: true
Expand Down
39 changes: 33 additions & 6 deletions api/galactic/worker/v1alpha1/contract.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,12 +92,16 @@ const (
ReasonClaimExists = "ClaimExists"
ReasonClaimNotFound = "ClaimNotFound"
ReasonGenerationConflict = "GenerationConflict"
ReasonNoSession = "NoSession"
ReasonStaleGeneration = "StaleGeneration"
ReasonResyncing = "Resyncing"
ReasonShuttingDown = "ShuttingDown"
ReasonClaimLimit = "ClaimLimit"
ReasonTableRangeExhausted = "TableRangeExhausted"
// ReasonClaimHeldByAnotherPod refuses a call on a routed_link or socket
// claim from a pod other than the one the claim's link serves, while
// that pod still runs.
ReasonClaimHeldByAnotherPod = "ClaimHeldByAnotherPod"
ReasonNoSession = "NoSession"
ReasonStaleGeneration = "StaleGeneration"
ReasonResyncing = "Resyncing"
ReasonShuttingDown = "ShuttingDown"
ReasonClaimLimit = "ClaimLimit"
ReasonTableRangeExhausted = "TableRangeExhausted"
)

// Status reasons, carried in conditions, destination status and
Expand Down Expand Up @@ -139,6 +143,29 @@ const (
// ReasonReturnPathNotProgrammed is a destination whose return traffic
// this node cannot yet decapsulate into the VPC's table.
ReasonReturnPathNotProgrammed = "ReturnPathNotProgrammed"
// ReasonHandoffHookMissing is a Local destination whose handoff program
// is not on its hook: Worker's own tc program, or the program the
// client attaches to the NAT end of a veth pair.
ReasonHandoffHookMissing = "HandoffHookMissing"
// ReasonDispatcherSlotUnclaimed is a local.tap destination with no
// receiving device whose XDP dispatcher runs a live public slot.
ReasonDispatcherSlotUnclaimed = "DispatcherSlotUnclaimed"
// ReasonHandoffMTUTooSmall is a Local destination whose interface's MTU
// is below the receiving path's, so a packet the path carries would
// not reach the guest.
ReasonHandoffMTUTooSmall = "HandoffMTUTooSmall"
)

// Handoff interface roles, carried in HandoffInterface.
const (
// HandoffRoleNAT is the NAT end of a service-internal veth pair.
HandoffRoleNAT = "nat"
// HandoffRoleTenant is the tenant end of a service-internal veth pair.
HandoffRoleTenant = "tenant"
// HandoffRoleHost is the host end of a routed link.
HandoffRoleHost = "host"
// HandoffRolePod is the routed link's end in the caller's pod.
HandoffRolePod = "pod"
)

// Route rejection reasons, carried in RejectedRoute.
Expand Down
137 changes: 112 additions & 25 deletions api/galactic/worker/v1alpha1/worker.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

19 changes: 19 additions & 0 deletions api/galactic/worker/v1alpha1/worker.proto
Original file line number Diff line number Diff line change
Expand Up @@ -257,4 +257,23 @@ message ClaimStatus {
bool released = 7;
// revision increases with every snapshot Worker produces for the claim.
uint64 revision = 8;
// handoff_interfaces are the interfaces Worker created for the claim's
// handoff, which the client attaches its own program to or configures:
// the NAT end of a veth_pair claim's service-internal veth pair, and both
// ends of a routed_link or socket claim's routed link. A tap claim has
// none. Each is listed once Worker has created it.
repeated HandoffInterface handoff_interfaces = 9;
}

// HandoffInterface is one interface Worker created for a claim's handoff.
message HandoffInterface {
// role is "nat" for the NAT end of a veth pair, in the host's network
// namespace; "tenant" for the pair's tenant end, in the VPC's VRF; "host"
// for a routed link's host end, in the VPC's VRF; or "pod" for the routed
// link's end in the caller's pod, which the pod configures.
string role = 1;
// name is the interface name in its network namespace.
string name = 2;
// ifindex is the interface index in its network namespace.
int32 ifindex = 3;
}
30 changes: 27 additions & 3 deletions cmd/galactic-worker/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,13 @@ import (
"go.datum.net/galactic/internal/nodelocator"
"go.datum.net/galactic/internal/ownership"
"go.datum.net/galactic/internal/plumbing/ebpf/attach"
"go.datum.net/galactic/internal/plumbing/ebpf/localprog"
"go.datum.net/galactic/internal/plumbing/ebpf/xdpdispatch"
"go.datum.net/galactic/internal/worker"
"go.datum.net/galactic/internal/worker/auth"
"go.datum.net/galactic/internal/worker/fabric"
"go.datum.net/galactic/internal/worker/hostscope"
"go.datum.net/galactic/internal/worker/local"
"go.datum.net/galactic/internal/worker/module"
"go.datum.net/galactic/internal/worker/routes"
"go.datum.net/galactic/internal/worker/sources"
Expand Down Expand Up @@ -75,6 +78,8 @@ type options struct {
claimLimit int
resyncInterval time.Duration
pinDir string
localPinDir string
dispatchDir string
hostGrace time.Duration
}

Expand Down Expand Up @@ -179,8 +184,16 @@ func run(o *options) error {
return err
}
fabric.InitMetrics()
pods := &local.ProcPods{ProcRoot: o.procRoot, Pods: auth.CgroupPodResolver{ProcRoot: o.procRoot}}
localKernel := &local.Kernel{PinDir: o.pinDir, LocalPinDir: o.localPinDir, DispatchDir: o.dispatchDir,
Pods: pods}
defer func() { _ = localKernel.Close() }()
loc, err := local.New(local.Config{Kernel: localKernel, Node: o.nodeName, Log: slog.Default()})
if err != nil {
return err
}
// The delivery modules this build runs.
reg, err := module.NewRegistry(fab)
reg, err := module.NewRegistry(fab, loc)
if err != nil {
return err
}
Expand All @@ -201,8 +214,9 @@ func run(o *options) error {
Allowlist: allow,
Node: o.nodeName,
},
Routes: tenantRoutes,
Log: slog.Default(),
Routes: tenantRoutes,
Log: slog.Default(),
PodAlive: pods.Alive,
})
if err != nil {
return err
Expand All @@ -225,6 +239,13 @@ func run(o *options) error {
}); err != nil {
return err
}
// A receiving device, a pair's NAT end or a failed tenant neighbor
// changing requeues the VPCs whose Local handoffs it touches.
if err := loc.Watch(ctx, w.HostChanged, func(err error) {
slog.Error("Local link and neighbor event subscription failed; relying on periodic resync", "error", err)
}); err != nil {
return err
}

mgrErr := make(chan error, 1)
go func() { mgrErr <- mgr.Start(ctx) }()
Expand Down Expand Up @@ -313,6 +334,9 @@ func newRootCommand() *cobra.Command {
f.DurationVar(&o.resyncInterval, "resync-interval", worker.DefaultResyncInterval,
"How often every VPC scope is reconciled again")
f.StringVar(&o.pinDir, "pin-dir", attach.PinDir, "bpffs directory of the pinned uSID maps")
f.StringVar(&o.localPinDir, "local-pin-dir", localprog.PinDir, "bpffs directory of the Local handoff maps")
f.StringVar(&o.dispatchDir, "xdp-dispatch-dir", xdpdispatch.PinDir,
"bpffs directory of the shared XDP dispatcher, read to find public receiving devices")
f.DurationVar(&o.hostGrace, "host-grace", worker.DefaultHostGrace,
"How long a VPC's host state must go unused before Worker reclaims it")
f.Bool("build-info", false, "Print build information and exit")
Expand Down
6 changes: 6 additions & 0 deletions deploy/containerlab/docs/handoffs.md
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,12 @@ own owner) for the client's prefix and the routed link in both VRFs, which is
what Worker programs for a Local destination; see
[contract amendments](../../../docs/worker/contract.md#local-handoffs).

Worker's Local module (`internal/worker/local`,
[#1435](https://github.com/datum-cloud/galactic/issues/1435)) now programs the
tc side of these handoffs as product code; its tests reuse the stand-ins for
the gateway's and NAT's XDP programs (`localprog/localfixture`). See
[the Local module](../../../docs/worker/contract.md#the-local-module).

## Cells

Each cell builds one receiving device, places the prototypes in one mode, and
Expand Down
Loading
Loading