From 4021a55ad99a273b67f21271ac8acc5c833e2ab0 Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Wed, 18 Nov 2026 18:02:53 -0500 Subject: [PATCH 1/6] refactor(installer): Move node locator derivation into nodelocator Galactic Worker resolves every destination by comparing node locators, and must read them from the same trusted source as the locator's local route: the node's BGPRouter Block and Node-ID. The derivation lived unexported in internal/installer, which Worker should not import. Move it to internal/nodelocator.Prefix and keep the installer's nodeLocatorPrefix as a thin wrapper, so its behavior and tests stay as they are. Co-Authored-By: Claude --- internal/installer/locatorroute.go | 52 ++------------------- internal/nodelocator/nodelocator.go | 70 +++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 48 deletions(-) create mode 100644 internal/nodelocator/nodelocator.go diff --git a/internal/installer/locatorroute.go b/internal/installer/locatorroute.go index 33cc1b41..aabcd14c 100644 --- a/internal/installer/locatorroute.go +++ b/internal/installer/locatorroute.go @@ -16,7 +16,7 @@ import ( "golang.org/x/sys/unix" "sigs.k8s.io/controller-runtime/pkg/client" - bgpv1alpha1 "go.datum.net/galactic/api/network/v1alpha1" + "go.datum.net/galactic/internal/nodelocator" "go.datum.net/galactic/internal/plumbing/ebpf/uformat" "go.datum.net/galactic/internal/plumbing/nldump" ) @@ -150,56 +150,12 @@ func staleLocatorLocalRoute(r netlink.Route, loIndex int, keep netip.Prefix) boo return netip.PrefixFrom(got.Unmap(), ones) != keep } -// nodeLocatorPrefix returns the /64 carved out of this node's BGPRouter -// locator, or the zero Prefix when this node has no BGPRouter or that router -// carries no locator yet. Both are ordinary during bring-up and neither is an -// error. The router is the Galactic-owned one (no routerClassName) matched on -// its target name, the same way every other component here finds a node's -// router; more than one is an error rather than an arbitrary pick. +// nodeLocatorPrefix returns the /64 of this node's uSID locator, or the zero +// Prefix while it has none; nodelocator.Prefix describes the rules. func nodeLocatorPrefix( ctx context.Context, k8s client.Client, namespace, nodeName string, ) (netip.Prefix, error) { - routers := &bgpv1alpha1.BGPRouterList{} - if err := k8s.List(ctx, routers, client.InNamespace(namespace)); err != nil { - return netip.Prefix{}, fmt.Errorf("list BGPRouters in namespace %s: %w", namespace, err) - } - - var mine []bgpv1alpha1.BGPRouter - for _, r := range bgpv1alpha1.LegacyOwnedRouters(routers.Items) { - if r.Spec.TargetRef.Name == nodeName { - mine = append(mine, r) - } - } - if len(mine) > 1 { - return netip.Prefix{}, fmt.Errorf("ambiguous BGP config: %d BGPRouters target node %s in namespace %s", - len(mine), nodeName, namespace) - } - if len(mine) == 0 { - return netip.Prefix{}, nil - } - r := mine[0] - if r.Spec.SRv6Locator == "" || r.Spec.NodeID == 0 { - return netip.Prefix{}, nil - } - locator, err := netip.ParsePrefix(r.Spec.SRv6Locator) - if err != nil { - return netip.Prefix{}, fmt.Errorf("parse SRv6 locator %q: %w", r.Spec.SRv6Locator, err) - } - if !locator.Addr().Is6() || locator.Bits() != uformat.BlockBits { - return netip.Prefix{}, fmt.Errorf( - "SRv6 locator %q must be an IPv6 /%d uSID Block", r.Spec.SRv6Locator, uformat.BlockBits) - } - if r.Spec.NodeID < uformat.NodeIDMin || r.Spec.NodeID > uformat.NodeIDMax { - return netip.Prefix{}, fmt.Errorf("BGPRouter %s: nodeID %d outside [%#x,%#x]", - r.Name, r.Spec.NodeID, uint16(uformat.NodeIDMin), uint16(uformat.NodeIDMax)) - } - - // Node-ID occupies bits 49-64, immediately after the 48-bit Block -- - // the same layout uformat.Block/NodeID read back out of a SID. - b := locator.Addr().As16() - b[6] = byte(uint16(r.Spec.NodeID) >> 8) - b[7] = byte(uint16(r.Spec.NodeID)) - return netip.PrefixFrom(netip.AddrFrom16(b), uformat.BlockBits+uformat.NodeIDBits), nil + return nodelocator.Prefix(ctx, k8s, namespace, nodeName) } // reconcileLocatorLocalRoute is Run's non-fatal wrapper: a missing BGPRouter or diff --git a/internal/nodelocator/nodelocator.go b/internal/nodelocator/nodelocator.go new file mode 100644 index 00000000..578b64d7 --- /dev/null +++ b/internal/nodelocator/nodelocator.go @@ -0,0 +1,70 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package nodelocator derives a node's SRv6 uSID locator from its BGPRouter, +// the single trusted source of a node's physical locator. +package nodelocator + +import ( + "context" + "fmt" + "net/netip" + + "sigs.k8s.io/controller-runtime/pkg/client" + + bgpv1alpha1 "go.datum.net/galactic/api/network/v1alpha1" + "go.datum.net/galactic/internal/plumbing/ebpf/uformat" +) + +// Prefix returns the /64 carved out of a node's BGPRouter +// locator, or the zero Prefix when the node has no BGPRouter or that router +// carries no locator yet. Both are ordinary during bring-up and neither is an +// error. The router is the Galactic-owned one (no routerClassName) matched on +// its target name, the same way every Galactic component finds a node's +// router; more than one is an error rather than an arbitrary pick. +func Prefix( + ctx context.Context, k8s client.Reader, namespace, nodeName string, +) (netip.Prefix, error) { + routers := &bgpv1alpha1.BGPRouterList{} + if err := k8s.List(ctx, routers, client.InNamespace(namespace)); err != nil { + return netip.Prefix{}, fmt.Errorf("list BGPRouters in namespace %s: %w", namespace, err) + } + + var mine []bgpv1alpha1.BGPRouter + for _, r := range bgpv1alpha1.LegacyOwnedRouters(routers.Items) { + if r.Spec.TargetRef.Name == nodeName { + mine = append(mine, r) + } + } + if len(mine) > 1 { + return netip.Prefix{}, fmt.Errorf("ambiguous BGP config: %d BGPRouters target node %s in namespace %s", + len(mine), nodeName, namespace) + } + if len(mine) == 0 { + return netip.Prefix{}, nil + } + r := mine[0] + if r.Spec.SRv6Locator == "" || r.Spec.NodeID == 0 { + return netip.Prefix{}, nil + } + locator, err := netip.ParsePrefix(r.Spec.SRv6Locator) + if err != nil { + return netip.Prefix{}, fmt.Errorf("parse SRv6 locator %q: %w", r.Spec.SRv6Locator, err) + } + if !locator.Addr().Is6() || locator.Bits() != uformat.BlockBits { + return netip.Prefix{}, fmt.Errorf( + "SRv6 locator %q must be an IPv6 /%d uSID Block", r.Spec.SRv6Locator, uformat.BlockBits) + } + if r.Spec.NodeID < uformat.NodeIDMin || r.Spec.NodeID > uformat.NodeIDMax { + return netip.Prefix{}, fmt.Errorf("BGPRouter %s: nodeID %d outside [%#x,%#x]", + r.Name, r.Spec.NodeID, uint16(uformat.NodeIDMin), uint16(uformat.NodeIDMax)) + } + + // Node-ID occupies bits 49-64, immediately after the 48-bit Block -- + // the same layout uformat.Block/NodeID read back out of a SID. + b := locator.Addr().As16() + b[6] = byte(uint16(r.Spec.NodeID) >> 8) + b[7] = byte(uint16(r.Spec.NodeID)) + return netip.PrefixFrom(netip.AddrFrom16(b), uformat.BlockBits+uformat.NodeIDBits), nil +} From 2a21098d72d23732c6a3b58776b588235936798e Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Wed, 18 Nov 2026 18:02:59 -0500 Subject: [PATCH 2/6] feat(worker): Add the Worker v1alpha1 API and client library Service agents need one published interface to Galactic Worker. This adds the galactic.worker.v1alpha1 protobuf service from docs/worker/contract.md (GetCapabilities, Request, Update, Observe and Release) and the contract's names: capabilities, conditions, refusal and status reasons, the error domain and the session metadata key. The proto lives in api/galactic/worker/v1alpha1 because buf's STANDARD lint requires the directory to match the package. The service keeps the contract's name, Worker, with SERVICE_SUFFIX ignored. pkg/workerclient is the client library. It imports only the API and gRPC, so an independent client can depend on it without Galactic internals. It dials the unix socket, opens and renews a session, reads the projected token for every call, drops duplicate and reordered snapshots by revision, and retries the codes the contract says to retry, with backoff from 200 ms to 30 s and full jitter. Co-Authored-By: Claude --- api/galactic/worker/v1alpha1/contract.go | 118 ++ api/galactic/worker/v1alpha1/worker.pb.go | 1830 +++++++++++++++++ api/galactic/worker/v1alpha1/worker.proto | 251 +++ .../worker/v1alpha1/worker_grpc.pb.go | 316 +++ pkg/workerclient/client.go | 351 ++++ 5 files changed, 2866 insertions(+) create mode 100644 api/galactic/worker/v1alpha1/contract.go create mode 100644 api/galactic/worker/v1alpha1/worker.pb.go create mode 100644 api/galactic/worker/v1alpha1/worker.proto create mode 100644 api/galactic/worker/v1alpha1/worker_grpc.pb.go create mode 100644 pkg/workerclient/client.go diff --git a/api/galactic/worker/v1alpha1/contract.go b/api/galactic/worker/v1alpha1/contract.go new file mode 100644 index 00000000..f1584c5a --- /dev/null +++ b/api/galactic/worker/v1alpha1/contract.go @@ -0,0 +1,118 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package workerv1alpha1 + +// The names below are part of the published contract +// (docs/worker/contract.md). Clients compare against them; Worker emits them. + +const ( + // Version is the API version this package serves. + Version = "v1alpha1" + // ErrorDomain is the google.rpc.ErrorInfo domain of every refusal. + ErrorDomain = "worker.galactic.datumapis.com" + // TokenAudience is the audience of the projected ServiceAccount token a + // client presents. + TokenAudience = "galactic-worker" + // DefaultSocketPath is where Worker serves the API. + DefaultSocketPath = "/run/galactic-worker/worker.sock" + // SessionMetadataKey carries the session ID on every call but the first + // GetCapabilities. A GetCapabilities that carries a live session ID + // renews that session instead of starting a new one. + SessionMetadataKey = "x-galactic-worker-session" +) + +// ErrorInfo metadata keys. +const ( + // MetadataCurrentGeneration is the claim's current generation, on + // StaleGeneration and GenerationConflict. + MetadataCurrentGeneration = "current_generation" + // MetadataServedVersions lists the versions Worker serves, comma + // separated, on UnsupportedVersion. + MetadataServedVersions = "served_versions" +) + +// Capability names. +const ( + CapabilityLocalTap = "local.tap" + CapabilityLocalVethPair = "local.veth_pair" + CapabilityLocalRoutedLink = "local.routed_link" + CapabilityLocalSocket = "local.socket" + CapabilityFabricSRv6 = "fabric.srv6" +) + +// Capabilities lists every capability in the order GetCapabilities reports +// them. +var Capabilities = []string{ + CapabilityLocalTap, CapabilityLocalVethPair, CapabilityLocalRoutedLink, + CapabilityLocalSocket, CapabilityFabricSRv6, +} + +// HandoffCapability is the Local capability a handoff kind needs, or "" for +// HANDOFF_UNSPECIFIED. +func HandoffCapability(h Handoff) string { + switch h { + case Handoff_HANDOFF_TAP: + return CapabilityLocalTap + case Handoff_HANDOFF_VETH_PAIR: + return CapabilityLocalVethPair + case Handoff_HANDOFF_ROUTED_LINK: + return CapabilityLocalRoutedLink + case Handoff_HANDOFF_SOCKET: + return CapabilityLocalSocket + default: + return "" + } +} + +// Condition types. +const ( + ConditionAdmitted = "Admitted" + ConditionResolved = "Resolved" + ConditionProgrammed = "Programmed" + ConditionReady = "Ready" + ConditionWithdrawing = "Withdrawing" +) + +// Refusal reasons, carried in google.rpc.ErrorInfo. +const ( + ReasonMalformedSpec = "MalformedSpec" + ReasonWrongNode = "WrongNode" + ReasonNoIdentity = "NoIdentity" + ReasonIdentityExpired = "IdentityExpired" + ReasonVPCNotAuthorized = "VPCNotAuthorized" + ReasonAttachmentClaimedByAnotherCaller = "AttachmentClaimedByAnotherCaller" + ReasonNotOwner = "NotOwner" + ReasonCapabilityNotAllowed = "CapabilityNotAllowed" + ReasonUnsupportedVersion = "UnsupportedVersion" + ReasonClaimExists = "ClaimExists" + ReasonClaimNotFound = "ClaimNotFound" + ReasonGenerationConflict = "GenerationConflict" + ReasonNoSession = "NoSession" + ReasonStaleGeneration = "StaleGeneration" + ReasonResyncing = "Resyncing" + ReasonShuttingDown = "ShuttingDown" + ReasonClaimLimit = "ClaimLimit" + ReasonTableRangeExhausted = "TableRangeExhausted" +) + +// Status reasons, carried in conditions, destination status and +// capabilities. +const ( + ReasonPending = "Pending" + ReasonAsExpected = "AsExpected" + ReasonModeUnavailable = "ModeUnavailable" + ReasonDestinationKindUnsupported = "DestinationKindUnsupported" + ReasonDestinationNotFound = "DestinationNotFound" + ReasonLocatorUnknown = "LocatorUnknown" + ReasonModuleNotRunning = "ModuleNotRunning" + ReasonDatapathNotLoaded = "DatapathNotLoaded" + ReasonHookUnsupported = "HookUnsupported" + ReasonHandoffInterfaceMissing = "HandoffInterfaceMissing" + ReasonProgrammingFailed = "ProgrammingFailed" + ReasonScopeOwnedElsewhere = "ScopeOwnedElsewhere" + ReasonAuthorizationRevoked = "AuthorizationRevoked" + ReasonLeaseExpired = "LeaseExpired" + ReasonReleased = "Released" +) diff --git a/api/galactic/worker/v1alpha1/worker.pb.go b/api/galactic/worker/v1alpha1/worker.pb.go new file mode 100644 index 00000000..cafcc3f2 --- /dev/null +++ b/api/galactic/worker/v1alpha1/worker.pb.go @@ -0,0 +1,1830 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// The Galactic Worker service contract: the node-local API a service agent +// calls to deliver tenant traffic between its own handoff point and tenant +// destinations. docs/worker/contract.md is the normative description of every +// operation, state and error. +// +// Worker serves it on a unix socket only. Every call except GetCapabilities +// carries the session ID GetCapabilities returned, in the +// x-galactic-worker-session metadata key, and every call carries a projected +// ServiceAccount token with audience galactic-worker as +// "authorization: Bearer ". Every refusal is a gRPC status carrying a +// google.rpc.ErrorInfo whose domain is worker.galactic.datumapis.com and whose +// reason is one of the machine reasons the contract lists. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: galactic/worker/v1alpha1/worker.proto + +package workerv1alpha1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + durationpb "google.golang.org/protobuf/types/known/durationpb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// CapabilityState reports whether a capability can be programmed now. +type CapabilityState int32 + +const ( + CapabilityState_CAPABILITY_STATE_UNSPECIFIED CapabilityState = 0 + CapabilityState_CAPABILITY_STATE_AVAILABLE CapabilityState = 1 + CapabilityState_CAPABILITY_STATE_UNAVAILABLE CapabilityState = 2 +) + +// Enum value maps for CapabilityState. +var ( + CapabilityState_name = map[int32]string{ + 0: "CAPABILITY_STATE_UNSPECIFIED", + 1: "CAPABILITY_STATE_AVAILABLE", + 2: "CAPABILITY_STATE_UNAVAILABLE", + } + CapabilityState_value = map[string]int32{ + "CAPABILITY_STATE_UNSPECIFIED": 0, + "CAPABILITY_STATE_AVAILABLE": 1, + "CAPABILITY_STATE_UNAVAILABLE": 2, + } +) + +func (x CapabilityState) Enum() *CapabilityState { + p := new(CapabilityState) + *p = x + return p +} + +func (x CapabilityState) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (CapabilityState) Descriptor() protoreflect.EnumDescriptor { + return file_galactic_worker_v1alpha1_worker_proto_enumTypes[0].Descriptor() +} + +func (CapabilityState) Type() protoreflect.EnumType { + return &file_galactic_worker_v1alpha1_worker_proto_enumTypes[0] +} + +func (x CapabilityState) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use CapabilityState.Descriptor instead. +func (CapabilityState) EnumDescriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP(), []int{0} +} + +// Handoff is how traffic passes between the service and the tenant side. +type Handoff int32 + +const ( + Handoff_HANDOFF_UNSPECIFIED Handoff = 0 + Handoff_HANDOFF_TAP Handoff = 1 + Handoff_HANDOFF_VETH_PAIR Handoff = 2 + Handoff_HANDOFF_ROUTED_LINK Handoff = 3 + Handoff_HANDOFF_SOCKET Handoff = 4 +) + +// Enum value maps for Handoff. +var ( + Handoff_name = map[int32]string{ + 0: "HANDOFF_UNSPECIFIED", + 1: "HANDOFF_TAP", + 2: "HANDOFF_VETH_PAIR", + 3: "HANDOFF_ROUTED_LINK", + 4: "HANDOFF_SOCKET", + } + Handoff_value = map[string]int32{ + "HANDOFF_UNSPECIFIED": 0, + "HANDOFF_TAP": 1, + "HANDOFF_VETH_PAIR": 2, + "HANDOFF_ROUTED_LINK": 3, + "HANDOFF_SOCKET": 4, + } +) + +func (x Handoff) Enum() *Handoff { + p := new(Handoff) + *p = x + return p +} + +func (x Handoff) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (Handoff) Descriptor() protoreflect.EnumDescriptor { + return file_galactic_worker_v1alpha1_worker_proto_enumTypes[1].Descriptor() +} + +func (Handoff) Type() protoreflect.EnumType { + return &file_galactic_worker_v1alpha1_worker_proto_enumTypes[1] +} + +func (x Handoff) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use Handoff.Descriptor instead. +func (Handoff) EnumDescriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP(), []int{1} +} + +// Mode is a delivery mode: the preference in a spec, or the result of +// resolving a destination. +type Mode int32 + +const ( + Mode_MODE_UNSPECIFIED Mode = 0 + // MODE_AUTO lets Worker resolve each destination. A spec only. + Mode_MODE_AUTO Mode = 1 + Mode_MODE_LOCAL Mode = 2 + Mode_MODE_FABRIC Mode = 3 +) + +// Enum value maps for Mode. +var ( + Mode_name = map[int32]string{ + 0: "MODE_UNSPECIFIED", + 1: "MODE_AUTO", + 2: "MODE_LOCAL", + 3: "MODE_FABRIC", + } + Mode_value = map[string]int32{ + "MODE_UNSPECIFIED": 0, + "MODE_AUTO": 1, + "MODE_LOCAL": 2, + "MODE_FABRIC": 3, + } +) + +func (x Mode) Enum() *Mode { + p := new(Mode) + *p = x + return p +} + +func (x Mode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (Mode) Descriptor() protoreflect.EnumDescriptor { + return file_galactic_worker_v1alpha1_worker_proto_enumTypes[2].Descriptor() +} + +func (Mode) Type() protoreflect.EnumType { + return &file_galactic_worker_v1alpha1_worker_proto_enumTypes[2] +} + +func (x Mode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use Mode.Descriptor instead. +func (Mode) EnumDescriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP(), []int{2} +} + +// ConditionStatus is the status of one condition. +type ConditionStatus int32 + +const ( + ConditionStatus_CONDITION_STATUS_UNSPECIFIED ConditionStatus = 0 + ConditionStatus_CONDITION_STATUS_TRUE ConditionStatus = 1 + ConditionStatus_CONDITION_STATUS_FALSE ConditionStatus = 2 + ConditionStatus_CONDITION_STATUS_UNKNOWN ConditionStatus = 3 +) + +// Enum value maps for ConditionStatus. +var ( + ConditionStatus_name = map[int32]string{ + 0: "CONDITION_STATUS_UNSPECIFIED", + 1: "CONDITION_STATUS_TRUE", + 2: "CONDITION_STATUS_FALSE", + 3: "CONDITION_STATUS_UNKNOWN", + } + ConditionStatus_value = map[string]int32{ + "CONDITION_STATUS_UNSPECIFIED": 0, + "CONDITION_STATUS_TRUE": 1, + "CONDITION_STATUS_FALSE": 2, + "CONDITION_STATUS_UNKNOWN": 3, + } +) + +func (x ConditionStatus) Enum() *ConditionStatus { + p := new(ConditionStatus) + *p = x + return p +} + +func (x ConditionStatus) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ConditionStatus) Descriptor() protoreflect.EnumDescriptor { + return file_galactic_worker_v1alpha1_worker_proto_enumTypes[3].Descriptor() +} + +func (ConditionStatus) Type() protoreflect.EnumType { + return &file_galactic_worker_v1alpha1_worker_proto_enumTypes[3] +} + +func (x ConditionStatus) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ConditionStatus.Descriptor instead. +func (ConditionStatus) EnumDescriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP(), []int{3} +} + +// WithdrawalScope names what a withdrawal stops. +type WithdrawalScope int32 + +const ( + WithdrawalScope_WITHDRAWAL_SCOPE_UNSPECIFIED WithdrawalScope = 0 + WithdrawalScope_WITHDRAWAL_SCOPE_DESTINATION WithdrawalScope = 1 + WithdrawalScope_WITHDRAWAL_SCOPE_CLAIM WithdrawalScope = 2 + WithdrawalScope_WITHDRAWAL_SCOPE_CALLER WithdrawalScope = 3 + WithdrawalScope_WITHDRAWAL_SCOPE_VPC WithdrawalScope = 4 +) + +// Enum value maps for WithdrawalScope. +var ( + WithdrawalScope_name = map[int32]string{ + 0: "WITHDRAWAL_SCOPE_UNSPECIFIED", + 1: "WITHDRAWAL_SCOPE_DESTINATION", + 2: "WITHDRAWAL_SCOPE_CLAIM", + 3: "WITHDRAWAL_SCOPE_CALLER", + 4: "WITHDRAWAL_SCOPE_VPC", + } + WithdrawalScope_value = map[string]int32{ + "WITHDRAWAL_SCOPE_UNSPECIFIED": 0, + "WITHDRAWAL_SCOPE_DESTINATION": 1, + "WITHDRAWAL_SCOPE_CLAIM": 2, + "WITHDRAWAL_SCOPE_CALLER": 3, + "WITHDRAWAL_SCOPE_VPC": 4, + } +) + +func (x WithdrawalScope) Enum() *WithdrawalScope { + p := new(WithdrawalScope) + *p = x + return p +} + +func (x WithdrawalScope) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (WithdrawalScope) Descriptor() protoreflect.EnumDescriptor { + return file_galactic_worker_v1alpha1_worker_proto_enumTypes[4].Descriptor() +} + +func (WithdrawalScope) Type() protoreflect.EnumType { + return &file_galactic_worker_v1alpha1_worker_proto_enumTypes[4] +} + +func (x WithdrawalScope) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use WithdrawalScope.Descriptor instead. +func (WithdrawalScope) EnumDescriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP(), []int{4} +} + +type GetCapabilitiesRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // versions are the API versions the client supports, such as "v1alpha1". + // Worker picks the highest one it also serves. + Versions []string `protobuf:"bytes,1,rep,name=versions,proto3" json:"versions,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetCapabilitiesRequest) Reset() { + *x = GetCapabilitiesRequest{} + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetCapabilitiesRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetCapabilitiesRequest) ProtoMessage() {} + +func (x *GetCapabilitiesRequest) ProtoReflect() protoreflect.Message { + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetCapabilitiesRequest.ProtoReflect.Descriptor instead. +func (*GetCapabilitiesRequest) Descriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP(), []int{0} +} + +func (x *GetCapabilitiesRequest) GetVersions() []string { + if x != nil { + return x.Versions + } + return nil +} + +type GetCapabilitiesResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // version is the API version chosen for the session. + Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` + // session_id identifies the session. Send it with every later call. + SessionId string `protobuf:"bytes,2,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` + // lease is how long the session lives without a renewal. + Lease *durationpb.Duration `protobuf:"bytes,3,opt,name=lease,proto3" json:"lease,omitempty"` + // renew_interval is how often a client renews the lease, by a call or an + // open Observe stream. + RenewInterval *durationpb.Duration `protobuf:"bytes,4,opt,name=renew_interval,json=renewInterval,proto3" json:"renew_interval,omitempty"` + // capabilities lists every capability with its current state. + Capabilities []*Capability `protobuf:"bytes,5,rep,name=capabilities,proto3" json:"capabilities,omitempty"` + // node is the node this Worker runs on, which every claim key must name. + Node string `protobuf:"bytes,6,opt,name=node,proto3" json:"node,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetCapabilitiesResponse) Reset() { + *x = GetCapabilitiesResponse{} + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetCapabilitiesResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetCapabilitiesResponse) ProtoMessage() {} + +func (x *GetCapabilitiesResponse) ProtoReflect() protoreflect.Message { + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetCapabilitiesResponse.ProtoReflect.Descriptor instead. +func (*GetCapabilitiesResponse) Descriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP(), []int{1} +} + +func (x *GetCapabilitiesResponse) GetVersion() string { + if x != nil { + return x.Version + } + return "" +} + +func (x *GetCapabilitiesResponse) GetSessionId() string { + if x != nil { + return x.SessionId + } + return "" +} + +func (x *GetCapabilitiesResponse) GetLease() *durationpb.Duration { + if x != nil { + return x.Lease + } + return nil +} + +func (x *GetCapabilitiesResponse) GetRenewInterval() *durationpb.Duration { + if x != nil { + return x.RenewInterval + } + return nil +} + +func (x *GetCapabilitiesResponse) GetCapabilities() []*Capability { + if x != nil { + return x.Capabilities + } + return nil +} + +func (x *GetCapabilitiesResponse) GetNode() string { + if x != nil { + return x.Node + } + return "" +} + +type Capability struct { + state protoimpl.MessageState `protogen:"open.v1"` + // name is one of local.tap, local.veth_pair, local.routed_link, + // local.socket and fabric.srv6. + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + State CapabilityState `protobuf:"varint,2,opt,name=state,proto3,enum=galactic.worker.v1alpha1.CapabilityState" json:"state,omitempty"` + // reason is a machine reason when the capability is unavailable, such as + // ModuleNotRunning, DatapathNotLoaded or HookUnsupported. + Reason string `protobuf:"bytes,3,opt,name=reason,proto3" json:"reason,omitempty"` + Message string `protobuf:"bytes,4,opt,name=message,proto3" json:"message,omitempty"` + // allowed reports whether the caller's ServiceAccount may use it. + Allowed bool `protobuf:"varint,5,opt,name=allowed,proto3" json:"allowed,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Capability) Reset() { + *x = Capability{} + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Capability) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Capability) ProtoMessage() {} + +func (x *Capability) ProtoReflect() protoreflect.Message { + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Capability.ProtoReflect.Descriptor instead. +func (*Capability) Descriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP(), []int{2} +} + +func (x *Capability) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Capability) GetState() CapabilityState { + if x != nil { + return x.State + } + return CapabilityState_CAPABILITY_STATE_UNSPECIFIED +} + +func (x *Capability) GetReason() string { + if x != nil { + return x.Reason + } + return "" +} + +func (x *Capability) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +func (x *Capability) GetAllowed() bool { + if x != nil { + return x.Allowed + } + return false +} + +// ClaimKey names a claim. The caller is part of the key too, but Worker +// derives it from the connection; a client never supplies it. +type ClaimKey struct { + state protoimpl.MessageState `protogen:"open.v1"` + Node string `protobuf:"bytes,1,opt,name=node,proto3" json:"node,omitempty"` + // vpc is the base62 identifier of the VPC of the attachment. + Vpc string `protobuf:"bytes,2,opt,name=vpc,proto3" json:"vpc,omitempty"` + // attachment_uid is the UID of the VPCAttachment or EdgeAttachment that is + // the service's handoff point in the VPC. + AttachmentUid string `protobuf:"bytes,3,opt,name=attachment_uid,json=attachmentUid,proto3" json:"attachment_uid,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ClaimKey) Reset() { + *x = ClaimKey{} + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ClaimKey) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClaimKey) ProtoMessage() {} + +func (x *ClaimKey) ProtoReflect() protoreflect.Message { + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClaimKey.ProtoReflect.Descriptor instead. +func (*ClaimKey) Descriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP(), []int{3} +} + +func (x *ClaimKey) GetNode() string { + if x != nil { + return x.Node + } + return "" +} + +func (x *ClaimKey) GetVpc() string { + if x != nil { + return x.Vpc + } + return "" +} + +func (x *ClaimKey) GetAttachmentUid() string { + if x != nil { + return x.AttachmentUid + } + return "" +} + +type AttachmentRef struct { + state protoimpl.MessageState `protogen:"open.v1"` + // uid is a VPCAttachment or EdgeAttachment UID. + Uid string `protobuf:"bytes,1,opt,name=uid,proto3" json:"uid,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AttachmentRef) Reset() { + *x = AttachmentRef{} + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AttachmentRef) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AttachmentRef) ProtoMessage() {} + +func (x *AttachmentRef) ProtoReflect() protoreflect.Message { + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AttachmentRef.ProtoReflect.Descriptor instead. +func (*AttachmentRef) Descriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP(), []int{4} +} + +func (x *AttachmentRef) GetUid() string { + if x != nil { + return x.Uid + } + return "" +} + +type ShardRef struct { + state protoimpl.MessageState `protogen:"open.v1"` + // group names the shard group. + Group string `protobuf:"bytes,1,opt,name=group,proto3" json:"group,omitempty"` + Index uint32 `protobuf:"varint,2,opt,name=index,proto3" json:"index,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ShardRef) Reset() { + *x = ShardRef{} + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ShardRef) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ShardRef) ProtoMessage() {} + +func (x *ShardRef) ProtoReflect() protoreflect.Message { + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ShardRef.ProtoReflect.Descriptor instead. +func (*ShardRef) Descriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP(), []int{5} +} + +func (x *ShardRef) GetGroup() string { + if x != nil { + return x.Group + } + return "" +} + +func (x *ShardRef) GetIndex() uint32 { + if x != nil { + return x.Index + } + return 0 +} + +type BackendRef struct { + state protoimpl.MessageState `protogen:"open.v1"` + // node is the backend's node name. + Node string `protobuf:"bytes,1,opt,name=node,proto3" json:"node,omitempty"` + // slot is the slot the gateway encodes in the backend SID. + Slot uint32 `protobuf:"varint,2,opt,name=slot,proto3" json:"slot,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BackendRef) Reset() { + *x = BackendRef{} + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BackendRef) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BackendRef) ProtoMessage() {} + +func (x *BackendRef) ProtoReflect() protoreflect.Message { + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BackendRef.ProtoReflect.Descriptor instead. +func (*BackendRef) Descriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP(), []int{6} +} + +func (x *BackendRef) GetNode() string { + if x != nil { + return x.Node + } + return "" +} + +func (x *BackendRef) GetSlot() uint32 { + if x != nil { + return x.Slot + } + return 0 +} + +// Destination is one tenant destination of a claim. +type Destination struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Target: + // + // *Destination_Attachment + // *Destination_Shard + // *Destination_Backend + Target isDestination_Target `protobuf_oneof:"target"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Destination) Reset() { + *x = Destination{} + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Destination) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Destination) ProtoMessage() {} + +func (x *Destination) ProtoReflect() protoreflect.Message { + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Destination.ProtoReflect.Descriptor instead. +func (*Destination) Descriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP(), []int{7} +} + +func (x *Destination) GetTarget() isDestination_Target { + if x != nil { + return x.Target + } + return nil +} + +func (x *Destination) GetAttachment() *AttachmentRef { + if x != nil { + if x, ok := x.Target.(*Destination_Attachment); ok { + return x.Attachment + } + } + return nil +} + +func (x *Destination) GetShard() *ShardRef { + if x != nil { + if x, ok := x.Target.(*Destination_Shard); ok { + return x.Shard + } + } + return nil +} + +func (x *Destination) GetBackend() *BackendRef { + if x != nil { + if x, ok := x.Target.(*Destination_Backend); ok { + return x.Backend + } + } + return nil +} + +type isDestination_Target interface { + isDestination_Target() +} + +type Destination_Attachment struct { + Attachment *AttachmentRef `protobuf:"bytes,1,opt,name=attachment,proto3,oneof"` +} + +type Destination_Shard struct { + // shard is resolved by #1437. Until then it reports + // DestinationKindUnsupported. + Shard *ShardRef `protobuf:"bytes,2,opt,name=shard,proto3,oneof"` +} + +type Destination_Backend struct { + // backend is resolved by #1438. Until then it reports + // DestinationKindUnsupported. + Backend *BackendRef `protobuf:"bytes,3,opt,name=backend,proto3,oneof"` +} + +func (*Destination_Attachment) isDestination_Target() {} + +func (*Destination_Shard) isDestination_Target() {} + +func (*Destination_Backend) isDestination_Target() {} + +type ClaimSpec struct { + state protoimpl.MessageState `protogen:"open.v1"` + Handoff Handoff `protobuf:"varint,1,opt,name=handoff,proto3,enum=galactic.worker.v1alpha1.Handoff" json:"handoff,omitempty"` + Destinations []*Destination `protobuf:"bytes,2,rep,name=destinations,proto3" json:"destinations,omitempty"` + // mode is the mode preference. Unset means MODE_AUTO. + Mode Mode `protobuf:"varint,3,opt,name=mode,proto3,enum=galactic.worker.v1alpha1.Mode" json:"mode,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ClaimSpec) Reset() { + *x = ClaimSpec{} + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ClaimSpec) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClaimSpec) ProtoMessage() {} + +func (x *ClaimSpec) ProtoReflect() protoreflect.Message { + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClaimSpec.ProtoReflect.Descriptor instead. +func (*ClaimSpec) Descriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP(), []int{8} +} + +func (x *ClaimSpec) GetHandoff() Handoff { + if x != nil { + return x.Handoff + } + return Handoff_HANDOFF_UNSPECIFIED +} + +func (x *ClaimSpec) GetDestinations() []*Destination { + if x != nil { + return x.Destinations + } + return nil +} + +func (x *ClaimSpec) GetMode() Mode { + if x != nil { + return x.Mode + } + return Mode_MODE_UNSPECIFIED +} + +type RequestRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Key *ClaimKey `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + // generation is client-assigned and must be at least 1. + Generation uint64 `protobuf:"varint,2,opt,name=generation,proto3" json:"generation,omitempty"` + Spec *ClaimSpec `protobuf:"bytes,3,opt,name=spec,proto3" json:"spec,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RequestRequest) Reset() { + *x = RequestRequest{} + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RequestRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RequestRequest) ProtoMessage() {} + +func (x *RequestRequest) ProtoReflect() protoreflect.Message { + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RequestRequest.ProtoReflect.Descriptor instead. +func (*RequestRequest) Descriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP(), []int{9} +} + +func (x *RequestRequest) GetKey() *ClaimKey { + if x != nil { + return x.Key + } + return nil +} + +func (x *RequestRequest) GetGeneration() uint64 { + if x != nil { + return x.Generation + } + return 0 +} + +func (x *RequestRequest) GetSpec() *ClaimSpec { + if x != nil { + return x.Spec + } + return nil +} + +type RequestResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Status *ClaimStatus `protobuf:"bytes,1,opt,name=status,proto3" json:"status,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RequestResponse) Reset() { + *x = RequestResponse{} + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RequestResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RequestResponse) ProtoMessage() {} + +func (x *RequestResponse) ProtoReflect() protoreflect.Message { + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RequestResponse.ProtoReflect.Descriptor instead. +func (*RequestResponse) Descriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP(), []int{10} +} + +func (x *RequestResponse) GetStatus() *ClaimStatus { + if x != nil { + return x.Status + } + return nil +} + +type UpdateRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Key *ClaimKey `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Generation uint64 `protobuf:"varint,2,opt,name=generation,proto3" json:"generation,omitempty"` + Spec *ClaimSpec `protobuf:"bytes,3,opt,name=spec,proto3" json:"spec,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *UpdateRequest) Reset() { + *x = UpdateRequest{} + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *UpdateRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UpdateRequest) ProtoMessage() {} + +func (x *UpdateRequest) ProtoReflect() protoreflect.Message { + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UpdateRequest.ProtoReflect.Descriptor instead. +func (*UpdateRequest) Descriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP(), []int{11} +} + +func (x *UpdateRequest) GetKey() *ClaimKey { + if x != nil { + return x.Key + } + return nil +} + +func (x *UpdateRequest) GetGeneration() uint64 { + if x != nil { + return x.Generation + } + return 0 +} + +func (x *UpdateRequest) GetSpec() *ClaimSpec { + if x != nil { + return x.Spec + } + return nil +} + +type UpdateResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Status *ClaimStatus `protobuf:"bytes,1,opt,name=status,proto3" json:"status,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *UpdateResponse) Reset() { + *x = UpdateResponse{} + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *UpdateResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UpdateResponse) ProtoMessage() {} + +func (x *UpdateResponse) ProtoReflect() protoreflect.Message { + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UpdateResponse.ProtoReflect.Descriptor instead. +func (*UpdateResponse) Descriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP(), []int{12} +} + +func (x *UpdateResponse) GetStatus() *ClaimStatus { + if x != nil { + return x.Status + } + return nil +} + +type ReleaseRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Key *ClaimKey `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + // generation must equal the claim's current generation. + Generation uint64 `protobuf:"varint,2,opt,name=generation,proto3" json:"generation,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReleaseRequest) Reset() { + *x = ReleaseRequest{} + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReleaseRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReleaseRequest) ProtoMessage() {} + +func (x *ReleaseRequest) ProtoReflect() protoreflect.Message { + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReleaseRequest.ProtoReflect.Descriptor instead. +func (*ReleaseRequest) Descriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP(), []int{13} +} + +func (x *ReleaseRequest) GetKey() *ClaimKey { + if x != nil { + return x.Key + } + return nil +} + +func (x *ReleaseRequest) GetGeneration() uint64 { + if x != nil { + return x.Generation + } + return 0 +} + +type ReleaseResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // status is the claim's status after the release was recorded, or unset + // when no such claim exists. + Status *ClaimStatus `protobuf:"bytes,1,opt,name=status,proto3" json:"status,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReleaseResponse) Reset() { + *x = ReleaseResponse{} + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReleaseResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReleaseResponse) ProtoMessage() {} + +func (x *ReleaseResponse) ProtoReflect() protoreflect.Message { + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReleaseResponse.ProtoReflect.Descriptor instead. +func (*ReleaseResponse) Descriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP(), []int{14} +} + +func (x *ReleaseResponse) GetStatus() *ClaimStatus { + if x != nil { + return x.Status + } + return nil +} + +type ObserveRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // key selects one claim. Unset selects every claim of the caller. + Key *ClaimKey `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ObserveRequest) Reset() { + *x = ObserveRequest{} + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ObserveRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ObserveRequest) ProtoMessage() {} + +func (x *ObserveRequest) ProtoReflect() protoreflect.Message { + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ObserveRequest.ProtoReflect.Descriptor instead. +func (*ObserveRequest) Descriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP(), []int{15} +} + +func (x *ObserveRequest) GetKey() *ClaimKey { + if x != nil { + return x.Key + } + return nil +} + +type ObserveResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Status *ClaimStatus `protobuf:"bytes,1,opt,name=status,proto3" json:"status,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ObserveResponse) Reset() { + *x = ObserveResponse{} + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ObserveResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ObserveResponse) ProtoMessage() {} + +func (x *ObserveResponse) ProtoReflect() protoreflect.Message { + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ObserveResponse.ProtoReflect.Descriptor instead. +func (*ObserveResponse) Descriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP(), []int{16} +} + +func (x *ObserveResponse) GetStatus() *ClaimStatus { + if x != nil { + return x.Status + } + return nil +} + +// Condition is one of Admitted, Resolved, Programmed, Ready and Withdrawing. +type Condition struct { + state protoimpl.MessageState `protogen:"open.v1"` + Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` + Status ConditionStatus `protobuf:"varint,2,opt,name=status,proto3,enum=galactic.worker.v1alpha1.ConditionStatus" json:"status,omitempty"` + Reason string `protobuf:"bytes,3,opt,name=reason,proto3" json:"reason,omitempty"` + Message string `protobuf:"bytes,4,opt,name=message,proto3" json:"message,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Condition) Reset() { + *x = Condition{} + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Condition) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Condition) ProtoMessage() {} + +func (x *Condition) ProtoReflect() protoreflect.Message { + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Condition.ProtoReflect.Descriptor instead. +func (*Condition) Descriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP(), []int{17} +} + +func (x *Condition) GetType() string { + if x != nil { + return x.Type + } + return "" +} + +func (x *Condition) GetStatus() ConditionStatus { + if x != nil { + return x.Status + } + return ConditionStatus_CONDITION_STATUS_UNSPECIFIED +} + +func (x *Condition) GetReason() string { + if x != nil { + return x.Reason + } + return "" +} + +func (x *Condition) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +type DestinationStatus struct { + state protoimpl.MessageState `protogen:"open.v1"` + Destination *Destination `protobuf:"bytes,1,opt,name=destination,proto3" json:"destination,omitempty"` + // mode is MODE_LOCAL or MODE_FABRIC once resolved. + Mode Mode `protobuf:"varint,2,opt,name=mode,proto3,enum=galactic.worker.v1alpha1.Mode" json:"mode,omitempty"` + Resolved bool `protobuf:"varint,3,opt,name=resolved,proto3" json:"resolved,omitempty"` + Programmed bool `protobuf:"varint,4,opt,name=programmed,proto3" json:"programmed,omitempty"` + Ready bool `protobuf:"varint,5,opt,name=ready,proto3" json:"ready,omitempty"` + // reason explains the first of resolved, programmed and ready that is + // false. + Reason string `protobuf:"bytes,6,opt,name=reason,proto3" json:"reason,omitempty"` + Message string `protobuf:"bytes,7,opt,name=message,proto3" json:"message,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DestinationStatus) Reset() { + *x = DestinationStatus{} + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DestinationStatus) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DestinationStatus) ProtoMessage() {} + +func (x *DestinationStatus) ProtoReflect() protoreflect.Message { + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[18] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DestinationStatus.ProtoReflect.Descriptor instead. +func (*DestinationStatus) Descriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP(), []int{18} +} + +func (x *DestinationStatus) GetDestination() *Destination { + if x != nil { + return x.Destination + } + return nil +} + +func (x *DestinationStatus) GetMode() Mode { + if x != nil { + return x.Mode + } + return Mode_MODE_UNSPECIFIED +} + +func (x *DestinationStatus) GetResolved() bool { + if x != nil { + return x.Resolved + } + return false +} + +func (x *DestinationStatus) GetProgrammed() bool { + if x != nil { + return x.Programmed + } + return false +} + +func (x *DestinationStatus) GetReady() bool { + if x != nil { + return x.Ready + } + return false +} + +func (x *DestinationStatus) GetReason() string { + if x != nil { + return x.Reason + } + return "" +} + +func (x *DestinationStatus) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +// ClaimStatus is a complete snapshot of one claim. A client drops a +// duplicate or reordered snapshot by comparing observed_generation and +// revision. +type ClaimStatus struct { + state protoimpl.MessageState `protogen:"open.v1"` + Key *ClaimKey `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + // generation is the accepted generation. + Generation uint64 `protobuf:"varint,2,opt,name=generation,proto3" json:"generation,omitempty"` + // observed_generation is the generation the conditions describe. + ObservedGeneration uint64 `protobuf:"varint,3,opt,name=observed_generation,json=observedGeneration,proto3" json:"observed_generation,omitempty"` + Conditions []*Condition `protobuf:"bytes,4,rep,name=conditions,proto3" json:"conditions,omitempty"` + Destinations []*DestinationStatus `protobuf:"bytes,5,rep,name=destinations,proto3" json:"destinations,omitempty"` + WithdrawalScope WithdrawalScope `protobuf:"varint,6,opt,name=withdrawal_scope,json=withdrawalScope,proto3,enum=galactic.worker.v1alpha1.WithdrawalScope" json:"withdrawal_scope,omitempty"` + // released is true in the last snapshot of a claim whose removal is + // confirmed. + Released bool `protobuf:"varint,7,opt,name=released,proto3" json:"released,omitempty"` + // revision increases with every snapshot Worker produces for the claim. + Revision uint64 `protobuf:"varint,8,opt,name=revision,proto3" json:"revision,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ClaimStatus) Reset() { + *x = ClaimStatus{} + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ClaimStatus) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClaimStatus) ProtoMessage() {} + +func (x *ClaimStatus) ProtoReflect() protoreflect.Message { + mi := &file_galactic_worker_v1alpha1_worker_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClaimStatus.ProtoReflect.Descriptor instead. +func (*ClaimStatus) Descriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP(), []int{19} +} + +func (x *ClaimStatus) GetKey() *ClaimKey { + if x != nil { + return x.Key + } + return nil +} + +func (x *ClaimStatus) GetGeneration() uint64 { + if x != nil { + return x.Generation + } + return 0 +} + +func (x *ClaimStatus) GetObservedGeneration() uint64 { + if x != nil { + return x.ObservedGeneration + } + return 0 +} + +func (x *ClaimStatus) GetConditions() []*Condition { + if x != nil { + return x.Conditions + } + return nil +} + +func (x *ClaimStatus) GetDestinations() []*DestinationStatus { + if x != nil { + return x.Destinations + } + return nil +} + +func (x *ClaimStatus) GetWithdrawalScope() WithdrawalScope { + if x != nil { + return x.WithdrawalScope + } + return WithdrawalScope_WITHDRAWAL_SCOPE_UNSPECIFIED +} + +func (x *ClaimStatus) GetReleased() bool { + if x != nil { + return x.Released + } + return false +} + +func (x *ClaimStatus) GetRevision() uint64 { + if x != nil { + return x.Revision + } + return 0 +} + +var File_galactic_worker_v1alpha1_worker_proto protoreflect.FileDescriptor + +const file_galactic_worker_v1alpha1_worker_proto_rawDesc = "" + + "\n" + + "%galactic/worker/v1alpha1/worker.proto\x12\x18galactic.worker.v1alpha1\x1a\x1egoogle/protobuf/duration.proto\"4\n" + + "\x16GetCapabilitiesRequest\x12\x1a\n" + + "\bversions\x18\x01 \x03(\tR\bversions\"\xa3\x02\n" + + "\x17GetCapabilitiesResponse\x12\x18\n" + + "\aversion\x18\x01 \x01(\tR\aversion\x12\x1d\n" + + "\n" + + "session_id\x18\x02 \x01(\tR\tsessionId\x12/\n" + + "\x05lease\x18\x03 \x01(\v2\x19.google.protobuf.DurationR\x05lease\x12@\n" + + "\x0erenew_interval\x18\x04 \x01(\v2\x19.google.protobuf.DurationR\rrenewInterval\x12H\n" + + "\fcapabilities\x18\x05 \x03(\v2$.galactic.worker.v1alpha1.CapabilityR\fcapabilities\x12\x12\n" + + "\x04node\x18\x06 \x01(\tR\x04node\"\xad\x01\n" + + "\n" + + "Capability\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12?\n" + + "\x05state\x18\x02 \x01(\x0e2).galactic.worker.v1alpha1.CapabilityStateR\x05state\x12\x16\n" + + "\x06reason\x18\x03 \x01(\tR\x06reason\x12\x18\n" + + "\amessage\x18\x04 \x01(\tR\amessage\x12\x18\n" + + "\aallowed\x18\x05 \x01(\bR\aallowed\"W\n" + + "\bClaimKey\x12\x12\n" + + "\x04node\x18\x01 \x01(\tR\x04node\x12\x10\n" + + "\x03vpc\x18\x02 \x01(\tR\x03vpc\x12%\n" + + "\x0eattachment_uid\x18\x03 \x01(\tR\rattachmentUid\"!\n" + + "\rAttachmentRef\x12\x10\n" + + "\x03uid\x18\x01 \x01(\tR\x03uid\"6\n" + + "\bShardRef\x12\x14\n" + + "\x05group\x18\x01 \x01(\tR\x05group\x12\x14\n" + + "\x05index\x18\x02 \x01(\rR\x05index\"4\n" + + "\n" + + "BackendRef\x12\x12\n" + + "\x04node\x18\x01 \x01(\tR\x04node\x12\x12\n" + + "\x04slot\x18\x02 \x01(\rR\x04slot\"\xe0\x01\n" + + "\vDestination\x12I\n" + + "\n" + + "attachment\x18\x01 \x01(\v2'.galactic.worker.v1alpha1.AttachmentRefH\x00R\n" + + "attachment\x12:\n" + + "\x05shard\x18\x02 \x01(\v2\".galactic.worker.v1alpha1.ShardRefH\x00R\x05shard\x12@\n" + + "\abackend\x18\x03 \x01(\v2$.galactic.worker.v1alpha1.BackendRefH\x00R\abackendB\b\n" + + "\x06target\"\xc7\x01\n" + + "\tClaimSpec\x12;\n" + + "\ahandoff\x18\x01 \x01(\x0e2!.galactic.worker.v1alpha1.HandoffR\ahandoff\x12I\n" + + "\fdestinations\x18\x02 \x03(\v2%.galactic.worker.v1alpha1.DestinationR\fdestinations\x122\n" + + "\x04mode\x18\x03 \x01(\x0e2\x1e.galactic.worker.v1alpha1.ModeR\x04mode\"\x9f\x01\n" + + "\x0eRequestRequest\x124\n" + + "\x03key\x18\x01 \x01(\v2\".galactic.worker.v1alpha1.ClaimKeyR\x03key\x12\x1e\n" + + "\n" + + "generation\x18\x02 \x01(\x04R\n" + + "generation\x127\n" + + "\x04spec\x18\x03 \x01(\v2#.galactic.worker.v1alpha1.ClaimSpecR\x04spec\"P\n" + + "\x0fRequestResponse\x12=\n" + + "\x06status\x18\x01 \x01(\v2%.galactic.worker.v1alpha1.ClaimStatusR\x06status\"\x9e\x01\n" + + "\rUpdateRequest\x124\n" + + "\x03key\x18\x01 \x01(\v2\".galactic.worker.v1alpha1.ClaimKeyR\x03key\x12\x1e\n" + + "\n" + + "generation\x18\x02 \x01(\x04R\n" + + "generation\x127\n" + + "\x04spec\x18\x03 \x01(\v2#.galactic.worker.v1alpha1.ClaimSpecR\x04spec\"O\n" + + "\x0eUpdateResponse\x12=\n" + + "\x06status\x18\x01 \x01(\v2%.galactic.worker.v1alpha1.ClaimStatusR\x06status\"f\n" + + "\x0eReleaseRequest\x124\n" + + "\x03key\x18\x01 \x01(\v2\".galactic.worker.v1alpha1.ClaimKeyR\x03key\x12\x1e\n" + + "\n" + + "generation\x18\x02 \x01(\x04R\n" + + "generation\"P\n" + + "\x0fReleaseResponse\x12=\n" + + "\x06status\x18\x01 \x01(\v2%.galactic.worker.v1alpha1.ClaimStatusR\x06status\"F\n" + + "\x0eObserveRequest\x124\n" + + "\x03key\x18\x01 \x01(\v2\".galactic.worker.v1alpha1.ClaimKeyR\x03key\"P\n" + + "\x0fObserveResponse\x12=\n" + + "\x06status\x18\x01 \x01(\v2%.galactic.worker.v1alpha1.ClaimStatusR\x06status\"\x94\x01\n" + + "\tCondition\x12\x12\n" + + "\x04type\x18\x01 \x01(\tR\x04type\x12A\n" + + "\x06status\x18\x02 \x01(\x0e2).galactic.worker.v1alpha1.ConditionStatusR\x06status\x12\x16\n" + + "\x06reason\x18\x03 \x01(\tR\x06reason\x12\x18\n" + + "\amessage\x18\x04 \x01(\tR\amessage\"\x94\x02\n" + + "\x11DestinationStatus\x12G\n" + + "\vdestination\x18\x01 \x01(\v2%.galactic.worker.v1alpha1.DestinationR\vdestination\x122\n" + + "\x04mode\x18\x02 \x01(\x0e2\x1e.galactic.worker.v1alpha1.ModeR\x04mode\x12\x1a\n" + + "\bresolved\x18\x03 \x01(\bR\bresolved\x12\x1e\n" + + "\n" + + "programmed\x18\x04 \x01(\bR\n" + + "programmed\x12\x14\n" + + "\x05ready\x18\x05 \x01(\bR\x05ready\x12\x16\n" + + "\x06reason\x18\x06 \x01(\tR\x06reason\x12\x18\n" + + "\amessage\x18\a \x01(\tR\amessage\"\xb8\x03\n" + + "\vClaimStatus\x124\n" + + "\x03key\x18\x01 \x01(\v2\".galactic.worker.v1alpha1.ClaimKeyR\x03key\x12\x1e\n" + + "\n" + + "generation\x18\x02 \x01(\x04R\n" + + "generation\x12/\n" + + "\x13observed_generation\x18\x03 \x01(\x04R\x12observedGeneration\x12C\n" + + "\n" + + "conditions\x18\x04 \x03(\v2#.galactic.worker.v1alpha1.ConditionR\n" + + "conditions\x12O\n" + + "\fdestinations\x18\x05 \x03(\v2+.galactic.worker.v1alpha1.DestinationStatusR\fdestinations\x12T\n" + + "\x10withdrawal_scope\x18\x06 \x01(\x0e2).galactic.worker.v1alpha1.WithdrawalScopeR\x0fwithdrawalScope\x12\x1a\n" + + "\breleased\x18\a \x01(\bR\breleased\x12\x1a\n" + + "\brevision\x18\b \x01(\x04R\brevision*u\n" + + "\x0fCapabilityState\x12 \n" + + "\x1cCAPABILITY_STATE_UNSPECIFIED\x10\x00\x12\x1e\n" + + "\x1aCAPABILITY_STATE_AVAILABLE\x10\x01\x12 \n" + + "\x1cCAPABILITY_STATE_UNAVAILABLE\x10\x02*w\n" + + "\aHandoff\x12\x17\n" + + "\x13HANDOFF_UNSPECIFIED\x10\x00\x12\x0f\n" + + "\vHANDOFF_TAP\x10\x01\x12\x15\n" + + "\x11HANDOFF_VETH_PAIR\x10\x02\x12\x17\n" + + "\x13HANDOFF_ROUTED_LINK\x10\x03\x12\x12\n" + + "\x0eHANDOFF_SOCKET\x10\x04*L\n" + + "\x04Mode\x12\x14\n" + + "\x10MODE_UNSPECIFIED\x10\x00\x12\r\n" + + "\tMODE_AUTO\x10\x01\x12\x0e\n" + + "\n" + + "MODE_LOCAL\x10\x02\x12\x0f\n" + + "\vMODE_FABRIC\x10\x03*\x88\x01\n" + + "\x0fConditionStatus\x12 \n" + + "\x1cCONDITION_STATUS_UNSPECIFIED\x10\x00\x12\x19\n" + + "\x15CONDITION_STATUS_TRUE\x10\x01\x12\x1a\n" + + "\x16CONDITION_STATUS_FALSE\x10\x02\x12\x1c\n" + + "\x18CONDITION_STATUS_UNKNOWN\x10\x03*\xa8\x01\n" + + "\x0fWithdrawalScope\x12 \n" + + "\x1cWITHDRAWAL_SCOPE_UNSPECIFIED\x10\x00\x12 \n" + + "\x1cWITHDRAWAL_SCOPE_DESTINATION\x10\x01\x12\x1a\n" + + "\x16WITHDRAWAL_SCOPE_CLAIM\x10\x02\x12\x1b\n" + + "\x17WITHDRAWAL_SCOPE_CALLER\x10\x03\x12\x18\n" + + "\x14WITHDRAWAL_SCOPE_VPC\x10\x042\xff\x03\n" + + "\x06Worker\x12v\n" + + "\x0fGetCapabilities\x120.galactic.worker.v1alpha1.GetCapabilitiesRequest\x1a1.galactic.worker.v1alpha1.GetCapabilitiesResponse\x12^\n" + + "\aRequest\x12(.galactic.worker.v1alpha1.RequestRequest\x1a).galactic.worker.v1alpha1.RequestResponse\x12[\n" + + "\x06Update\x12'.galactic.worker.v1alpha1.UpdateRequest\x1a(.galactic.worker.v1alpha1.UpdateResponse\x12`\n" + + "\aObserve\x12(.galactic.worker.v1alpha1.ObserveRequest\x1a).galactic.worker.v1alpha1.ObserveResponse0\x01\x12^\n" + + "\aRelease\x12(.galactic.worker.v1alpha1.ReleaseRequest\x1a).galactic.worker.v1alpha1.ReleaseResponseBCZAgo.datum.net/galactic/api/galactic/worker/v1alpha1;workerv1alpha1b\x06proto3" + +var ( + file_galactic_worker_v1alpha1_worker_proto_rawDescOnce sync.Once + file_galactic_worker_v1alpha1_worker_proto_rawDescData []byte +) + +func file_galactic_worker_v1alpha1_worker_proto_rawDescGZIP() []byte { + file_galactic_worker_v1alpha1_worker_proto_rawDescOnce.Do(func() { + file_galactic_worker_v1alpha1_worker_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_galactic_worker_v1alpha1_worker_proto_rawDesc), len(file_galactic_worker_v1alpha1_worker_proto_rawDesc))) + }) + return file_galactic_worker_v1alpha1_worker_proto_rawDescData +} + +var file_galactic_worker_v1alpha1_worker_proto_enumTypes = make([]protoimpl.EnumInfo, 5) +var file_galactic_worker_v1alpha1_worker_proto_msgTypes = make([]protoimpl.MessageInfo, 20) +var file_galactic_worker_v1alpha1_worker_proto_goTypes = []any{ + (CapabilityState)(0), // 0: galactic.worker.v1alpha1.CapabilityState + (Handoff)(0), // 1: galactic.worker.v1alpha1.Handoff + (Mode)(0), // 2: galactic.worker.v1alpha1.Mode + (ConditionStatus)(0), // 3: galactic.worker.v1alpha1.ConditionStatus + (WithdrawalScope)(0), // 4: galactic.worker.v1alpha1.WithdrawalScope + (*GetCapabilitiesRequest)(nil), // 5: galactic.worker.v1alpha1.GetCapabilitiesRequest + (*GetCapabilitiesResponse)(nil), // 6: galactic.worker.v1alpha1.GetCapabilitiesResponse + (*Capability)(nil), // 7: galactic.worker.v1alpha1.Capability + (*ClaimKey)(nil), // 8: galactic.worker.v1alpha1.ClaimKey + (*AttachmentRef)(nil), // 9: galactic.worker.v1alpha1.AttachmentRef + (*ShardRef)(nil), // 10: galactic.worker.v1alpha1.ShardRef + (*BackendRef)(nil), // 11: galactic.worker.v1alpha1.BackendRef + (*Destination)(nil), // 12: galactic.worker.v1alpha1.Destination + (*ClaimSpec)(nil), // 13: galactic.worker.v1alpha1.ClaimSpec + (*RequestRequest)(nil), // 14: galactic.worker.v1alpha1.RequestRequest + (*RequestResponse)(nil), // 15: galactic.worker.v1alpha1.RequestResponse + (*UpdateRequest)(nil), // 16: galactic.worker.v1alpha1.UpdateRequest + (*UpdateResponse)(nil), // 17: galactic.worker.v1alpha1.UpdateResponse + (*ReleaseRequest)(nil), // 18: galactic.worker.v1alpha1.ReleaseRequest + (*ReleaseResponse)(nil), // 19: galactic.worker.v1alpha1.ReleaseResponse + (*ObserveRequest)(nil), // 20: galactic.worker.v1alpha1.ObserveRequest + (*ObserveResponse)(nil), // 21: galactic.worker.v1alpha1.ObserveResponse + (*Condition)(nil), // 22: galactic.worker.v1alpha1.Condition + (*DestinationStatus)(nil), // 23: galactic.worker.v1alpha1.DestinationStatus + (*ClaimStatus)(nil), // 24: galactic.worker.v1alpha1.ClaimStatus + (*durationpb.Duration)(nil), // 25: google.protobuf.Duration +} +var file_galactic_worker_v1alpha1_worker_proto_depIdxs = []int32{ + 25, // 0: galactic.worker.v1alpha1.GetCapabilitiesResponse.lease:type_name -> google.protobuf.Duration + 25, // 1: galactic.worker.v1alpha1.GetCapabilitiesResponse.renew_interval:type_name -> google.protobuf.Duration + 7, // 2: galactic.worker.v1alpha1.GetCapabilitiesResponse.capabilities:type_name -> galactic.worker.v1alpha1.Capability + 0, // 3: galactic.worker.v1alpha1.Capability.state:type_name -> galactic.worker.v1alpha1.CapabilityState + 9, // 4: galactic.worker.v1alpha1.Destination.attachment:type_name -> galactic.worker.v1alpha1.AttachmentRef + 10, // 5: galactic.worker.v1alpha1.Destination.shard:type_name -> galactic.worker.v1alpha1.ShardRef + 11, // 6: galactic.worker.v1alpha1.Destination.backend:type_name -> galactic.worker.v1alpha1.BackendRef + 1, // 7: galactic.worker.v1alpha1.ClaimSpec.handoff:type_name -> galactic.worker.v1alpha1.Handoff + 12, // 8: galactic.worker.v1alpha1.ClaimSpec.destinations:type_name -> galactic.worker.v1alpha1.Destination + 2, // 9: galactic.worker.v1alpha1.ClaimSpec.mode:type_name -> galactic.worker.v1alpha1.Mode + 8, // 10: galactic.worker.v1alpha1.RequestRequest.key:type_name -> galactic.worker.v1alpha1.ClaimKey + 13, // 11: galactic.worker.v1alpha1.RequestRequest.spec:type_name -> galactic.worker.v1alpha1.ClaimSpec + 24, // 12: galactic.worker.v1alpha1.RequestResponse.status:type_name -> galactic.worker.v1alpha1.ClaimStatus + 8, // 13: galactic.worker.v1alpha1.UpdateRequest.key:type_name -> galactic.worker.v1alpha1.ClaimKey + 13, // 14: galactic.worker.v1alpha1.UpdateRequest.spec:type_name -> galactic.worker.v1alpha1.ClaimSpec + 24, // 15: galactic.worker.v1alpha1.UpdateResponse.status:type_name -> galactic.worker.v1alpha1.ClaimStatus + 8, // 16: galactic.worker.v1alpha1.ReleaseRequest.key:type_name -> galactic.worker.v1alpha1.ClaimKey + 24, // 17: galactic.worker.v1alpha1.ReleaseResponse.status:type_name -> galactic.worker.v1alpha1.ClaimStatus + 8, // 18: galactic.worker.v1alpha1.ObserveRequest.key:type_name -> galactic.worker.v1alpha1.ClaimKey + 24, // 19: galactic.worker.v1alpha1.ObserveResponse.status:type_name -> galactic.worker.v1alpha1.ClaimStatus + 3, // 20: galactic.worker.v1alpha1.Condition.status:type_name -> galactic.worker.v1alpha1.ConditionStatus + 12, // 21: galactic.worker.v1alpha1.DestinationStatus.destination:type_name -> galactic.worker.v1alpha1.Destination + 2, // 22: galactic.worker.v1alpha1.DestinationStatus.mode:type_name -> galactic.worker.v1alpha1.Mode + 8, // 23: galactic.worker.v1alpha1.ClaimStatus.key:type_name -> galactic.worker.v1alpha1.ClaimKey + 22, // 24: galactic.worker.v1alpha1.ClaimStatus.conditions:type_name -> galactic.worker.v1alpha1.Condition + 23, // 25: galactic.worker.v1alpha1.ClaimStatus.destinations:type_name -> galactic.worker.v1alpha1.DestinationStatus + 4, // 26: galactic.worker.v1alpha1.ClaimStatus.withdrawal_scope:type_name -> galactic.worker.v1alpha1.WithdrawalScope + 5, // 27: galactic.worker.v1alpha1.Worker.GetCapabilities:input_type -> galactic.worker.v1alpha1.GetCapabilitiesRequest + 14, // 28: galactic.worker.v1alpha1.Worker.Request:input_type -> galactic.worker.v1alpha1.RequestRequest + 16, // 29: galactic.worker.v1alpha1.Worker.Update:input_type -> galactic.worker.v1alpha1.UpdateRequest + 20, // 30: galactic.worker.v1alpha1.Worker.Observe:input_type -> galactic.worker.v1alpha1.ObserveRequest + 18, // 31: galactic.worker.v1alpha1.Worker.Release:input_type -> galactic.worker.v1alpha1.ReleaseRequest + 6, // 32: galactic.worker.v1alpha1.Worker.GetCapabilities:output_type -> galactic.worker.v1alpha1.GetCapabilitiesResponse + 15, // 33: galactic.worker.v1alpha1.Worker.Request:output_type -> galactic.worker.v1alpha1.RequestResponse + 17, // 34: galactic.worker.v1alpha1.Worker.Update:output_type -> galactic.worker.v1alpha1.UpdateResponse + 21, // 35: galactic.worker.v1alpha1.Worker.Observe:output_type -> galactic.worker.v1alpha1.ObserveResponse + 19, // 36: galactic.worker.v1alpha1.Worker.Release:output_type -> galactic.worker.v1alpha1.ReleaseResponse + 32, // [32:37] is the sub-list for method output_type + 27, // [27:32] is the sub-list for method input_type + 27, // [27:27] is the sub-list for extension type_name + 27, // [27:27] is the sub-list for extension extendee + 0, // [0:27] is the sub-list for field type_name +} + +func init() { file_galactic_worker_v1alpha1_worker_proto_init() } +func file_galactic_worker_v1alpha1_worker_proto_init() { + if File_galactic_worker_v1alpha1_worker_proto != nil { + return + } + file_galactic_worker_v1alpha1_worker_proto_msgTypes[7].OneofWrappers = []any{ + (*Destination_Attachment)(nil), + (*Destination_Shard)(nil), + (*Destination_Backend)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_galactic_worker_v1alpha1_worker_proto_rawDesc), len(file_galactic_worker_v1alpha1_worker_proto_rawDesc)), + NumEnums: 5, + NumMessages: 20, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_galactic_worker_v1alpha1_worker_proto_goTypes, + DependencyIndexes: file_galactic_worker_v1alpha1_worker_proto_depIdxs, + EnumInfos: file_galactic_worker_v1alpha1_worker_proto_enumTypes, + MessageInfos: file_galactic_worker_v1alpha1_worker_proto_msgTypes, + }.Build() + File_galactic_worker_v1alpha1_worker_proto = out.File + file_galactic_worker_v1alpha1_worker_proto_goTypes = nil + file_galactic_worker_v1alpha1_worker_proto_depIdxs = nil +} diff --git a/api/galactic/worker/v1alpha1/worker.proto b/api/galactic/worker/v1alpha1/worker.proto new file mode 100644 index 00000000..8bbfd367 --- /dev/null +++ b/api/galactic/worker/v1alpha1/worker.proto @@ -0,0 +1,251 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// The Galactic Worker service contract: the node-local API a service agent +// calls to deliver tenant traffic between its own handoff point and tenant +// destinations. docs/worker/contract.md is the normative description of every +// operation, state and error. +// +// Worker serves it on a unix socket only. Every call except GetCapabilities +// carries the session ID GetCapabilities returned, in the +// x-galactic-worker-session metadata key, and every call carries a projected +// ServiceAccount token with audience galactic-worker as +// "authorization: Bearer ". Every refusal is a gRPC status carrying a +// google.rpc.ErrorInfo whose domain is worker.galactic.datumapis.com and whose +// reason is one of the machine reasons the contract lists. + +syntax = "proto3"; + +package galactic.worker.v1alpha1; + +import "google/protobuf/duration.proto"; + +option go_package = "go.datum.net/galactic/api/galactic/worker/v1alpha1;workerv1alpha1"; + +// Worker runs once per node. The contract names the service Worker. +// buf:lint:ignore SERVICE_SUFFIX +service Worker { + // GetCapabilities negotiates the API version and starts a session. It must + // be the first call of a session. + rpc GetCapabilities(GetCapabilitiesRequest) returns (GetCapabilitiesResponse); + // Request creates a claim. It returns once the claim is validated, + // authorized and durably recorded, not once it is programmed. + rpc Request(RequestRequest) returns (RequestResponse); + // Update replaces the spec of an existing claim. + rpc Update(UpdateRequest) returns (UpdateResponse); + // Observe streams complete status snapshots for one claim or for every + // claim of the caller. An open stream renews the session lease. + rpc Observe(ObserveRequest) returns (stream ObserveResponse); + // Release removes a claim at its current generation. + rpc Release(ReleaseRequest) returns (ReleaseResponse); +} + +message GetCapabilitiesRequest { + // versions are the API versions the client supports, such as "v1alpha1". + // Worker picks the highest one it also serves. + repeated string versions = 1; +} + +message GetCapabilitiesResponse { + // version is the API version chosen for the session. + string version = 1; + // session_id identifies the session. Send it with every later call. + string session_id = 2; + // lease is how long the session lives without a renewal. + google.protobuf.Duration lease = 3; + // renew_interval is how often a client renews the lease, by a call or an + // open Observe stream. + google.protobuf.Duration renew_interval = 4; + // capabilities lists every capability with its current state. + repeated Capability capabilities = 5; + // node is the node this Worker runs on, which every claim key must name. + string node = 6; +} + +// CapabilityState reports whether a capability can be programmed now. +enum CapabilityState { + CAPABILITY_STATE_UNSPECIFIED = 0; + CAPABILITY_STATE_AVAILABLE = 1; + CAPABILITY_STATE_UNAVAILABLE = 2; +} + +message Capability { + // name is one of local.tap, local.veth_pair, local.routed_link, + // local.socket and fabric.srv6. + string name = 1; + CapabilityState state = 2; + // reason is a machine reason when the capability is unavailable, such as + // ModuleNotRunning, DatapathNotLoaded or HookUnsupported. + string reason = 3; + string message = 4; + // allowed reports whether the caller's ServiceAccount may use it. + bool allowed = 5; +} + +// ClaimKey names a claim. The caller is part of the key too, but Worker +// derives it from the connection; a client never supplies it. +message ClaimKey { + string node = 1; + // vpc is the base62 identifier of the VPC of the attachment. + string vpc = 2; + // attachment_uid is the UID of the VPCAttachment or EdgeAttachment that is + // the service's handoff point in the VPC. + string attachment_uid = 3; +} + +// Handoff is how traffic passes between the service and the tenant side. +enum Handoff { + HANDOFF_UNSPECIFIED = 0; + HANDOFF_TAP = 1; + HANDOFF_VETH_PAIR = 2; + HANDOFF_ROUTED_LINK = 3; + HANDOFF_SOCKET = 4; +} + +// Mode is a delivery mode: the preference in a spec, or the result of +// resolving a destination. +enum Mode { + MODE_UNSPECIFIED = 0; + // MODE_AUTO lets Worker resolve each destination. A spec only. + MODE_AUTO = 1; + MODE_LOCAL = 2; + MODE_FABRIC = 3; +} + +message AttachmentRef { + // uid is a VPCAttachment or EdgeAttachment UID. + string uid = 1; +} + +message ShardRef { + // group names the shard group. + string group = 1; + uint32 index = 2; +} + +message BackendRef { + // node is the backend's node name. + string node = 1; + // slot is the slot the gateway encodes in the backend SID. + uint32 slot = 2; +} + +// Destination is one tenant destination of a claim. +message Destination { + oneof target { + AttachmentRef attachment = 1; + // shard is resolved by #1437. Until then it reports + // DestinationKindUnsupported. + ShardRef shard = 2; + // backend is resolved by #1438. Until then it reports + // DestinationKindUnsupported. + BackendRef backend = 3; + } +} + +message ClaimSpec { + Handoff handoff = 1; + repeated Destination destinations = 2; + // mode is the mode preference. Unset means MODE_AUTO. + Mode mode = 3; +} + +message RequestRequest { + ClaimKey key = 1; + // generation is client-assigned and must be at least 1. + uint64 generation = 2; + ClaimSpec spec = 3; +} + +message RequestResponse { + ClaimStatus status = 1; +} + +message UpdateRequest { + ClaimKey key = 1; + uint64 generation = 2; + ClaimSpec spec = 3; +} + +message UpdateResponse { + ClaimStatus status = 1; +} + +message ReleaseRequest { + ClaimKey key = 1; + // generation must equal the claim's current generation. + uint64 generation = 2; +} + +message ReleaseResponse { + // status is the claim's status after the release was recorded, or unset + // when no such claim exists. + ClaimStatus status = 1; +} + +message ObserveRequest { + // key selects one claim. Unset selects every claim of the caller. + ClaimKey key = 1; +} + +message ObserveResponse { + ClaimStatus status = 1; +} + +// ConditionStatus is the status of one condition. +enum ConditionStatus { + CONDITION_STATUS_UNSPECIFIED = 0; + CONDITION_STATUS_TRUE = 1; + CONDITION_STATUS_FALSE = 2; + CONDITION_STATUS_UNKNOWN = 3; +} + +// Condition is one of Admitted, Resolved, Programmed, Ready and Withdrawing. +message Condition { + string type = 1; + ConditionStatus status = 2; + string reason = 3; + string message = 4; +} + +// WithdrawalScope names what a withdrawal stops. +enum WithdrawalScope { + WITHDRAWAL_SCOPE_UNSPECIFIED = 0; + WITHDRAWAL_SCOPE_DESTINATION = 1; + WITHDRAWAL_SCOPE_CLAIM = 2; + WITHDRAWAL_SCOPE_CALLER = 3; + WITHDRAWAL_SCOPE_VPC = 4; +} + +message DestinationStatus { + Destination destination = 1; + // mode is MODE_LOCAL or MODE_FABRIC once resolved. + Mode mode = 2; + bool resolved = 3; + bool programmed = 4; + bool ready = 5; + // reason explains the first of resolved, programmed and ready that is + // false. + string reason = 6; + string message = 7; +} + +// ClaimStatus is a complete snapshot of one claim. A client drops a +// duplicate or reordered snapshot by comparing observed_generation and +// revision. +message ClaimStatus { + ClaimKey key = 1; + // generation is the accepted generation. + uint64 generation = 2; + // observed_generation is the generation the conditions describe. + uint64 observed_generation = 3; + repeated Condition conditions = 4; + repeated DestinationStatus destinations = 5; + WithdrawalScope withdrawal_scope = 6; + // released is true in the last snapshot of a claim whose removal is + // confirmed. + bool released = 7; + // revision increases with every snapshot Worker produces for the claim. + uint64 revision = 8; +} diff --git a/api/galactic/worker/v1alpha1/worker_grpc.pb.go b/api/galactic/worker/v1alpha1/worker_grpc.pb.go new file mode 100644 index 00000000..e47489de --- /dev/null +++ b/api/galactic/worker/v1alpha1/worker_grpc.pb.go @@ -0,0 +1,316 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// The Galactic Worker service contract: the node-local API a service agent +// calls to deliver tenant traffic between its own handoff point and tenant +// destinations. docs/worker/contract.md is the normative description of every +// operation, state and error. +// +// Worker serves it on a unix socket only. Every call except GetCapabilities +// carries the session ID GetCapabilities returned, in the +// x-galactic-worker-session metadata key, and every call carries a projected +// ServiceAccount token with audience galactic-worker as +// "authorization: Bearer ". Every refusal is a gRPC status carrying a +// google.rpc.ErrorInfo whose domain is worker.galactic.datumapis.com and whose +// reason is one of the machine reasons the contract lists. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc (unknown) +// source: galactic/worker/v1alpha1/worker.proto + +package workerv1alpha1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + Worker_GetCapabilities_FullMethodName = "/galactic.worker.v1alpha1.Worker/GetCapabilities" + Worker_Request_FullMethodName = "/galactic.worker.v1alpha1.Worker/Request" + Worker_Update_FullMethodName = "/galactic.worker.v1alpha1.Worker/Update" + Worker_Observe_FullMethodName = "/galactic.worker.v1alpha1.Worker/Observe" + Worker_Release_FullMethodName = "/galactic.worker.v1alpha1.Worker/Release" +) + +// WorkerClient is the client API for Worker service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Worker runs once per node. The contract names the service Worker. +// buf:lint:ignore SERVICE_SUFFIX +type WorkerClient interface { + // GetCapabilities negotiates the API version and starts a session. It must + // be the first call of a session. + GetCapabilities(ctx context.Context, in *GetCapabilitiesRequest, opts ...grpc.CallOption) (*GetCapabilitiesResponse, error) + // Request creates a claim. It returns once the claim is validated, + // authorized and durably recorded, not once it is programmed. + Request(ctx context.Context, in *RequestRequest, opts ...grpc.CallOption) (*RequestResponse, error) + // Update replaces the spec of an existing claim. + Update(ctx context.Context, in *UpdateRequest, opts ...grpc.CallOption) (*UpdateResponse, error) + // Observe streams complete status snapshots for one claim or for every + // claim of the caller. An open stream renews the session lease. + Observe(ctx context.Context, in *ObserveRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ObserveResponse], error) + // Release removes a claim at its current generation. + Release(ctx context.Context, in *ReleaseRequest, opts ...grpc.CallOption) (*ReleaseResponse, error) +} + +type workerClient struct { + cc grpc.ClientConnInterface +} + +func NewWorkerClient(cc grpc.ClientConnInterface) WorkerClient { + return &workerClient{cc} +} + +func (c *workerClient) GetCapabilities(ctx context.Context, in *GetCapabilitiesRequest, opts ...grpc.CallOption) (*GetCapabilitiesResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetCapabilitiesResponse) + err := c.cc.Invoke(ctx, Worker_GetCapabilities_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *workerClient) Request(ctx context.Context, in *RequestRequest, opts ...grpc.CallOption) (*RequestResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(RequestResponse) + err := c.cc.Invoke(ctx, Worker_Request_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *workerClient) Update(ctx context.Context, in *UpdateRequest, opts ...grpc.CallOption) (*UpdateResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(UpdateResponse) + err := c.cc.Invoke(ctx, Worker_Update_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *workerClient) Observe(ctx context.Context, in *ObserveRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ObserveResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &Worker_ServiceDesc.Streams[0], Worker_Observe_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[ObserveRequest, ObserveResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Worker_ObserveClient = grpc.ServerStreamingClient[ObserveResponse] + +func (c *workerClient) Release(ctx context.Context, in *ReleaseRequest, opts ...grpc.CallOption) (*ReleaseResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ReleaseResponse) + err := c.cc.Invoke(ctx, Worker_Release_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// WorkerServer is the server API for Worker service. +// All implementations must embed UnimplementedWorkerServer +// for forward compatibility. +// +// Worker runs once per node. The contract names the service Worker. +// buf:lint:ignore SERVICE_SUFFIX +type WorkerServer interface { + // GetCapabilities negotiates the API version and starts a session. It must + // be the first call of a session. + GetCapabilities(context.Context, *GetCapabilitiesRequest) (*GetCapabilitiesResponse, error) + // Request creates a claim. It returns once the claim is validated, + // authorized and durably recorded, not once it is programmed. + Request(context.Context, *RequestRequest) (*RequestResponse, error) + // Update replaces the spec of an existing claim. + Update(context.Context, *UpdateRequest) (*UpdateResponse, error) + // Observe streams complete status snapshots for one claim or for every + // claim of the caller. An open stream renews the session lease. + Observe(*ObserveRequest, grpc.ServerStreamingServer[ObserveResponse]) error + // Release removes a claim at its current generation. + Release(context.Context, *ReleaseRequest) (*ReleaseResponse, error) + mustEmbedUnimplementedWorkerServer() +} + +// UnimplementedWorkerServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedWorkerServer struct{} + +func (UnimplementedWorkerServer) GetCapabilities(context.Context, *GetCapabilitiesRequest) (*GetCapabilitiesResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetCapabilities not implemented") +} +func (UnimplementedWorkerServer) Request(context.Context, *RequestRequest) (*RequestResponse, error) { + return nil, status.Error(codes.Unimplemented, "method Request not implemented") +} +func (UnimplementedWorkerServer) Update(context.Context, *UpdateRequest) (*UpdateResponse, error) { + return nil, status.Error(codes.Unimplemented, "method Update not implemented") +} +func (UnimplementedWorkerServer) Observe(*ObserveRequest, grpc.ServerStreamingServer[ObserveResponse]) error { + return status.Error(codes.Unimplemented, "method Observe not implemented") +} +func (UnimplementedWorkerServer) Release(context.Context, *ReleaseRequest) (*ReleaseResponse, error) { + return nil, status.Error(codes.Unimplemented, "method Release not implemented") +} +func (UnimplementedWorkerServer) mustEmbedUnimplementedWorkerServer() {} +func (UnimplementedWorkerServer) testEmbeddedByValue() {} + +// UnsafeWorkerServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to WorkerServer will +// result in compilation errors. +type UnsafeWorkerServer interface { + mustEmbedUnimplementedWorkerServer() +} + +func RegisterWorkerServer(s grpc.ServiceRegistrar, srv WorkerServer) { + // If the following call panics, it indicates UnimplementedWorkerServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&Worker_ServiceDesc, srv) +} + +func _Worker_GetCapabilities_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetCapabilitiesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(WorkerServer).GetCapabilities(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Worker_GetCapabilities_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(WorkerServer).GetCapabilities(ctx, req.(*GetCapabilitiesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Worker_Request_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RequestRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(WorkerServer).Request(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Worker_Request_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(WorkerServer).Request(ctx, req.(*RequestRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Worker_Update_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(UpdateRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(WorkerServer).Update(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Worker_Update_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(WorkerServer).Update(ctx, req.(*UpdateRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Worker_Observe_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(ObserveRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(WorkerServer).Observe(m, &grpc.GenericServerStream[ObserveRequest, ObserveResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Worker_ObserveServer = grpc.ServerStreamingServer[ObserveResponse] + +func _Worker_Release_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReleaseRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(WorkerServer).Release(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Worker_Release_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(WorkerServer).Release(ctx, req.(*ReleaseRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// Worker_ServiceDesc is the grpc.ServiceDesc for Worker service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Worker_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "galactic.worker.v1alpha1.Worker", + HandlerType: (*WorkerServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetCapabilities", + Handler: _Worker_GetCapabilities_Handler, + }, + { + MethodName: "Request", + Handler: _Worker_Request_Handler, + }, + { + MethodName: "Update", + Handler: _Worker_Update_Handler, + }, + { + MethodName: "Release", + Handler: _Worker_Release_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "Observe", + Handler: _Worker_Observe_Handler, + ServerStreams: true, + }, + }, + Metadata: "galactic/worker/v1alpha1/worker.proto", +} diff --git a/pkg/workerclient/client.go b/pkg/workerclient/client.go new file mode 100644 index 00000000..cb90123c --- /dev/null +++ b/pkg/workerclient/client.go @@ -0,0 +1,351 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package workerclient is the client library of Galactic Worker's service +// API (docs/worker/contract.md). It is part of the published interface: it +// imports only the API package and gRPC, never Galactic internals. +// +// A Client dials Worker's unix socket, negotiates the API version and opens +// a session with GetCapabilities, and keeps that session alive. Every call +// carries the caller's projected ServiceAccount token, read again for each +// call because the kubelet rotates it. Calls that the contract says to retry +// (UNAVAILABLE, RESOURCE_EXHAUSTED, an expired identity or a lost session) +// are retried with exponential backoff from 200 ms to 30 s with full jitter. +package workerclient + +import ( + "context" + "errors" + "fmt" + "math/rand/v2" + "net" + "os" + "strings" + "sync" + "time" + + "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" +) + +// DefaultTokenFile is where a pod mounts its projected Worker token by +// convention. +const DefaultTokenFile = "/var/run/secrets/galactic-worker/token" + +// Options configure a Client. +type Options struct { + // SocketPath is Worker's socket, workerv1alpha1.DefaultSocketPath when + // empty. + SocketPath string + // TokenFile is the projected token, DefaultTokenFile when empty and + // Token is nil. + TokenFile string + // Token returns the bearer token for a call. It overrides TokenFile. + Token func() (string, error) + // Versions are the API versions the client supports, newest last. + Versions []string + // RetryMin, RetryMax and Attempts bound retries: 200 ms, 30 s and 8 by + // default. + RetryMin time.Duration + RetryMax time.Duration + Attempts int + // NoKeepAlive disables the background session renewal. A client that + // keeps an Observe stream open does not need it. + NoKeepAlive bool +} + +// Client is a Worker client. +type Client struct { + opts Options + cc *grpc.ClientConn + api workerv1alpha1.WorkerClient + + mu sync.Mutex + session string + caps *workerv1alpha1.GetCapabilitiesResponse + + stop chan struct{} + done sync.WaitGroup +} + +// Dial connects to Worker and opens a session. +func Dial(ctx context.Context, opts Options) (*Client, error) { + if opts.SocketPath == "" { + opts.SocketPath = workerv1alpha1.DefaultSocketPath + } + if opts.Token == nil { + file := opts.TokenFile + if file == "" { + file = DefaultTokenFile + } + opts.Token = func() (string, error) { + b, err := os.ReadFile(file) + if err != nil { + return "", err + } + return strings.TrimSpace(string(b)), nil + } + } + if len(opts.Versions) == 0 { + opts.Versions = []string{workerv1alpha1.Version} + } + if opts.RetryMin == 0 { + opts.RetryMin = 200 * time.Millisecond + } + if opts.RetryMax == 0 { + opts.RetryMax = 30 * time.Second + } + if opts.Attempts == 0 { + opts.Attempts = 8 + } + socket := opts.SocketPath + cc, err := grpc.NewClient("passthrough:///galactic-worker", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", socket) + })) + if err != nil { + return nil, fmt.Errorf("workerclient: %w", err) + } + c := &Client{opts: opts, cc: cc, api: workerv1alpha1.NewWorkerClient(cc), stop: make(chan struct{})} + if err := c.openSession(ctx, false); err != nil { + _ = cc.Close() + return nil, err + } + if !opts.NoKeepAlive { + c.done.Add(1) + go c.keepAlive() + } + return c, nil +} + +// Close stops the session renewal and closes the connection. Claims stay: +// they belong to the caller's identity, and a client that reconnects within +// the lease finds them unchanged. +func (c *Client) Close() error { + close(c.stop) + c.done.Wait() + return c.cc.Close() +} + +// Capabilities returns the latest GetCapabilities response. +func (c *Client) Capabilities() *workerv1alpha1.GetCapabilitiesResponse { + c.mu.Lock() + defer c.mu.Unlock() + return c.caps +} + +// Session returns the current session ID. +func (c *Client) Session() string { + c.mu.Lock() + defer c.mu.Unlock() + return c.session +} + +// Request creates a claim. +func (c *Client) Request(ctx context.Context, key *workerv1alpha1.ClaimKey, generation uint64, + spec *workerv1alpha1.ClaimSpec) (*workerv1alpha1.ClaimStatus, error) { + var out *workerv1alpha1.ClaimStatus + err := c.retry(ctx, func(ctx context.Context) error { + r, err := c.api.Request(ctx, &workerv1alpha1.RequestRequest{Key: key, Generation: generation, Spec: spec}) + out = r.GetStatus() + return err + }) + return out, err +} + +// Update replaces a claim's spec. +func (c *Client) Update(ctx context.Context, key *workerv1alpha1.ClaimKey, generation uint64, + spec *workerv1alpha1.ClaimSpec) (*workerv1alpha1.ClaimStatus, error) { + var out *workerv1alpha1.ClaimStatus + err := c.retry(ctx, func(ctx context.Context) error { + r, err := c.api.Update(ctx, &workerv1alpha1.UpdateRequest{Key: key, Generation: generation, Spec: spec}) + out = r.GetStatus() + return err + }) + return out, err +} + +// Release removes a claim at its current generation. It returns nil status +// when no such claim exists. +func (c *Client) Release(ctx context.Context, key *workerv1alpha1.ClaimKey, generation uint64) ( + *workerv1alpha1.ClaimStatus, error) { + var out *workerv1alpha1.ClaimStatus + err := c.retry(ctx, func(ctx context.Context) error { + r, err := c.api.Release(ctx, &workerv1alpha1.ReleaseRequest{Key: key, Generation: generation}) + out = r.GetStatus() + return err + }) + return out, err +} + +// Observer receives status snapshots. +type Observer struct { + stream grpc.ServerStreamingClient[workerv1alpha1.ObserveResponse] + last map[string]uint64 +} + +// Recv returns the next snapshot, dropping duplicates and reorderings. +func (o *Observer) Recv() (*workerv1alpha1.ClaimStatus, error) { + for { + r, err := o.stream.Recv() + if err != nil { + return nil, err + } + st := r.GetStatus() + k := st.GetKey() + id := k.GetNode() + "/" + k.GetVpc() + "/" + k.GetAttachmentUid() + if st.GetRevision() <= o.last[id] { + continue + } + o.last[id] = st.GetRevision() + return st, nil + } +} + +// Observe opens a status stream for key, or for every claim of the caller +// when key is nil. Cancel ctx to close it. +func (c *Client) Observe(ctx context.Context, key *workerv1alpha1.ClaimKey) (*Observer, error) { + var stream grpc.ServerStreamingClient[workerv1alpha1.ObserveResponse] + err := c.retry(ctx, func(ctx context.Context) error { + s, err := c.api.Observe(ctx, &workerv1alpha1.ObserveRequest{Key: key}) + if err != nil { + return err + } + // Worker sends headers once it admits the stream. A stream that + // ends without them was refused; Recv returns the refusal so it is + // retried like any other call. + md, err := s.Header() + if err != nil { + return err + } + if md == nil { + if _, err := s.Recv(); err != nil { + return err + } + return errors.New("workerclient: Observe stream ended without headers") + } + stream = s + return nil + }) + if err != nil { + return nil, err + } + return &Observer{stream: stream, last: map[string]uint64{}}, nil +} + +// Reason returns the contract reason of a Worker error, or "". +func Reason(err error) string { + st, ok := status.FromError(err) + if !ok { + return "" + } + for _, d := range st.Details() { + if info, ok := d.(*errdetails.ErrorInfo); ok && info.GetDomain() == workerv1alpha1.ErrorDomain { + return info.GetReason() + } + } + return "" +} + +// Retryable reports whether the contract says to retry err with backoff. +func Retryable(err error) bool { + switch status.Code(err) { + case codes.Unavailable, codes.ResourceExhausted: + return true + case codes.Unauthenticated: + return Reason(err) == workerv1alpha1.ReasonIdentityExpired + } + return false +} + +func (c *Client) callContext(ctx context.Context, withSession bool) (context.Context, error) { + token, err := c.opts.Token() + if err != nil { + return nil, fmt.Errorf("workerclient: read token: %w", err) + } + pairs := []string{"authorization", "Bearer " + token} + if withSession { + if s := c.Session(); s != "" { + pairs = append(pairs, workerv1alpha1.SessionMetadataKey, s) + } + } + return metadata.AppendToOutgoingContext(ctx, pairs...), nil +} + +// openSession calls GetCapabilities, renewing the current session when +// renew is set. +func (c *Client) openSession(ctx context.Context, renew bool) error { + cctx, err := c.callContext(ctx, renew) + if err != nil { + return err + } + resp, err := c.api.GetCapabilities(cctx, &workerv1alpha1.GetCapabilitiesRequest{Versions: c.opts.Versions}) + if err != nil { + return err + } + c.mu.Lock() + c.session, c.caps = resp.GetSessionId(), resp + c.mu.Unlock() + return nil +} + +func (c *Client) keepAlive() { + defer c.done.Done() + interval := c.Capabilities().GetRenewInterval().AsDuration() + if interval <= 0 { + interval = 10 * time.Second + } + t := time.NewTicker(interval) + defer t.Stop() + for { + select { + case <-c.stop: + return + case <-t.C: + ctx, cancel := context.WithTimeout(context.Background(), interval) + _ = c.openSession(ctx, true) + cancel() + } + } +} + +// retry runs call, retrying as the contract says. +func (c *Client) retry(ctx context.Context, call func(context.Context) error) error { + var err error + for attempt := range c.opts.Attempts { + var cctx context.Context + if cctx, err = c.callContext(ctx, true); err != nil { + return err + } + err = call(cctx) + if err == nil { + return nil + } + if Reason(err) == workerv1alpha1.ReasonNoSession { + if serr := c.openSession(ctx, false); serr != nil { + err = errors.Join(err, serr) + } + continue + } + if !Retryable(err) { + return err + } + ceiling := min(c.opts.RetryMin< Date: Wed, 18 Nov 2026 18:03:13 -0500 Subject: [PATCH 3/6] feat(worker): Implement Worker reconciliation and its service Galactic Worker is the node-local service agents call to deliver tenant traffic, and the single writer of shared host state (docs/worker). This implements its reconciliation core and the authenticated service, with no delivery module built in: Local (#1435) and Fabric (#1436) plug into internal/worker/module. Until then every capability reports ModuleNotRunning. Calls are authenticated over a unix socket (mode 0660): TokenReview of the projected token for audience galactic-worker, cached until it expires; the bound pod must be on this node; the SO_PEERCRED process must run in that pod, which the cgroup path shows; and a static YAML allowlist names each ServiceAccount's capabilities. A mutating call validates the request, authorizes the claim against trusted sources (the attachment's VPC, node and admission) and records desired state durably before it returns. A refusal records nothing, takes no ticket and reaches no module. Every refusal carries a contract reason and is counted in galactic_worker_refusals_total. Reconciliation runs per VPC scope from a deduplicating queue, so duplicate watch events coalesce. It resolves each destination by locator (own locator Local, any other Fabric, never a fallback), takes the scope's ticket from the internal/edge/ownership fence and commits the scope's complete desired state through the modules, then reads the host back. Desired state is durable; programmed and observed state are rebuilt from the host, and Ready stays separate from Programmed. On restart Worker fences and adopts each scope an earlier incarnation owned, refuses changes there with Resyncing until the snapshot commits, and never fences a scope another writer owns. A failed commit leaves the scope resync-required. A release during that repair fences the scope, so the repair's late commit cannot land. Only the owner releases a claim, at its current generation. A caller with no live session for a lease (120 s) has its claims withdrawn, as does a claim whose attachment loses its authority. Tests cover concurrent clients and racing callers; duplicate, stale and out-of-order calls; crashes before programming, mid-apply and before a release is recorded, then restart; duplicate watch events; lease expiry; revocation; withdrawal during repair; and unavailable modules. A privileged test programs real links, routes, rules and a pinned map, then shows that a cross-VPC request leaves all of them unchanged. Co-Authored-By: Claude --- Taskfile.yaml | 1 + cmd/galactic-worker/main.go | 27 ++ cmd/galactic-worker/root.go | 265 ++++++++++++ cmd/galactic-worker/scrape_test.go | 42 ++ internal/instrumentation/lint_test.go | 1 + internal/logging/logging.go | 1 + internal/worker/api.go | 184 ++++++++ internal/worker/auth/allowlist.go | 88 ++++ internal/worker/auth/auth.go | 248 +++++++++++ internal/worker/auth/auth_test.go | 240 +++++++++++ internal/worker/auth/peercred.go | 146 +++++++ internal/worker/claims.go | 424 +++++++++++++++++++ internal/worker/harness_test.go | 423 +++++++++++++++++++ internal/worker/metrics.go | 75 ++++ internal/worker/module/fake/fake.go | 371 ++++++++++++++++ internal/worker/module/module.go | 216 ++++++++++ internal/worker/reconcile.go | 491 +++++++++++++++++++++ internal/worker/recovery_test.go | 352 +++++++++++++++ internal/worker/refusal/refusal.go | 56 +++ internal/worker/sessions.go | 126 ++++++ internal/worker/socket.go | 53 +++ internal/worker/sources/sources.go | 217 ++++++++++ internal/worker/sources/sources_test.go | 76 ++++ internal/worker/status.go | 112 +++++ internal/worker/store/store.go | 151 +++++++ internal/worker/store/store_test.go | 61 +++ internal/worker/worker.go | 313 ++++++++++++++ internal/worker/worker_test.go | 493 ++++++++++++++++++++++ internal/worker/zeromutation_root_test.go | 243 +++++++++++ 29 files changed, 5496 insertions(+) create mode 100644 cmd/galactic-worker/main.go create mode 100644 cmd/galactic-worker/root.go create mode 100644 cmd/galactic-worker/scrape_test.go create mode 100644 internal/worker/api.go create mode 100644 internal/worker/auth/allowlist.go create mode 100644 internal/worker/auth/auth.go create mode 100644 internal/worker/auth/auth_test.go create mode 100644 internal/worker/auth/peercred.go create mode 100644 internal/worker/claims.go create mode 100644 internal/worker/harness_test.go create mode 100644 internal/worker/metrics.go create mode 100644 internal/worker/module/fake/fake.go create mode 100644 internal/worker/module/module.go create mode 100644 internal/worker/reconcile.go create mode 100644 internal/worker/recovery_test.go create mode 100644 internal/worker/refusal/refusal.go create mode 100644 internal/worker/sessions.go create mode 100644 internal/worker/socket.go create mode 100644 internal/worker/sources/sources.go create mode 100644 internal/worker/sources/sources_test.go create mode 100644 internal/worker/status.go create mode 100644 internal/worker/store/store.go create mode 100644 internal/worker/store/store_test.go create mode 100644 internal/worker/worker.go create mode 100644 internal/worker/worker_test.go create mode 100644 internal/worker/zeromutation_root_test.go diff --git a/Taskfile.yaml b/Taskfile.yaml index c258469a..de160275 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -171,6 +171,7 @@ tasks: - go build -ldflags "{{.LDFLAGS}}" -o bin/galactic-gateway ./cmd/galactic-gateway - go build -ldflags "{{.LDFLAGS}}" -o bin/galactic-nat ./cmd/galactic-nat - go build -ldflags "{{.LDFLAGS}}" -o bin/galactic-vrf ./cmd/galactic-vrf + - go build -ldflags "{{.LDFLAGS}}" -o bin/galactic-worker ./cmd/galactic-worker - go build -ldflags "{{.LDFLAGS}}" -o bin/fabric-config-agent ./cmd/fabric-config-agent - go build -ldflags "{{.LDFLAGS}}" -o bin/fabric-api ./cmd/fabric-api - go build -ldflags "{{.LDFLAGS}}" -o bin/galactic-controller ./cmd/galactic-controller diff --git a/cmd/galactic-worker/main.go b/cmd/galactic-worker/main.go new file mode 100644 index 00000000..5e6d3458 --- /dev/null +++ b/cmd/galactic-worker/main.go @@ -0,0 +1,27 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Command galactic-worker is Galactic Worker: the node-local service agents +// call to deliver tenant traffic, and the single writer of shared host state +// (docs/worker/README.md). This binary wires internal/worker to its unix +// socket, the Kubernetes trusted sources, TokenReview, the allowlist, the +// ownership fence, metrics and a gRPC health server. +// +// No delivery module is built in yet: Local (#1435) and Fabric (#1436) +// register here. Until then every capability reports ModuleNotRunning and an +// accepted claim reports Programmed=False with that reason. +// +// Worker needs the host PID namespace and read access to the host's /proc +// to map a connecting process to its pod. +package main + +import ( + "os" +) + +func main() { + if err := newRootCommand().Execute(); err != nil { + os.Exit(1) + } +} diff --git a/cmd/galactic-worker/root.go b/cmd/galactic-worker/root.go new file mode 100644 index 00000000..4c74babb --- /dev/null +++ b/cmd/galactic-worker/root.go @@ -0,0 +1,265 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "os" + "strings" + "time" + + "github.com/spf13/cobra" + "google.golang.org/grpc" + grpchealth "google.golang.org/grpc/health" + "google.golang.org/grpc/health/grpc_health_v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + "go.datum.net/galactic/internal/config" + "go.datum.net/galactic/internal/edge/ownership" + "go.datum.net/galactic/internal/logging" + "go.datum.net/galactic/internal/metadata" + "go.datum.net/galactic/internal/worker" + "go.datum.net/galactic/internal/worker/auth" + "go.datum.net/galactic/internal/worker/module" + "go.datum.net/galactic/internal/worker/sources" + "go.datum.net/galactic/internal/worker/store" +) + +const ( + appName = "galactic-worker" + appDesc = `Galactic Worker: node-local delivery API for tenant services + + Find more information at: https://www.datum.net/docs` + + defaultMetricsPort = 9104 + defaultHealthPort = 9105 + cacheSyncTimeout = 2 * time.Minute + envNodeName = "NODE_NAME" + socketGroupNoneOption = -1 +) + +// options are the command's flags. +type options struct { + nodeName string + namespace string + socket string + socketGroup int + stateDir string + ownershipDir string + allowlist string + procRoot string + metricsPort int + healthPort int + lease time.Duration + renewInterval time.Duration + claimLimit int + resyncInterval time.Duration +} + +func (o *options) validate() error { + var errs []error + if o.nodeName == "" { + errs = append(errs, fmt.Errorf("--node-name (or %s) is required", envNodeName)) + } + if o.allowlist == "" { + errs = append(errs, errors.New("--allowlist is required")) + } + if o.claimLimit < 1 { + errs = append(errs, errors.New("--claim-limit must be at least 1")) + } + if o.renewInterval <= 0 || o.lease <= o.renewInterval { + errs = append(errs, errors.New("--lease must be longer than --renew-interval, which must be positive")) + } + return errors.Join(errs...) +} + +// modules returns the delivery modules this build runs. Local (#1435) and +// Fabric (#1436) add themselves here. +func modules() []module.Module { return nil } + +func run(o *options) error { + allow, err := auth.LoadAllowlist(o.allowlist) + if err != nil { + return err + } + st, err := store.Open(o.stateDir) + if err != nil { + return err + } + // The fence directory is shared with every other writer on the node, + // so it must already exist rather than be created privately here. + fence, err := ownership.Open(o.ownershipDir) + if err != nil { + return err + } + reg, err := module.NewRegistry(modules()...) + if err != nil { + return err + } + + scheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(sources.AddToScheme(scheme)) + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + HealthProbeBindAddress: "0", + Metrics: metricsserver.Options{BindAddress: fmt.Sprintf(":%d", o.metricsPort)}, + }) + if err != nil { + return fmt.Errorf("create manager: %w", err) + } + worker.InitMetrics() + + ctx, cancel := context.WithCancelCause(ctrl.SetupSignalHandler()) + defer cancel(nil) + + healthLis, err := (&net.ListenConfig{}).Listen(ctx, "tcp", fmt.Sprintf(":%d", o.healthPort)) + if err != nil { + return fmt.Errorf("listen on gRPC health port %d: %w", o.healthPort, err) + } + healthGRPC := grpc.NewServer() + health := grpchealth.NewServer() + grpc_health_v1.RegisterHealthServer(healthGRPC, health) + health.SetServingStatus("", grpc_health_v1.HealthCheckResponse_NOT_SERVING) + health.SetServingStatus(config.GRPCReadinessService, grpc_health_v1.HealthCheckResponse_NOT_SERVING) + go func() { + if err := healthGRPC.Serve(healthLis); err != nil { + cancel(fmt.Errorf("gRPC health server: %w", err)) + } + }() + defer healthGRPC.GracefulStop() + + if err := sources.RegisterIndexes(ctx, mgr.GetFieldIndexer()); err != nil { + return err + } + // TokenReview is a create, which the cache never serves. + direct, err := client.New(mgr.GetConfig(), client.Options{Scheme: scheme}) + if err != nil { + return fmt.Errorf("create API client: %w", err) + } + w, err := worker.New(worker.Config{ + Node: o.nodeName, Lease: o.lease, RenewInterval: o.renewInterval, + ClaimLimit: o.claimLimit, ResyncInterval: o.resyncInterval, + }, worker.Deps{ + Store: st, + Fence: fence, + Sources: sources.Kube{Reader: mgr.GetClient(), Namespace: o.namespace}, + Modules: reg, + Auth: &auth.Authenticator{ + Reviewer: auth.KubeTokenReviewer{Client: direct}, + Pods: auth.CgroupPodResolver{ProcRoot: o.procRoot}, + Allowlist: allow, + Node: o.nodeName, + }, + Log: slog.Default(), + }) + if err != nil { + return err + } + if err := sources.Watch(ctx, mgr.GetCache(), w.Changed); err != nil { + return err + } + + mgrErr := make(chan error, 1) + go func() { mgrErr <- mgr.Start(ctx) }() + syncCtx, syncCancel := context.WithTimeout(ctx, cacheSyncTimeout) + synced := mgr.GetCache().WaitForCacheSync(syncCtx) + syncCancel() + if !synced { + cancel(errors.New("trusted-source cache did not sync")) + return errors.Join(context.Cause(ctx), <-mgrErr) + } + + // Desired state loads and reconciliation starts only once trusted + // sources are readable, so a restart never revokes a claim against an + // empty cache. + if err := w.Start(ctx); err != nil { + return err + } + lis, err := worker.Listen(ctx, o.socket, o.socketGroup) + if err != nil { + return err + } + srv := w.NewServer() + go func() { + if err := srv.Serve(lis); err != nil { + cancel(fmt.Errorf("worker API server: %w", err)) + } + }() + health.SetServingStatus("", grpc_health_v1.HealthCheckResponse_SERVING) + health.SetServingStatus(config.GRPCReadinessService, grpc_health_v1.HealthCheckResponse_SERVING) + slog.Info("serving the Worker API", "socket", o.socket, "node", o.nodeName, "owner", w.Owner()) + + err = <-mgrErr + w.Shutdown() + health.Shutdown() + srv.GracefulStop() + if cause := context.Cause(ctx); cause != nil && !errors.Is(cause, context.Canceled) { + return errors.Join(cause, err) + } + return err +} + +func newRootCommand() *cobra.Command { + var logOpts logging.Options + o := &options{} + cmd := &cobra.Command{ + Use: appName, + Short: strings.Split(appDesc, "\n")[0], + Long: appDesc, + PersistentPreRunE: func(*cobra.Command, []string) error { + return logOpts.Setup() + }, + RunE: func(cmd *cobra.Command, _ []string) error { + if ok, _ := cmd.Flags().GetBool("build-info"); ok { + fmt.Println(metadata.BuildInfo(appName)) + return nil + } + if ok, _ := cmd.Flags().GetBool("version"); ok { + fmt.Printf("%s version %s\n", appName, metadata.Version) + return nil + } + if o.nodeName == "" { + o.nodeName = os.Getenv(envNodeName) + } + if err := o.validate(); err != nil { + return err + } + return run(o) + }, + } + f := cmd.Flags() + f.StringVar(&o.nodeName, "node-name", "", "This node's name (env "+envNodeName+")") + f.StringVar(&o.namespace, "namespace", config.DefaultNamespace, "Namespace of the BGPRouters that carry node locators") + f.StringVar(&o.socket, "socket", workerv1alpha1.DefaultSocketPath, "Unix socket serving the Worker API") + f.IntVar(&o.socketGroup, "socket-group", socketGroupNoneOption, + "Numeric group that owns the socket; clients join it. -1 keeps the process's group") + f.StringVar(&o.stateDir, "state-dir", store.DefaultDir, "Directory of durable desired state") + f.StringVar(&o.ownershipDir, "ownership-dir", ownership.DefaultDir, + "Ownership fence directory shared with every writer on the node; must exist") + f.StringVar(&o.allowlist, "allowlist", "", "YAML file mapping ServiceAccounts to the capabilities they may use") + f.StringVar(&o.procRoot, "proc-root", "/proc", "The host's /proc, for mapping a peer process to its pod") + f.IntVar(&o.metricsPort, "metrics-port", defaultMetricsPort, "Metrics listen port") + f.IntVar(&o.healthPort, "grpc-health-port", defaultHealthPort, "gRPC health listen port") + f.DurationVar(&o.lease, "lease", worker.DefaultLease, "Session lease") + f.DurationVar(&o.renewInterval, "renew-interval", worker.DefaultRenewInterval, "Session renewal interval clients use") + f.IntVar(&o.claimLimit, "claim-limit", worker.DefaultClaimLimit, "Claims one caller may hold") + f.DurationVar(&o.resyncInterval, "resync-interval", worker.DefaultResyncInterval, + "How often every VPC scope is reconciled again") + f.Bool("build-info", false, "Print build information and exit") + f.BoolP("version", "V", false, "Print version and exit") + logOpts.BindFlags(cmd.PersistentFlags()) + return cmd +} diff --git a/cmd/galactic-worker/scrape_test.go b/cmd/galactic-worker/scrape_test.go new file mode 100644 index 00000000..35cc96c8 --- /dev/null +++ b/cmd/galactic-worker/scrape_test.go @@ -0,0 +1,42 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "testing" + + "go.miloapis.com/instrumentation/instrumenttest" + "go.miloapis.com/instrumentation/metrics" + + "go.datum.net/galactic/internal/worker" +) + +// TestMetricsScrape scrapes what the manager serves at /metrics once run has +// created every series, and lints them. +func TestMetricsScrape(t *testing.T) { + worker.InitMetrics() + instrumenttest.AssertScrapes(t, metrics.Handler()) + instrumenttest.Lint(t) +} + +func TestOptionsValidate(t *testing.T) { + o := &options{nodeName: "node-a", allowlist: "a.yaml", claimLimit: 1, + lease: worker.DefaultLease, renewInterval: worker.DefaultRenewInterval} + if err := o.validate(); err != nil { + t.Fatalf("validate() = %v", err) + } + for name, mutate := range map[string]func(*options){ + "no node": func(o *options) { o.nodeName = "" }, + "no allowlist": func(o *options) { o.allowlist = "" }, + "no claims": func(o *options) { o.claimLimit = 0 }, + "lease too short": func(o *options) { o.lease = o.renewInterval }, + } { + bad := *o + mutate(&bad) + if bad.validate() == nil { + t.Errorf("%s: validate() = nil, want an error", name) + } + } +} diff --git a/internal/instrumentation/lint_test.go b/internal/instrumentation/lint_test.go index 4518b5b2..1a60f4b6 100644 --- a/internal/instrumentation/lint_test.go +++ b/internal/instrumentation/lint_test.go @@ -37,6 +37,7 @@ import ( _ "go.datum.net/galactic/internal/plumbing/nldump" _ "go.datum.net/galactic/internal/runtime/gobgp" _ "go.datum.net/galactic/internal/serviceroute" + _ "go.datum.net/galactic/internal/worker" ) const ( diff --git a/internal/logging/logging.go b/internal/logging/logging.go index e89a8a5a..2be3e603 100644 --- a/internal/logging/logging.go +++ b/internal/logging/logging.go @@ -60,6 +60,7 @@ var Binaries = []string{ "galactic-nat", "galactic-router", "galactic-vrf", + "galactic-worker", "masque-authority", "masque-provider", } diff --git a/internal/worker/api.go b/internal/worker/api.go new file mode 100644 index 00000000..4033ec14 --- /dev/null +++ b/internal/worker/api.go @@ -0,0 +1,184 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package worker + +import ( + "context" + "slices" + "sort" + "strings" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/durationpb" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + "go.datum.net/galactic/internal/worker/module" + "go.datum.net/galactic/internal/worker/refusal" +) + +// servedVersions are the API versions this Worker serves, oldest first. +var servedVersions = []string{workerv1alpha1.Version} + +// Register registers the Worker service on s. +func (w *Worker) Register(s *grpc.Server) { workerv1alpha1.RegisterWorkerServer(s, w) } + +// GetCapabilities implements workerv1alpha1.WorkerServer. +func (w *Worker) GetCapabilities(ctx context.Context, req *workerv1alpha1.GetCapabilitiesRequest) ( + *workerv1alpha1.GetCapabilitiesResponse, error) { + caller, err := w.d.Auth.Authenticate(ctx) + if err != nil { + return nil, w.refuse(err) + } + version := "" + for i := len(servedVersions) - 1; i >= 0 && version == ""; i-- { + if slices.Contains(req.GetVersions(), servedVersions[i]) { + version = servedVersions[i] + } + } + if version == "" { + return nil, w.refuse(refusal.New(codes.FailedPrecondition, workerv1alpha1.ReasonUnsupportedVersion, + "no common API version", map[string]string{ + workerv1alpha1.MetadataServedVersions: strings.Join(servedVersions, ","), + })) + } + id, err := w.openSession(ctx, caller) + if err != nil { + return nil, err + } + resp := &workerv1alpha1.GetCapabilitiesResponse{ + Version: version, + SessionId: id, + Lease: durationpb.New(w.cfg.Lease), + RenewInterval: durationpb.New(w.cfg.RenewInterval), + Node: w.cfg.Node, + } + for _, c := range w.d.Modules.Capabilities() { + state := workerv1alpha1.CapabilityState_CAPABILITY_STATE_UNAVAILABLE + if c.Available { + state = workerv1alpha1.CapabilityState_CAPABILITY_STATE_AVAILABLE + } + resp.Capabilities = append(resp.Capabilities, &workerv1alpha1.Capability{ + Name: c.Name, State: state, Reason: c.Reason, Message: c.Message, Allowed: caller.Allowed(c.Name), + }) + } + return resp, nil +} + +// Request implements workerv1alpha1.WorkerServer. +func (w *Worker) Request(ctx context.Context, req *workerv1alpha1.RequestRequest) ( + *workerv1alpha1.RequestResponse, error) { + caller, err := w.begin(ctx) + if err != nil { + return nil, err + } + st, err := w.put(ctx, caller, req.GetKey(), req.GetGeneration(), req.GetSpec(), false) + if err != nil { + return nil, err + } + return &workerv1alpha1.RequestResponse{Status: st}, nil +} + +// Update implements workerv1alpha1.WorkerServer. +func (w *Worker) Update(ctx context.Context, req *workerv1alpha1.UpdateRequest) ( + *workerv1alpha1.UpdateResponse, error) { + caller, err := w.begin(ctx) + if err != nil { + return nil, err + } + st, err := w.put(ctx, caller, req.GetKey(), req.GetGeneration(), req.GetSpec(), true) + if err != nil { + return nil, err + } + return &workerv1alpha1.UpdateResponse{Status: st}, nil +} + +// Release implements workerv1alpha1.WorkerServer. +func (w *Worker) Release(ctx context.Context, req *workerv1alpha1.ReleaseRequest) ( + *workerv1alpha1.ReleaseResponse, error) { + caller, err := w.begin(ctx) + if err != nil { + return nil, err + } + st, err := w.release(caller, req.GetKey(), req.GetGeneration()) + if err != nil { + return nil, err + } + return &workerv1alpha1.ReleaseResponse{Status: st}, nil +} + +// Observe implements workerv1alpha1.WorkerServer. +func (w *Worker) Observe(req *workerv1alpha1.ObserveRequest, + stream grpc.ServerStreamingServer[workerv1alpha1.ObserveResponse]) error { + ctx := stream.Context() + caller, err := w.begin(ctx) + if err != nil { + return err + } + sub := &subscriber{caller: caller.ServiceAccount, notify: make(chan struct{}, 1), + pending: map[module.ClaimKey]*workerv1alpha1.ClaimStatus{}} + if req.GetKey() != nil { + k, err := w.validateKey(req.GetKey()) + if err != nil { + return w.refuse(err) + } + sub.key = &k + } + w.mu.Lock() + if sub.key != nil { + if c := w.claims[*sub.key]; c != nil && c.rec.Owner != caller.ServiceAccount { + w.mu.Unlock() + return w.refuse(refusal.New(codes.PermissionDenied, workerv1alpha1.ReasonNotOwner, + "another caller owns this claim", nil)) + } + } + for k, c := range w.claims { + if sub.wants(c.rec.Owner, k) { + sub.pending[k] = proto.Clone(c.status).(*workerv1alpha1.ClaimStatus) + } + } + w.subs[sub] = struct{}{} + w.mu.Unlock() + defer func() { + w.mu.Lock() + delete(w.subs, sub) + w.mu.Unlock() + }() + if err := stream.SendHeader(metadata.MD{}); err != nil { + return err + } + sub.notify <- struct{}{} + + renew := time.NewTicker(w.cfg.RenewInterval) + defer renew.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-renew.C: + // An open stream renews the session lease. + if err := w.touchSession(ctx, caller); err != nil { + return err + } + case <-sub.notify: + w.mu.Lock() + batch := make([]*workerv1alpha1.ClaimStatus, 0, len(sub.pending)) + for _, st := range sub.pending { + batch = append(batch, st) + } + clear(sub.pending) + w.mu.Unlock() + sort.Slice(batch, func(i, j int) bool { return batch[i].GetRevision() < batch[j].GetRevision() }) + for _, st := range batch { + if err := stream.Send(&workerv1alpha1.ObserveResponse{Status: st}); err != nil { + return err + } + } + } + } +} diff --git a/internal/worker/auth/allowlist.go b/internal/worker/auth/allowlist.go new file mode 100644 index 00000000..a2574260 --- /dev/null +++ b/internal/worker/auth/allowlist.go @@ -0,0 +1,88 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package auth + +import ( + "fmt" + "os" + "slices" + "strings" + + "sigs.k8s.io/yaml" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" +) + +// Allowlist maps each ServiceAccount, as "namespace/name", to the +// capabilities it may use. A ServiceAccount it does not list may use none. +type Allowlist struct { + entries map[string][]string +} + +// allowlistFile is the YAML form of an allowlist: +// +// serviceAccounts: +// - serviceAccount: galactic-system/galactic-nat +// capabilities: [local.veth_pair, fabric.srv6] +type allowlistFile struct { + ServiceAccounts []struct { + ServiceAccount string `json:"serviceAccount"` + Capabilities []string `json:"capabilities"` + } `json:"serviceAccounts"` +} + +// LoadAllowlist reads an allowlist from a YAML file. +func LoadAllowlist(path string) (*Allowlist, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("auth: read allowlist: %w", err) + } + return ParseAllowlist(b) +} + +// ParseAllowlist parses the YAML form of an allowlist. It refuses an unknown +// capability, a malformed or repeated ServiceAccount, and unknown fields. +func ParseAllowlist(b []byte) (*Allowlist, error) { + var f allowlistFile + if err := yaml.UnmarshalStrict(b, &f); err != nil { + return nil, fmt.Errorf("auth: parse allowlist: %w", err) + } + a := &Allowlist{entries: map[string][]string{}} + for _, e := range f.ServiceAccounts { + ns, name, ok := strings.Cut(e.ServiceAccount, "/") + if !ok || ns == "" || name == "" || strings.Contains(name, "/") { + return nil, fmt.Errorf("auth: allowlist: %q is not namespace/name", e.ServiceAccount) + } + if _, dup := a.entries[e.ServiceAccount]; dup { + return nil, fmt.Errorf("auth: allowlist: %s is listed twice", e.ServiceAccount) + } + for _, c := range e.Capabilities { + if !slices.Contains(workerv1alpha1.Capabilities, c) { + return nil, fmt.Errorf("auth: allowlist: %s: unknown capability %q", e.ServiceAccount, c) + } + } + a.entries[e.ServiceAccount] = slices.Clone(e.Capabilities) + } + return a, nil +} + +// NewAllowlist builds an allowlist from a map, for tests and embedding. +func NewAllowlist(entries map[string][]string) *Allowlist { + a := &Allowlist{entries: map[string][]string{}} + for k, v := range entries { + a.entries[k] = slices.Clone(v) + } + return a +} + +// Capabilities returns the capabilities serviceAccount may use, and whether +// it is listed at all. +func (a *Allowlist) Capabilities(serviceAccount string) ([]string, bool) { + if a == nil { + return nil, false + } + c, ok := a.entries[serviceAccount] + return slices.Clone(c), ok +} diff --git a/internal/worker/auth/auth.go b/internal/worker/auth/auth.go new file mode 100644 index 00000000..7f3e378b --- /dev/null +++ b/internal/worker/auth/auth.go @@ -0,0 +1,248 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package auth authenticates a caller of Worker's API, as +// docs/worker/contract.md "Transport, identity and access" requires: +// +// 1. The call carries a projected ServiceAccount token with audience +// galactic-worker. TokenReview validates it; the result is cached until +// the token expires. +// 2. The token is bound to a pod on this node. +// 3. The connecting process (SO_PEERCRED) runs in that pod. +// 4. A static allowlist names the capabilities the ServiceAccount may use. +package auth + +import ( + "context" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "slices" + "strings" + "sync" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + authenticationv1 "k8s.io/api/authentication/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + "go.datum.net/galactic/internal/worker/refusal" +) + +// Extra keys the API server sets on a token bound to a pod. +const ( + ExtraPodUID = "authentication.kubernetes.io/pod-uid" + ExtraPodName = "authentication.kubernetes.io/pod-name" + ExtraNodeName = "authentication.kubernetes.io/node-name" +) + +const serviceAccountPrefix = "system:serviceaccount:" + +// Caller is an authenticated client. +type Caller struct { + // ServiceAccount is "namespace/name". It is the claim owner: a client + // that restarts keeps ownership of its claims. + ServiceAccount string + PodUID string + PodName string + // Capabilities are the capabilities the allowlist grants. + Capabilities []string +} + +// Allowed reports whether the caller may use capability. +func (c Caller) Allowed(capability string) bool { + return slices.Contains(c.Capabilities, capability) +} + +// TokenReviewer authenticates a bearer token for the Worker audience. +type TokenReviewer interface { + Review(ctx context.Context, token string) (authenticationv1.UserInfo, error) +} + +// KubeTokenReviewer reviews tokens with the API server's TokenReview. +type KubeTokenReviewer struct { + Client client.Client +} + +// Review implements TokenReviewer. A token not issued for the Worker +// audience is refused. +func (k KubeTokenReviewer) Review(ctx context.Context, token string) (authenticationv1.UserInfo, error) { + tr := &authenticationv1.TokenReview{Spec: authenticationv1.TokenReviewSpec{ + Token: token, Audiences: []string{workerv1alpha1.TokenAudience}, + }} + if err := k.Client.Create(ctx, tr); err != nil { + return authenticationv1.UserInfo{}, fmt.Errorf("review token: %w", err) + } + if !tr.Status.Authenticated || !slices.Contains(tr.Status.Audiences, workerv1alpha1.TokenAudience) { + return authenticationv1.UserInfo{}, fmt.Errorf("token not authenticated for %s", workerv1alpha1.TokenAudience) + } + return tr.Status.User, nil +} + +// Authenticator runs the authentication chain for one call. +type Authenticator struct { + Reviewer TokenReviewer + Pods PodResolver + Allowlist *Allowlist + // Node is this node's name; a token bound to a pod elsewhere is refused. + Node string + // Now is the clock, time.Now when nil. + Now func() time.Time + + mu sync.Mutex + cache map[[sha256.Size]byte]cached +} + +type cached struct { + user authenticationv1.UserInfo + expires time.Time +} + +const ( + maxCached = 1024 + // fallbackTTL bounds caching of a token whose expiry cannot be read. + fallbackTTL = time.Minute +) + +func (a *Authenticator) now() time.Time { + if a.Now != nil { + return a.Now() + } + return time.Now() +} + +// Authenticate returns the caller of the call in ctx, or a refusal. +func (a *Authenticator) Authenticate(ctx context.Context) (Caller, error) { + token := bearer(ctx) + if token == "" { + return Caller{}, refusal.New(codes.Unauthenticated, workerv1alpha1.ReasonNoIdentity, + "no bearer token; send a projected ServiceAccount token with audience "+workerv1alpha1.TokenAudience, nil) + } + user, err := a.review(ctx, token) + if err != nil { + return Caller{}, err + } + sa, ok := strings.CutPrefix(user.Username, serviceAccountPrefix) + if !ok { + return Caller{}, refusal.New(codes.Unauthenticated, workerv1alpha1.ReasonNoIdentity, + "token is not a ServiceAccount token", nil) + } + ns, name, ok := strings.Cut(sa, ":") + if !ok { + return Caller{}, refusal.New(codes.Unauthenticated, workerv1alpha1.ReasonNoIdentity, + "malformed ServiceAccount username", nil) + } + podUID := extra(user, ExtraPodUID) + if podUID == "" { + return Caller{}, refusal.New(codes.Unauthenticated, workerv1alpha1.ReasonNoIdentity, + "token is not bound to a pod", nil) + } + if node := extra(user, ExtraNodeName); node != a.Node { + return Caller{}, refusal.New(codes.Unauthenticated, workerv1alpha1.ReasonNoIdentity, + fmt.Sprintf("token is bound to a pod on node %q, not this node", node), nil) + } + peerInfo, ok := PeerFromContext(ctx) + if !ok { + return Caller{}, refusal.New(codes.Unauthenticated, workerv1alpha1.ReasonNoIdentity, + "connection carries no peer credentials", nil) + } + peerPod, err := a.Pods.PodUID(peerInfo.PID) + if err != nil || peerPod != podUID { + return Caller{}, refusal.New(codes.Unauthenticated, workerv1alpha1.ReasonNoIdentity, + "the connecting process does not run in the token's pod", nil) + } + c := Caller{ServiceAccount: ns + "/" + name, PodUID: podUID, PodName: extra(user, ExtraPodName)} + caps, listed := a.Allowlist.Capabilities(c.ServiceAccount) + if !listed { + return Caller{}, refusal.New(codes.PermissionDenied, workerv1alpha1.ReasonCapabilityNotAllowed, + "ServiceAccount "+c.ServiceAccount+" is not in the allowlist", nil) + } + c.Capabilities = caps + return c, nil +} + +// review returns the token's user from the cache or from TokenReview. +func (a *Authenticator) review(ctx context.Context, token string) (authenticationv1.UserInfo, error) { + key := sha256.Sum256([]byte(token)) + now := a.now() + exp, hasExp := tokenExpiry(token) + if hasExp && !now.Before(exp) { + return authenticationv1.UserInfo{}, refusal.New(codes.Unauthenticated, workerv1alpha1.ReasonIdentityExpired, + "token has expired; read the projected token again", nil) + } + a.mu.Lock() + if c, ok := a.cache[key]; ok && now.Before(c.expires) { + a.mu.Unlock() + return c.user, nil + } + a.mu.Unlock() + + user, err := a.Reviewer.Review(ctx, token) + if err != nil { + return authenticationv1.UserInfo{}, refusal.New(codes.Unauthenticated, workerv1alpha1.ReasonNoIdentity, + "token review refused the token", nil) + } + if !hasExp { + exp = now.Add(fallbackTTL) + } + a.mu.Lock() + defer a.mu.Unlock() + if a.cache == nil { + a.cache = map[[sha256.Size]byte]cached{} + } + if len(a.cache) >= maxCached { + for k, c := range a.cache { + if !now.Before(c.expires) { + delete(a.cache, k) + } + } + if len(a.cache) >= maxCached { + clear(a.cache) + } + } + a.cache[key] = cached{user: user, expires: exp} + return user, nil +} + +// bearer returns the bearer token of the call, or "". +func bearer(ctx context.Context) string { + md, _ := metadata.FromIncomingContext(ctx) + for _, v := range md.Get("authorization") { + if t, ok := strings.CutPrefix(v, "Bearer "); ok { + return strings.TrimSpace(t) + } + } + return "" +} + +func extra(u authenticationv1.UserInfo, key string) string { + if v := u.Extra[key]; len(v) == 1 { + return v[0] + } + return "" +} + +// tokenExpiry reads the exp claim of a JWT without verifying it. It only +// bounds how long a TokenReview result is cached and refuses a token that +// has visibly expired; TokenReview remains the only check of validity. +func tokenExpiry(token string) (time.Time, bool) { + parts := strings.Split(token, ".") + if len(parts) != 3 { + return time.Time{}, false + } + b, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + return time.Time{}, false + } + var claims struct { + Exp int64 `json:"exp"` + } + if json.Unmarshal(b, &claims) != nil || claims.Exp == 0 { + return time.Time{}, false + } + return time.Unix(claims.Exp, 0), true +} diff --git a/internal/worker/auth/auth_test.go b/internal/worker/auth/auth_test.go new file mode 100644 index 00000000..56c91e10 --- /dev/null +++ b/internal/worker/auth/auth_test.go @@ -0,0 +1,240 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package auth + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/health" + "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + authenticationv1 "k8s.io/api/authentication/v1" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + "go.datum.net/galactic/internal/worker/refusal" +) + +const podOne = "pod-1" + +func TestParseAllowlist(t *testing.T) { + a, err := ParseAllowlist([]byte(` +serviceAccounts: +- serviceAccount: galactic-system/galactic-nat + capabilities: [local.veth_pair, fabric.srv6] +- serviceAccount: galactic-system/galactic-gateway + capabilities: [local.tap] +`)) + if err != nil { + t.Fatal(err) + } + if c, ok := a.Capabilities("galactic-system/galactic-nat"); !ok || len(c) != 2 { + t.Fatalf("nat capabilities = %v, %t", c, ok) + } + if _, ok := a.Capabilities("galactic-system/unknown"); ok { + t.Fatal("an unlisted ServiceAccount is listed") + } + for name, doc := range map[string]string{ + "unknown capability": "serviceAccounts:\n- serviceAccount: a/b\n capabilities: [local.everything]\n", + "no namespace": "serviceAccounts:\n- serviceAccount: b\n capabilities: []\n", + "listed twice": "serviceAccounts:\n- serviceAccount: a/b\n- serviceAccount: a/b\n", + "unknown field": "serviceAccounts:\n- serviceAccount: a/b\n role: admin\n", + } { + if _, err := ParseAllowlist([]byte(doc)); err == nil { + t.Errorf("%s: ParseAllowlist() = nil error", name) + } + } +} + +func TestPodUIDFromCgroup(t *testing.T) { + const uid = "0f1e2d3c-4b5a-6978-8796-a5b4c3d2e1f0" + for name, tc := range map[string]struct { + cgroup string + want string + }{ + "systemd v2": {"0::/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod" + + "0f1e2d3c_4b5a_6978_8796_a5b4c3d2e1f0.slice/cri-containerd-abc.scope\n", uid}, + "cgroupfs v1": {"12:memory:/kubepods/besteffort/pod" + uid + "/0123456789abcdef\n" + + "11:cpu:/kubepods/besteffort/pod" + uid + "/0123456789abcdef\n", uid}, + "not in a pod": {"0::/system.slice/kubelet.service\n", ""}, + "two pods": {"1:a:/kubepods/pod" + uid + "/x\n2:b:/kubepods/pod" + + "11111111-2222-3333-4444-555555555555/y\n", ""}, + } { + got, err := PodUIDFromCgroup(tc.cgroup) + if tc.want == "" { + if err == nil { + t.Errorf("%s: got %q, want an error", name, got) + } + continue + } + if err != nil || got != tc.want { + t.Errorf("%s: got %q, %v; want %q", name, got, err, tc.want) + } + } + if _, err := (CgroupPodResolver{}).PodUID(0); err == nil { + t.Error("PID 0 resolved; a peer outside the host PID namespace must be refused") + } +} + +func TestCgroupPodResolverReadsProc(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "42"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "42", "cgroup"), + []byte("0::/kubepods/pod11111111-2222-3333-4444-555555555555/c\n"), 0o644); err != nil { + t.Fatal(err) + } + got, err := CgroupPodResolver{ProcRoot: root}.PodUID(42) + if err != nil || got != "11111111-2222-3333-4444-555555555555" { + t.Fatalf("PodUID = %q, %v", got, err) + } +} + +type reviewer struct { + calls atomic.Int32 + users map[string]authenticationv1.UserInfo +} + +func (r *reviewer) Review(_ context.Context, token string) (authenticationv1.UserInfo, error) { + r.calls.Add(1) + u, ok := r.users[token] + if !ok { + return authenticationv1.UserInfo{}, errors.New("not authenticated") + } + return u, nil +} + +type pods map[int32]string + +func (p pods) PodUID(pid int32) (string, error) { + if uid, ok := p[pid]; ok { + return uid, nil + } + return "", fmt.Errorf("pid %d not in a pod", pid) +} + +func jwt(exp time.Time) string { + enc := base64.RawURLEncoding.EncodeToString + return enc([]byte(`{"alg":"none"}`)) + "." + enc([]byte(fmt.Sprintf(`{"exp":%d}`, exp.Unix()))) + ".sig" +} + +func user(sa, pod, node string) authenticationv1.UserInfo { + return authenticationv1.UserInfo{Username: "system:serviceaccount:" + sa, + Extra: map[string]authenticationv1.ExtraValue{ + ExtraPodUID: {pod}, ExtraPodName: {"p"}, ExtraNodeName: {node}, + }} +} + +// TestAuthenticatorOverUnixSocket runs the chain on a real unix socket, so +// SO_PEERCRED reports this test process. +func TestAuthenticatorOverUnixSocket(t *testing.T) { + now := time.Unix(1_700_000_000, 0) + valid, expired := jwt(now.Add(time.Hour)), jwt(now.Add(-time.Minute)) + rev := &reviewer{users: map[string]authenticationv1.UserInfo{ + valid: user("galactic-system:galactic-nat", podOne, "node-a"), + "other-node": user("galactic-system:galactic-nat", podOne, "node-b"), + "other-pod": user("galactic-system:galactic-nat", "pod-2", "node-a"), + "unlisted": user("galactic-system:intruder", podOne, "node-a"), + "user": {Username: "alice", Extra: map[string]authenticationv1.ExtraValue{ExtraPodUID: {podOne}}}, + "unbound": {Username: "system:serviceaccount:galactic-system:galactic-nat"}, + }} + a := &Authenticator{ + Reviewer: rev, + Pods: pods{int32(os.Getpid()): podOne}, + Allowlist: NewAllowlist(map[string][]string{ + "galactic-system/galactic-nat": {workerv1alpha1.CapabilityLocalVethPair}, + }), + Node: "node-a", + Now: func() time.Time { return now }, + } + + results := make(chan any, 1) + sock := filepath.Join(t.TempDir(), "w.sock") + lis, err := (&net.ListenConfig{}).Listen(t.Context(), "unix", sock) + if err != nil { + t.Fatal(err) + } + srv := grpc.NewServer(grpc.Creds(PeerCredentials{}), grpc.UnaryInterceptor( + func(ctx context.Context, req any, info *grpc.UnaryServerInfo, h grpc.UnaryHandler) (any, error) { + if p, ok := PeerFromContext(ctx); !ok || p.PID != int32(os.Getpid()) { + return nil, status.Error(codes.Internal, "peer credentials missing or wrong") + } + c, err := a.Authenticate(ctx) + if err != nil { + return nil, err + } + results <- c + return h(ctx, req) + })) + grpc_health_v1.RegisterHealthServer(srv, health.NewServer()) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + cc, err := grpc.NewClient("passthrough:///w", grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", sock) + })) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cc.Close() }) + hc := grpc_health_v1.NewHealthClient(cc) + call := func(token string) error { + ctx := context.Background() + if token != "" { + ctx = metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer "+token) + } + _, err := hc.Check(ctx, &grpc_health_v1.HealthCheckRequest{}) + return err + } + + if err := call(valid); err != nil { + t.Fatalf("valid token: %v", err) + } + c := (<-results).(Caller) + if c.ServiceAccount != "galactic-system/galactic-nat" || c.PodUID != podOne || + !c.Allowed(workerv1alpha1.CapabilityLocalVethPair) || c.Allowed(workerv1alpha1.CapabilityFabricSRv6) { + t.Fatalf("caller = %+v", c) + } + if err := call(valid); err != nil { + t.Fatal(err) + } + <-results + if n := rev.calls.Load(); n != 1 { + t.Fatalf("TokenReview ran %d times, want 1: the result is cached until the token expires", n) + } + + for name, tc := range map[string]struct { + token string + code codes.Code + reason string + }{ + "no token": {"", codes.Unauthenticated, workerv1alpha1.ReasonNoIdentity}, + "expired": {expired, codes.Unauthenticated, workerv1alpha1.ReasonIdentityExpired}, + "refused": {"forged", codes.Unauthenticated, workerv1alpha1.ReasonNoIdentity}, + "other node": {"other-node", codes.Unauthenticated, workerv1alpha1.ReasonNoIdentity}, + "other pod": {"other-pod", codes.Unauthenticated, workerv1alpha1.ReasonNoIdentity}, + "not an SA": {"user", codes.Unauthenticated, workerv1alpha1.ReasonNoIdentity}, + "not bound": {"unbound", codes.Unauthenticated, workerv1alpha1.ReasonNoIdentity}, + "not listed": {"unlisted", codes.PermissionDenied, workerv1alpha1.ReasonCapabilityNotAllowed}, + } { + err := call(tc.token) + if !refusal.Is(err, tc.code, tc.reason) { + t.Errorf("%s: got %v (reason %q), want %s/%s", name, err, refusal.Reason(err), tc.code, tc.reason) + } + } +} diff --git a/internal/worker/auth/peercred.go b/internal/worker/auth/peercred.go new file mode 100644 index 00000000..b1dccf45 --- /dev/null +++ b/internal/worker/auth/peercred.go @@ -0,0 +1,146 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package auth + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "regexp" + "strings" + + "golang.org/x/sys/unix" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/peer" +) + +// PeerInfo is the AuthInfo of a connection on Worker's unix socket: the +// kernel's record of the connecting process (SO_PEERCRED). +type PeerInfo struct { + credentials.CommonAuthInfo + PID int32 + UID uint32 + GID uint32 +} + +// AuthType implements credentials.AuthInfo. +func (PeerInfo) AuthType() string { return "peercred" } + +// PeerCredentials are server transport credentials for a unix socket. They +// secure nothing on the wire, which never leaves the host, and record the +// peer's credentials for each connection. +type PeerCredentials struct{} + +var errNotUnix = errors.New("auth: peer credentials need a unix socket connection") + +// ServerHandshake reads SO_PEERCRED from the accepted connection. +func (PeerCredentials) ServerHandshake(conn net.Conn) (net.Conn, credentials.AuthInfo, error) { + uc, ok := conn.(*net.UnixConn) + if !ok { + return nil, nil, errNotUnix + } + raw, err := uc.SyscallConn() + if err != nil { + return nil, nil, fmt.Errorf("auth: peer credentials: %w", err) + } + var cred *unix.Ucred + var credErr error + if err := raw.Control(func(fd uintptr) { + cred, credErr = unix.GetsockoptUcred(int(fd), unix.SOL_SOCKET, unix.SO_PEERCRED) + }); err != nil { + return nil, nil, fmt.Errorf("auth: peer credentials: %w", err) + } + if credErr != nil { + return nil, nil, fmt.Errorf("auth: SO_PEERCRED: %w", credErr) + } + return conn, PeerInfo{ + CommonAuthInfo: credentials.CommonAuthInfo{SecurityLevel: credentials.PrivacyAndIntegrity}, + PID: cred.Pid, UID: cred.Uid, GID: cred.Gid, + }, nil +} + +// ClientHandshake is not supported: these credentials are for the server. +func (PeerCredentials) ClientHandshake(context.Context, string, net.Conn) (net.Conn, credentials.AuthInfo, error) { + return nil, nil, errors.New("auth: peer credentials are server-side only") +} + +// Info implements credentials.TransportCredentials. +func (PeerCredentials) Info() credentials.ProtocolInfo { + return credentials.ProtocolInfo{SecurityProtocol: "peercred"} +} + +// Clone implements credentials.TransportCredentials. +func (PeerCredentials) Clone() credentials.TransportCredentials { return PeerCredentials{} } + +// OverrideServerName implements credentials.TransportCredentials. +// +//nolint:staticcheck // Required by the interface. +func (PeerCredentials) OverrideServerName(string) error { return nil } + +// PeerFromContext returns the peer credentials of the call's connection. +func PeerFromContext(ctx context.Context) (PeerInfo, bool) { + p, ok := peer.FromContext(ctx) + if !ok { + return PeerInfo{}, false + } + info, ok := p.AuthInfo.(PeerInfo) + return info, ok +} + +// PodResolver maps a process to the UID of the pod it runs in. +type PodResolver interface { + PodUID(pid int32) (string, error) +} + +// CgroupPodResolver finds a process's pod from its cgroup path, which the +// kubelet names after the pod UID under both cgroup drivers. Worker must share +// the host PID namespace and read the host's /proc: in any other PID +// namespace SO_PEERCRED reports PID 0 for a peer in another pod. +type CgroupPodResolver struct { + // ProcRoot is the proc mount, "/proc" when empty. + ProcRoot string +} + +// podUID matches the pod segment of a kubelet cgroup path: "pod" with +// the cgroupfs driver and "pod.slice" with the systemd driver. +var podUID = regexp.MustCompile(`pod([0-9a-f]{8}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{12})`) + +// PodUID implements PodResolver. +func (r CgroupPodResolver) PodUID(pid int32) (string, error) { + if pid <= 0 { + return "", fmt.Errorf("auth: peer PID %d is not visible; Worker needs the host PID namespace", pid) + } + root := r.ProcRoot + if root == "" { + root = "/proc" + } + b, err := os.ReadFile(fmt.Sprintf("%s/%d/cgroup", root, pid)) + if err != nil { + return "", fmt.Errorf("auth: read cgroup of PID %d: %w", pid, err) + } + return PodUIDFromCgroup(string(b)) +} + +// PodUIDFromCgroup returns the pod UID named in a /proc//cgroup file. +func PodUIDFromCgroup(cgroup string) (string, error) { + uid := "" + for line := range strings.SplitSeq(cgroup, "\n") { + m := podUID.FindAllStringSubmatch(line, -1) + if len(m) == 0 { + continue + } + found := strings.ReplaceAll(m[len(m)-1][1], "_", "-") + if uid != "" && uid != found { + return "", fmt.Errorf("auth: cgroup names two pods, %s and %s", uid, found) + } + uid = found + } + if uid == "" { + return "", errors.New("auth: process is not in a pod cgroup") + } + return uid, nil +} diff --git a/internal/worker/claims.go b/internal/worker/claims.go new file mode 100644 index 00000000..d43da96c --- /dev/null +++ b/internal/worker/claims.go @@ -0,0 +1,424 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package worker + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "regexp" + "strconv" + + "google.golang.org/grpc/codes" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + "go.datum.net/galactic/internal/worker/auth" + "go.datum.net/galactic/internal/worker/module" + "go.datum.net/galactic/internal/worker/refusal" + "go.datum.net/galactic/internal/worker/store" +) + +type keyGen struct { + key module.ClaimKey + gen uint64 +} + +// vpcID matches a base62 VPC identifier. +var vpcID = regexp.MustCompile(`^[0-9A-Za-z]{1,16}$`) + +func errShuttingDown() error { + return refusal.New(codes.Unavailable, workerv1alpha1.ReasonShuttingDown, "Worker is shutting down", nil) +} + +func malformed(msg string) error { + return refusal.New(codes.InvalidArgument, workerv1alpha1.ReasonMalformedSpec, msg, nil) +} + +// validateKey checks a claim key's form and node. +func (w *Worker) validateKey(key *workerv1alpha1.ClaimKey) (module.ClaimKey, error) { + if key == nil || key.GetNode() == "" || key.GetAttachmentUid() == "" { + return module.ClaimKey{}, malformed("key needs node, vpc and attachment_uid") + } + if key.GetNode() != w.cfg.Node { + return module.ClaimKey{}, refusal.New(codes.InvalidArgument, workerv1alpha1.ReasonWrongNode, + fmt.Sprintf("this Worker serves node %q, not %q", w.cfg.Node, key.GetNode()), nil) + } + if !vpcID.MatchString(key.GetVpc()) { + return module.ClaimKey{}, malformed("vpc must be a base62 VPC identifier") + } + return module.ClaimKey{Node: key.GetNode(), VPC: key.GetVpc(), AttachmentUID: key.GetAttachmentUid()}, nil +} + +// validateSpec checks a spec's form. +func validateSpec(gen uint64, spec *workerv1alpha1.ClaimSpec) error { + if gen == 0 { + return malformed("generation must be at least 1") + } + if spec == nil { + return malformed("spec is required") + } + if workerv1alpha1.HandoffCapability(spec.GetHandoff()) == "" { + return malformed("handoff must be tap, veth_pair, routed_link or socket") + } + switch spec.GetMode() { + case workerv1alpha1.Mode_MODE_UNSPECIFIED, workerv1alpha1.Mode_MODE_AUTO, + workerv1alpha1.Mode_MODE_LOCAL, workerv1alpha1.Mode_MODE_FABRIC: + default: + return malformed("mode must be auto, local or fabric") + } + if len(spec.GetDestinations()) == 0 { + return malformed("at least one destination is required") + } + seen := map[string]bool{} + for i, d := range spec.GetDestinations() { + ok := false + switch t := d.GetTarget().(type) { + case *workerv1alpha1.Destination_Attachment: + ok = t.Attachment.GetUid() != "" + case *workerv1alpha1.Destination_Shard: + ok = t.Shard.GetGroup() != "" + case *workerv1alpha1.Destination_Backend: + ok = t.Backend.GetNode() != "" + } + if !ok { + return malformed(fmt.Sprintf("destination %d names no attachment, shard or backend", i)) + } + id := module.DestinationID(d) + if seen[id] { + return malformed("destination " + id + " is listed twice") + } + seen[id] = true + } + return nil +} + +// specHash is the hash of a spec's deterministic encoding. +func specHash(spec *workerv1alpha1.ClaimSpec) string { + b, _ := proto.MarshalOptions{Deterministic: true}.Marshal(spec) + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]) +} + +func genMD(gen uint64) map[string]string { + return map[string]string{workerv1alpha1.MetadataCurrentGeneration: strconv.FormatUint(gen, 10)} +} + +// decide applies the generation rules of Request (update false) or Update +// to the existing claim c. It returns noop for a duplicate. +func decide(c *claim, caller string, gen uint64, hash string, update bool) (noop bool, err error) { + if c != nil && c.rec.Owner != caller { + if update { + return false, refusal.New(codes.PermissionDenied, workerv1alpha1.ReasonNotOwner, + "another caller owns this claim", nil) + } + return false, refusal.New(codes.PermissionDenied, workerv1alpha1.ReasonAttachmentClaimedByAnotherCaller, + "another caller holds a claim on this attachment", nil) + } + if !update { + switch { + case c == nil: + return false, nil + case c.rec.Releasing && gen > c.rec.Generation: + return false, nil + case gen == c.rec.Generation && hash == c.rec.SpecHash && !c.rec.Releasing: + return true, nil + case gen == c.rec.Generation && !c.rec.Releasing: + return false, refusal.New(codes.FailedPrecondition, workerv1alpha1.ReasonGenerationConflict, + "the same generation was accepted with a different spec", genMD(c.rec.Generation)) + default: + return false, refusal.New(codes.FailedPrecondition, workerv1alpha1.ReasonClaimExists, + "the claim exists; use Update, or Request a newer generation once it is released", + genMD(c.rec.Generation)) + } + } + switch { + case c == nil || c.rec.Releasing: + return false, refusal.New(codes.FailedPrecondition, workerv1alpha1.ReasonClaimNotFound, + "no such claim; use Request", nil) + case gen < c.rec.Generation: + return false, refusal.New(codes.Aborted, workerv1alpha1.ReasonStaleGeneration, + "a newer generation is accepted", genMD(c.rec.Generation)) + case gen == c.rec.Generation && hash == c.rec.SpecHash: + return true, nil + case gen == c.rec.Generation: + return false, refusal.New(codes.FailedPrecondition, workerv1alpha1.ReasonGenerationConflict, + "the same generation was accepted with a different spec", genMD(c.rec.Generation)) + } + return false, nil +} + +// admit authorizes a claim against trusted sources and the caller's +// capabilities. It writes nothing. +func (w *Worker) admit(ctx context.Context, caller auth.Caller, k module.ClaimKey, + spec *workerv1alpha1.ClaimSpec) error { + notAuthorized := func(msg string) error { + return refusal.New(codes.PermissionDenied, workerv1alpha1.ReasonVPCNotAuthorized, msg, nil) + } + if hc := workerv1alpha1.HandoffCapability(spec.GetHandoff()); !caller.Allowed(hc) { + return refusal.New(codes.PermissionDenied, workerv1alpha1.ReasonCapabilityNotAllowed, + caller.ServiceAccount+" may not use "+hc, nil) + } + att, ok, err := w.d.Sources.Attachment(ctx, k.AttachmentUID) + if err != nil { + return refusal.New(codes.Unavailable, workerv1alpha1.ReasonResyncing, "trusted sources unavailable", nil) + } + switch { + case !ok: + return notAuthorized("the claim's attachment does not exist") + case att.VPC != k.VPC: + return notAuthorized("the claim's attachment is not in the claimed VPC") + case att.Node != w.cfg.Node: + return notAuthorized("the claim's attachment is not on this node") + case !att.Admitted: + return notAuthorized("the claim's attachment is not admitted") + } + own, _ := w.d.Sources.Locator(ctx, w.cfg.Node) + needFabric := spec.GetMode() == workerv1alpha1.Mode_MODE_FABRIC + for _, d := range spec.GetDestinations() { + t, ok := d.GetTarget().(*workerv1alpha1.Destination_Attachment) + if !ok { + continue + } + dst, ok, err := w.d.Sources.Attachment(ctx, t.Attachment.GetUid()) + if err != nil { + return refusal.New(codes.Unavailable, workerv1alpha1.ReasonResyncing, "trusted sources unavailable", nil) + } + // Two VPCs may use the same addresses, so only VPC identity is + // compared. + if !ok { + return notAuthorized("destination " + t.Attachment.GetUid() + " does not exist") + } + if dst.VPC != k.VPC { + return notAuthorized("destination " + t.Attachment.GetUid() + " is in another VPC") + } + if spec.GetMode() == workerv1alpha1.Mode_MODE_LOCAL { + continue + } + if loc, _ := w.d.Sources.Locator(ctx, dst.Node); own.IsValid() && loc.IsValid() && loc != own { + needFabric = true + } + } + if needFabric && !caller.Allowed(workerv1alpha1.CapabilityFabricSRv6) { + return refusal.New(codes.PermissionDenied, workerv1alpha1.ReasonCapabilityNotAllowed, + caller.ServiceAccount+" may not use "+workerv1alpha1.CapabilityFabricSRv6, nil) + } + return nil +} + +// put runs Request (update false) or Update. +func (w *Worker) put(ctx context.Context, caller auth.Caller, key *workerv1alpha1.ClaimKey, gen uint64, + spec *workerv1alpha1.ClaimSpec, update bool) (*workerv1alpha1.ClaimStatus, error) { + if w.shutting.Load() { + return nil, w.refuse(errShuttingDown()) + } + k, err := w.validateKey(key) + if err != nil { + return nil, w.refuse(err) + } + if err := validateSpec(gen, spec); err != nil { + return nil, w.refuse(err) + } + hash := specHash(spec) + + lock := w.keyLock(k) + lock.Lock() + defer lock.Unlock() + + w.mu.Lock() + c := w.claims[k] + noop, err := decide(c, caller.ServiceAccount, gen, hash, update) + if err == nil && noop { + st := proto.Clone(c.status).(*workerv1alpha1.ClaimStatus) + w.mu.Unlock() + return st, nil + } + if err == nil { + if s, ok := w.scopes[k.VPC]; ok && s.resyncing { + err = refusal.New(codes.Unavailable, workerv1alpha1.ReasonResyncing, + "the VPC scope is being reconciled after a restart", nil) + } + } + if err == nil && c == nil && w.countOwned(caller.ServiceAccount) >= w.cfg.ClaimLimit { + err = refusal.New(codes.ResourceExhausted, workerv1alpha1.ReasonClaimLimit, + fmt.Sprintf("a caller may hold at most %d claims", w.cfg.ClaimLimit), nil) + } + w.mu.Unlock() + if err != nil { + return nil, w.refuse(err) + } + + // Authorization reads trusted sources. It records nothing and takes no + // ticket, so a refusal leaves the host and the store untouched. + if err := w.admit(ctx, caller, k, spec); err != nil { + w.log.Info("refused claim", "caller", caller.ServiceAccount, "vpc", k.VPC, + "attachment", k.AttachmentUID, "generation", gen, "reason", refusal.Reason(err)) + return nil, w.refuse(err) + } + + raw, err := protojson.Marshal(spec) + if err != nil { + return nil, refusal.New(codes.Internal, workerv1alpha1.ReasonMalformedSpec, "encode spec", nil) + } + rec := store.Record{Node: k.Node, VPC: k.VPC, AttachmentUID: k.AttachmentUID, Owner: caller.ServiceAccount, + Generation: gen, SpecHash: hash, Spec: raw} + if err := w.d.Store.Put(rec); err != nil { + w.log.Error("record claim", "error", err) + return nil, refusal.New(codes.Unavailable, workerv1alpha1.ReasonResyncing, "cannot record the claim durably", nil) + } + + w.mu.Lock() + prev := w.claims[k] + nc := &claim{rec: rec, spec: proto.Clone(spec).(*workerv1alpha1.ClaimSpec)} + var observed uint64 + if prev != nil && !prev.rec.Releasing { + observed = prev.status.GetObservedGeneration() + } + nc.status = w.initialStatus(nc, observed) + if prev != nil && !prev.rec.Releasing { + // Keep reporting what the previous generation achieved until the + // new one is reconciled; observed_generation says which it is. + nc.status.Conditions = prev.status.GetConditions() + nc.status.Destinations = prev.status.GetDestinations() + } + w.claims[k] = nc + w.scope(k.VPC) + w.publish(nc) + st := proto.Clone(nc.status).(*workerv1alpha1.ClaimStatus) + w.mu.Unlock() + updateClaimGauge(w) + w.queue.Add(k.VPC) + return st, nil +} + +// countOwned counts the live claims of caller. w.mu must be held. +func (w *Worker) countOwned(caller string) int { + n := 0 + for _, c := range w.claims { + if c.rec.Owner == caller { + n++ + } + } + return n +} + +// release runs Release. +func (w *Worker) release(caller auth.Caller, key *workerv1alpha1.ClaimKey, gen uint64) ( + *workerv1alpha1.ClaimStatus, error) { + if w.shutting.Load() { + return nil, w.refuse(errShuttingDown()) + } + k, err := w.validateKey(key) + if err != nil { + return nil, w.refuse(err) + } + lock := w.keyLock(k) + lock.Lock() + defer lock.Unlock() + + w.mu.Lock() + c := w.claims[k] + switch { + case c == nil: + w.mu.Unlock() + return nil, nil + case c.rec.Owner != caller.ServiceAccount: + err = refusal.New(codes.PermissionDenied, workerv1alpha1.ReasonNotOwner, "another caller owns this claim", nil) + case gen < c.rec.Generation: + err = refusal.New(codes.Aborted, workerv1alpha1.ReasonStaleGeneration, + "a newer generation is accepted", genMD(c.rec.Generation)) + case gen > c.rec.Generation: + err = refusal.New(codes.FailedPrecondition, workerv1alpha1.ReasonGenerationConflict, + "generation is newer than the claim's", genMD(c.rec.Generation)) + case c.rec.Releasing: + st := proto.Clone(c.status).(*workerv1alpha1.ClaimStatus) + w.mu.Unlock() + return st, nil + case w.scope(k.VPC).resyncing: + err = refusal.New(codes.Unavailable, workerv1alpha1.ReasonResyncing, + "the VPC scope is being reconciled after a restart", nil) + } + w.mu.Unlock() + if err != nil { + return nil, w.refuse(err) + } + st, err := w.withdrawLocked(k, gen, store.WithdrawClaim, workerv1alpha1.ReasonReleased) + if err != nil { + return nil, err + } + w.fenceIfRepairing(k.VPC) + return st, nil +} + +// withdraw durably marks the claim at gen releasing and queues its scope. +// The record is deleted once read-back confirms no programmed state of the +// claim remains. +func (w *Worker) withdraw(k module.ClaimKey, gen uint64, scope, reason string) error { + lock := w.keyLock(k) + lock.Lock() + defer lock.Unlock() + _, err := w.withdrawLocked(k, gen, scope, reason) + return err +} + +// withdrawLocked is withdraw for a caller holding the key's lock. +func (w *Worker) withdrawLocked(k module.ClaimKey, gen uint64, scope, reason string) ( + *workerv1alpha1.ClaimStatus, error) { + w.mu.Lock() + c := w.claims[k] + if c == nil || c.rec.Generation != gen || c.rec.Releasing { + w.mu.Unlock() + return nil, nil + } + rec := c.rec + w.mu.Unlock() + rec.Releasing, rec.Withdrawal, rec.WithdrawReason = true, scope, reason + if err := w.d.Store.Put(rec); err != nil { + w.log.Error("record withdrawal", "error", err) + return nil, refusal.New(codes.Unavailable, workerv1alpha1.ReasonResyncing, "cannot record the release durably", nil) + } + w.mu.Lock() + if cur := w.claims[k]; cur == c { + c.rec = rec + w.setWithdrawing(c) + w.publish(c) + } + st := proto.Clone(c.status).(*workerv1alpha1.ClaimStatus) + w.mu.Unlock() + w.log.Info("withdrawing claim", "owner", rec.Owner, "vpc", k.VPC, "attachment", k.AttachmentUID, + "generation", gen, "scope", scope, "reason", reason) + w.queue.Add(k.VPC) + return st, nil +} + +// fenceIfRepairing fences a scope whose last commit failed, so a repair that +// holds a ticket while preparing its snapshot cannot commit state that still +// includes a released claim. The next reconcile adopts the scope and commits +// a snapshot from current desired state. +func (w *Worker) fenceIfRepairing(vpc string) { + w.mu.Lock() + s := w.scopes[vpc] + repairing := s != nil && s.repair + w.mu.Unlock() + if !repairing { + return + } + gen, err := w.d.Fence.Fence(module.Scope{VPC: vpc}.Name()) + if err != nil { + w.log.Error("fence scope for release", "vpc", vpc, "error", err) + return + } + w.mu.Lock() + w.scope(vpc).fenced = gen + w.mu.Unlock() +} + +// refuse counts a refusal and returns it. +func (w *Worker) refuse(err error) error { + countRefusal(refusal.Reason(err)) + return err +} diff --git a/internal/worker/harness_test.go b/internal/worker/harness_test.go new file mode 100644 index 00000000..27994f90 --- /dev/null +++ b/internal/worker/harness_test.go @@ -0,0 +1,423 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package worker + +import ( + "context" + "errors" + "net" + "net/netip" + "os" + "path/filepath" + "sort" + "sync" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" + "google.golang.org/protobuf/proto" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + "go.datum.net/galactic/internal/edge/ownership" + "go.datum.net/galactic/internal/worker/auth" + "go.datum.net/galactic/internal/worker/module" + "go.datum.net/galactic/internal/worker/module/fake" + "go.datum.net/galactic/internal/worker/refusal" + "go.datum.net/galactic/internal/worker/sources" + "go.datum.net/galactic/internal/worker/store" +) + +// Test identities and topology. Addresses are documentation prefixes. +const ( + node = "node-a" + otherNode = "node-b" + vpcA = "vpcA" + vpcB = "vpcB" + + natSA = "galactic-system/nat" + gwSA = "galactic-system/gateway" + otherSA = "galactic-system/other" + + callerMD = "x-test-caller" +) + +var ( + locA = netip.MustParsePrefix("2001:db8:0:1::/64") + locB = netip.MustParsePrefix("2001:db8:0:2::/64") + + testCaps = map[string][]string{ + natSA: {workerv1alpha1.CapabilityLocalVethPair, workerv1alpha1.CapabilityFabricSRv6}, + gwSA: {workerv1alpha1.CapabilityLocalTap}, + otherSA: {workerv1alpha1.CapabilityLocalVethPair, workerv1alpha1.CapabilityFabricSRv6}, + } +) + +// metaAuth authenticates a test caller named in metadata. +type metaAuth struct{} + +func (metaAuth) Authenticate(ctx context.Context) (auth.Caller, error) { + md, _ := metadata.FromIncomingContext(ctx) + v := md.Get(callerMD) + if len(v) != 1 { + return auth.Caller{}, refusal.New(codes.Unauthenticated, workerv1alpha1.ReasonNoIdentity, "no caller", nil) + } + caps, ok := testCaps[v[0]] + if !ok { + return auth.Caller{}, refusal.New(codes.PermissionDenied, workerv1alpha1.ReasonCapabilityNotAllowed, + "not listed", nil) + } + return auth.Caller{ServiceAccount: v[0], Capabilities: caps}, nil +} + +// faultyStore fails chosen operations once, standing in for a crash at +// that point. +type faultyStore struct { + store.Store + failDelete atomic.Int32 + failPut atomic.Int32 +} + +var errInjectedStore = errors.New("injected store failure") + +func (f *faultyStore) Put(r store.Record) error { + if f.failPut.Load() > 0 { + f.failPut.Add(-1) + return errInjectedStore + } + return f.Store.Put(r) +} + +func (f *faultyStore) Delete(n, v, a string) error { + if f.failDelete.Load() > 0 { + f.failDelete.Add(-1) + return errInjectedStore + } + return f.Store.Delete(n, v, a) +} + +// env is one Worker over persistent test state that survives restarts. +type env struct { + t *testing.T + stateDir string + fenceDir string + host *fake.Host + src *sources.Static + store *faultyStore + fence *ownership.Authority + cfg Config + clock *clock + modules func(*fake.Host) []module.Module + // pauseOnStart pauses the Local module's Plan before Start, so the + // first reconcile of a restart holds until the test resumes it. + pauseOnStart bool + startPaused <-chan struct{} + + // Per incarnation. + w *Worker + local *fake.Module + fabric *fake.Module + cancel context.CancelFunc + srv *grpc.Server + cc *grpc.ClientConn + api workerv1alpha1.WorkerClient + + mu sync.Mutex + sessions map[string]string +} + +type clock struct { + mu sync.Mutex + now time.Time +} + +func (c *clock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.now +} + +func (c *clock) Advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.now = c.now.Add(d) +} + +func newEnv(t *testing.T) *env { + t.Helper() + dir := t.TempDir() + st, err := store.Open(filepath.Join(dir, "claims")) + if err != nil { + t.Fatal(err) + } + fence, err := ownership.New(filepath.Join(dir, "ownership")) + if err != nil { + t.Fatal(err) + } + src := sources.NewStatic() + src.SetLocator(node, locA) + src.SetLocator(otherNode, locB) + // The service's handoff attachment and guests in two VPCs that use the + // same guest addresses; only VPC identity tells them apart. + for _, a := range []sources.Attachment{ + {UID: "svc-a", VPC: vpcA, Node: node}, + {UID: "svc-a2", VPC: vpcA, Node: node}, + {UID: "svc-b", VPC: vpcB, Node: node}, + {UID: "guest-a1", VPC: vpcA, Node: node}, + {UID: "guest-a2", VPC: vpcA, Node: otherNode}, + {UID: "guest-a3", VPC: vpcA, Node: node}, + {UID: "guest-b1", VPC: vpcB, Node: node}, + } { + a.Kind, a.Admitted = sources.KindVPCAttachment, true + src.SetAttachment(a) + } + e := &env{ + t: t, stateDir: filepath.Join(dir, "claims"), fenceDir: filepath.Join(dir, "ownership"), + host: fake.NewHost(), src: src, store: &faultyStore{Store: st}, fence: fence, + clock: &clock{now: time.Unix(1_700_000_000, 0)}, + cfg: Config{Node: node, RetryMin: 5 * time.Millisecond, RetryMax: 50 * time.Millisecond, + ResyncInterval: time.Hour, RenewInterval: time.Hour, Lease: 2 * time.Hour}, + } + t.Cleanup(e.crash) + return e +} + +// start starts a new Worker incarnation with fresh module objects over the +// persistent host. +func (e *env) start() { + e.t.Helper() + e.local = fake.New("local", workerv1alpha1.Mode_MODE_LOCAL, e.host) + e.fabric = fake.New("fabric", workerv1alpha1.Mode_MODE_FABRIC, e.host) + if e.pauseOnStart { + e.startPaused = e.local.PausePlan() + } + mods := []module.Module{e.local, e.fabric} + if e.modules != nil { + mods = e.modules(e.host) + } + reg, err := module.NewRegistry(mods...) + if err != nil { + e.t.Fatal(err) + } + w, err := New(e.cfg, Deps{Store: e.store, Fence: e.fence, Sources: e.src, Modules: reg, Auth: metaAuth{}, + Now: e.clock.Now}) + if err != nil { + e.t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + if err := w.Start(ctx); err != nil { + cancel() + e.t.Fatal(err) + } + lis := bufconn.Listen(1 << 20) + srv := grpc.NewServer() + w.Register(srv) + go func() { _ = srv.Serve(lis) }() + cc, err := grpc.NewClient("passthrough:///worker", + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { return lis.DialContext(ctx) }), + grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + cancel() + e.t.Fatal(err) + } + e.w, e.cancel, e.srv, e.cc, e.api = w, cancel, srv, cc, workerv1alpha1.NewWorkerClient(cc) + e.mu.Lock() + e.sessions = map[string]string{} + e.mu.Unlock() +} + +// crash stops the incarnation without any cleanup, as a killed process +// would. Host, store and fence state stay. +func (e *env) crash() { + if e.cancel == nil { + return + } + e.cancel() + e.srv.Stop() + _ = e.cc.Close() + e.cancel = nil + if e.local != nil { + e.local.Resume() + e.local.ResumePlan() + } +} + +// ctx returns a call context for caller with a live session. +func (e *env) ctx(caller string) context.Context { + e.t.Helper() + e.mu.Lock() + id := e.sessions[caller] + e.mu.Unlock() + if id == "" { + resp, err := e.api.GetCapabilities(metadata.AppendToOutgoingContext(context.Background(), callerMD, caller), + &workerv1alpha1.GetCapabilitiesRequest{Versions: []string{workerv1alpha1.Version}}) + if err != nil { + e.t.Fatalf("GetCapabilities(%s): %v", caller, err) + } + id = resp.GetSessionId() + e.mu.Lock() + e.sessions[caller] = id + e.mu.Unlock() + } + return metadata.AppendToOutgoingContext(context.Background(), callerMD, caller, + workerv1alpha1.SessionMetadataKey, id) +} + +func key(vpc, att string) *workerv1alpha1.ClaimKey { + return &workerv1alpha1.ClaimKey{Node: node, Vpc: vpc, AttachmentUid: att} +} + +func mkey(vpc, att string) module.ClaimKey { + return module.ClaimKey{Node: node, VPC: vpc, AttachmentUID: att} +} + +func spec(h workerv1alpha1.Handoff, uids ...string) *workerv1alpha1.ClaimSpec { + s := &workerv1alpha1.ClaimSpec{Handoff: h} + for _, u := range uids { + s.Destinations = append(s.Destinations, &workerv1alpha1.Destination{ + Target: &workerv1alpha1.Destination_Attachment{Attachment: &workerv1alpha1.AttachmentRef{Uid: u}}}) + } + return s +} + +func veth(uids ...string) *workerv1alpha1.ClaimSpec { + return spec(workerv1alpha1.Handoff_HANDOFF_VETH_PAIR, uids...) +} + +func (e *env) request(caller string, k *workerv1alpha1.ClaimKey, gen uint64, s *workerv1alpha1.ClaimSpec) error { + _, err := e.api.Request(e.ctx(caller), &workerv1alpha1.RequestRequest{Key: k, Generation: gen, Spec: s}) + return err +} + +func (e *env) update(caller string, k *workerv1alpha1.ClaimKey, gen uint64, s *workerv1alpha1.ClaimSpec) error { + _, err := e.api.Update(e.ctx(caller), &workerv1alpha1.UpdateRequest{Key: k, Generation: gen, Spec: s}) + return err +} + +func (e *env) release(caller string, k *workerv1alpha1.ClaimKey, gen uint64) error { + _, err := e.api.Release(e.ctx(caller), &workerv1alpha1.ReleaseRequest{Key: k, Generation: gen}) + return err +} + +func (e *env) mustOK(err error) { + e.t.Helper() + if err != nil { + e.t.Fatalf("unexpected error: %v", err) + } +} + +func wantRefusal(t *testing.T, err error, code codes.Code, reason string) { + t.Helper() + if status.Code(err) != code || refusal.Reason(err) != reason { + t.Fatalf("got %v (reason %q), want %s/%s", err, refusal.Reason(err), code, reason) + } +} + +// status returns a copy of the claim's status, or nil. +func (e *env) status(k module.ClaimKey) *workerv1alpha1.ClaimStatus { + e.w.mu.Lock() + defer e.w.mu.Unlock() + c := e.w.claims[k] + if c == nil { + return nil + } + return proto.Clone(c.status).(*workerv1alpha1.ClaimStatus) +} + +func condOf(st *workerv1alpha1.ClaimStatus, t string) *workerv1alpha1.Condition { + for _, c := range st.GetConditions() { + if c.GetType() == t { + return c + } + } + return nil +} + +func isTrue(st *workerv1alpha1.ClaimStatus, t string) bool { + return condOf(st, t).GetStatus() == condTrue +} + +// eventually polls cond until it holds or the deadline passes. +func (e *env) eventually(what string, cond func() bool) { + e.t.Helper() + deadline := time.Now().Add(10 * time.Second) + for !cond() { + if time.Now().After(deadline) { + e.t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(2 * time.Millisecond) + } +} + +// waitReady waits for a claim to be Ready at gen. +func (e *env) waitReady(k module.ClaimKey, gen uint64) *workerv1alpha1.ClaimStatus { + e.t.Helper() + var st *workerv1alpha1.ClaimStatus + e.eventually("claim "+k.String()+" ready", func() bool { + st = e.status(k) + return st.GetObservedGeneration() == gen && isTrue(st, workerv1alpha1.ConditionReady) + }) + return st +} + +// waitObserved waits for a claim to report on its first generation. +func (e *env) waitObserved(k module.ClaimKey) *workerv1alpha1.ClaimStatus { + const gen = 1 + e.t.Helper() + var st *workerv1alpha1.ClaimStatus + e.eventually("claim "+k.String()+" observed", func() bool { + st = e.status(k) + return st.GetObservedGeneration() == gen && condOf(st, workerv1alpha1.ConditionProgrammed).GetStatus() != condUnknown + }) + return st +} + +// waitGone waits for a released claim to be deleted. +func (e *env) waitGone(k module.ClaimKey) { + e.t.Helper() + e.eventually("claim "+k.String()+" released", func() bool { return e.status(k) == nil }) +} + +// idle waits until the queue holds no work. +func (e *env) idle() { + e.t.Helper() + e.eventually("queue idle", func() bool { return e.w.queue.Len() == 0 }) + time.Sleep(20 * time.Millisecond) + e.eventually("queue idle", func() bool { return e.w.queue.Len() == 0 }) +} + +// persisted is a byte snapshot of the store and fence directories. +func (e *env) persisted() map[string]string { + e.t.Helper() + out := map[string]string{} + for _, dir := range []string{e.stateDir, e.fenceDir} { + entries, err := os.ReadDir(dir) + if err != nil { + e.t.Fatal(err) + } + for _, ent := range entries { + b, err := os.ReadFile(filepath.Join(dir, ent.Name())) + if err != nil { + e.t.Fatal(err) + } + out[filepath.Join(filepath.Base(dir), ent.Name())] = string(b) + } + } + return out +} + +func rowIDs(rows []fake.Row) []string { + out := make([]string, 0, len(rows)) + for _, r := range rows { + out = append(out, r.Key.AttachmentUID+"→"+r.ID) + } + sort.Strings(out) + return out +} diff --git a/internal/worker/metrics.go b/internal/worker/metrics.go new file mode 100644 index 00000000..69554055 --- /dev/null +++ b/internal/worker/metrics.go @@ -0,0 +1,75 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package worker + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" + "go.miloapis.com/instrumentation/labels" + "go.miloapis.com/instrumentation/metrics" +) + +var ( + refusalsTotal = metrics.NewCounterVec(prometheus.CounterOpts{ + Name: "galactic_worker_refusals_total", + Help: "Calls Worker refused, by contract reason. A refusal records nothing and changes no host state.", + }, []string{labels.Reason}) + reconcilesTotal = metrics.NewCounterVec(prometheus.CounterOpts{ + Name: "galactic_worker_reconciles_total", + Help: "Reconcile passes over a VPC scope, by outcome.", + }, []string{labels.Outcome}) + reconcileSeconds = metrics.NewHistogram(prometheus.HistogramOpts{ + Name: "galactic_worker_reconcile_duration_seconds", + Help: "Time one reconcile pass over a VPC scope took.", + Buckets: prometheus.ExponentialBuckets(0.001, 4, 8), + }) + claimsGauge = metrics.NewGauge(prometheus.GaugeOpts{ + Name: "galactic_worker_claims", + Help: "Claims Worker holds, including claims being withdrawn.", + }) + sessionsGauge = metrics.NewGauge(prometheus.GaugeOpts{ + Name: "galactic_worker_sessions", + Help: "Live client sessions.", + }) +) + +// Outcome label values. +const ( + outcomeSuccess = "success" + outcomeError = "error" +) + +// InitMetrics creates every labelled series at zero, so a scrape shows +// them before the first event. +func InitMetrics() { + reconcilesTotal.WithLabelValues(outcomeSuccess) + reconcilesTotal.WithLabelValues(outcomeError) +} + +func countRefusal(reason string) { + if reason == "" { + reason = "Unknown" + } + refusalsTotal.WithLabelValues(reason).Inc() +} + +func observeReconcile(err error, d time.Duration) { + outcome := outcomeSuccess + if err != nil { + outcome = outcomeError + } + reconcilesTotal.WithLabelValues(outcome).Inc() + reconcileSeconds.Observe(d.Seconds()) +} + +func updateClaimGauge(w *Worker) { + w.mu.Lock() + n := len(w.claims) + w.mu.Unlock() + claimsGauge.Set(float64(n)) +} + +func setSessions(n int) { sessionsGauge.Set(float64(n)) } diff --git a/internal/worker/module/fake/fake.go b/internal/worker/module/fake/fake.go new file mode 100644 index 00000000..5c6088d7 --- /dev/null +++ b/internal/worker/module/fake/fake.go @@ -0,0 +1,371 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package fake is a delivery module for tests. It programs rows in a Host +// that outlives the module, so a test can restart Worker over the same host +// state, and it can inject failures, partial writes and pauses at Apply. +package fake + +import ( + "context" + "errors" + "maps" + "sort" + "strings" + "sync" + "sync/atomic" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + "go.datum.net/galactic/internal/worker/module" +) + +// Row is one programmed destination. +type Row struct { + Key module.ClaimKey + ID string + Generation uint64 +} + +// Kernel optionally mirrors every row write into real host state, so a +// privileged test can snapshot it. +type Kernel interface { + Add(scope module.Scope, r Row) error + Remove(scope module.Scope, r Row) error +} + +// Host is the state a fake module programs, shared across restarts. +type Host struct { + mu sync.Mutex + rows map[string]map[module.ClaimKey]map[string]uint64 // module/VPC → claim → dest → generation +} + +// NewHost returns an empty host. +func NewHost() *Host { return &Host{rows: map[string]map[module.ClaimKey]map[string]uint64{}} } + +// Rows returns every row of a scope, from every module, sorted. +func (h *Host) Rows(vpc string) []Row { + h.mu.Lock() + defer h.mu.Unlock() + var out []Row + for part, claims := range h.rows { + if !strings.HasSuffix(part, "/"+vpc) { + continue + } + for k, dests := range claims { + for id, gen := range dests { + out = append(out, Row{Key: k, ID: id, Generation: gen}) + } + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].Key != out[j].Key { + return out[i].Key.String() < out[j].Key.String() + } + return out[i].ID < out[j].ID + }) + return out +} + +// Module is a fake delivery module. +type Module struct { + ModuleName string + ModuleMode workerv1alpha1.Mode + Host *Host + Kernel Kernel + + mu sync.Mutex + caps []module.Capability + // failApply makes the next Apply calls fail after writing at most + // failAfter rows. + failApply int + failAfter int + // gate, when set, is received from before Apply writes. + gate chan struct{} + entered chan struct{} + planGate chan struct{} + planEntered chan struct{} + ready func(module.ClaimKey, string) (bool, string) + + PlanCalls atomic.Int64 + ApplyCalls atomic.Int64 + ReadCalls atomic.Int64 + Writes atomic.Int64 +} + +// New returns a fake module for mode, serving every capability of that mode. +func New(name string, mode workerv1alpha1.Mode, host *Host) *Module { + m := &Module{ModuleName: name, ModuleMode: mode, Host: host} + if mode == workerv1alpha1.Mode_MODE_FABRIC { + m.caps = []module.Capability{{Name: workerv1alpha1.CapabilityFabricSRv6, Available: true}} + } else { + for _, c := range workerv1alpha1.Capabilities { + if c != workerv1alpha1.CapabilityFabricSRv6 { + m.caps = append(m.caps, module.Capability{Name: c, Available: true}) + } + } + } + return m +} + +// SetCapability sets one capability's state. +func (m *Module) SetCapability(c module.Capability) { + m.mu.Lock() + defer m.mu.Unlock() + for i := range m.caps { + if m.caps[i].Name == c.Name { + m.caps[i] = c + return + } + } + m.caps = append(m.caps, c) +} + +// FailApply makes the next n Apply calls fail after writing at most after +// rows, leaving a partial commit behind. +func (m *Module) FailApply(n, after int) { + m.mu.Lock() + defer m.mu.Unlock() + m.failApply, m.failAfter = n, after +} + +// Pause makes Apply wait at entry until Resume. Entered receives once Apply +// is waiting. +func (m *Module) Pause() (entered <-chan struct{}) { + m.mu.Lock() + defer m.mu.Unlock() + m.gate = make(chan struct{}) + m.entered = make(chan struct{}, 1) + return m.entered +} + +// Resume releases a paused Apply. +func (m *Module) Resume() { + m.mu.Lock() + defer m.mu.Unlock() + if m.gate != nil { + close(m.gate) + m.gate = nil + } +} + +// PausePlan makes Plan wait at entry until ResumePlan, holding whatever +// ticket the caller took. Entered receives once Plan is waiting. +func (m *Module) PausePlan() (entered <-chan struct{}) { + m.mu.Lock() + defer m.mu.Unlock() + m.planGate = make(chan struct{}) + m.planEntered = make(chan struct{}, 1) + return m.planEntered +} + +// ResumePlan releases a paused Plan. +func (m *Module) ResumePlan() { + m.mu.Lock() + defer m.mu.Unlock() + if m.planGate != nil { + close(m.planGate) + m.planGate = nil + } +} + +// RepausePlan releases the Plan waiting now and makes the next Plan wait. +func (m *Module) RepausePlan() (entered <-chan struct{}) { + m.mu.Lock() + defer m.mu.Unlock() + if m.planGate != nil { + close(m.planGate) + } + m.planGate = make(chan struct{}) + m.planEntered = make(chan struct{}, 1) + return m.planEntered +} + +// SetReady sets the observed readiness of programmed destinations. +func (m *Module) SetReady(fn func(module.ClaimKey, string) (bool, string)) { + m.mu.Lock() + defer m.mu.Unlock() + m.ready = fn +} + +// Name implements module.Module. +func (m *Module) Name() string { return m.ModuleName } + +// Mode implements module.Module. +func (m *Module) Mode() workerv1alpha1.Mode { return m.ModuleMode } + +// Capabilities implements module.Module. +func (m *Module) Capabilities() []module.Capability { + m.mu.Lock() + defer m.mu.Unlock() + return append([]module.Capability(nil), m.caps...) +} + +// change is the complete desired row set of a scope. +type change struct { + want map[module.ClaimKey]map[string]uint64 + empty bool +} + +func (c change) Empty() bool { return c.empty } + +// Plan implements module.Module. +func (m *Module) Plan(ctx context.Context, scope module.Scope, desired []module.Claim) (module.Change, error) { + m.PlanCalls.Add(1) + m.mu.Lock() + gate, entered := m.planGate, m.planEntered + m.mu.Unlock() + if gate != nil { + select { + case entered <- struct{}{}: + default: + } + select { + case <-gate: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + want := map[module.ClaimKey]map[string]uint64{} + for _, c := range desired { + for _, d := range c.Destinations { + if want[c.Key] == nil { + want[c.Key] = map[string]uint64{} + } + want[c.Key][d.ID] = c.Generation + } + } + m.Host.mu.Lock() + have := m.Host.rows[m.ModuleName+"/"+scope.VPC] + equal := maps.EqualFunc(want, have, func(a, b map[string]uint64) bool { return maps.Equal(a, b) }) + m.Host.mu.Unlock() + return change{want: want, empty: equal}, nil +} + +var errInjected = errors.New("fake: injected apply failure") + +// Apply implements module.Module. It writes the difference row by row, so an +// injected failure leaves a partial commit. +func (m *Module) Apply(ctx context.Context, scope module.Scope, c module.Change) error { + m.ApplyCalls.Add(1) + m.mu.Lock() + gate, entered := m.gate, m.entered + fail, after := m.failApply > 0, m.failAfter + if fail { + m.failApply-- + } + m.mu.Unlock() + if gate != nil { + select { + case entered <- struct{}{}: + default: + } + select { + case <-gate: + case <-ctx.Done(): + return ctx.Err() + } + } + want := c.(change).want + m.Host.mu.Lock() + defer m.Host.mu.Unlock() + have := m.Host.rows[m.ModuleName+"/"+scope.VPC] + if have == nil { + have = map[module.ClaimKey]map[string]uint64{} + m.Host.rows[m.ModuleName+"/"+scope.VPC] = have + } + written := 0 + step := func() error { + if fail && written >= after { + return errInjected + } + written++ + m.Writes.Add(1) + return nil + } + // Removals first, in a stable order. + for _, k := range sortedKeys(have) { + for _, id := range sortedIDs(have[k]) { + if g, ok := want[k][id]; ok && g == have[k][id] { + continue + } + if _, keep := want[k][id]; keep { + continue + } + if err := step(); err != nil { + return err + } + if m.Kernel != nil { + if err := m.Kernel.Remove(scope, Row{Key: k, ID: id, Generation: have[k][id]}); err != nil { + return err + } + } + delete(have[k], id) + if len(have[k]) == 0 { + delete(have, k) + } + } + } + for _, k := range sortedKeys(want) { + for _, id := range sortedIDs(want[k]) { + g := want[k][id] + if cur, ok := have[k][id]; ok && cur == g { + continue + } + if err := step(); err != nil { + return err + } + if m.Kernel != nil { + if err := m.Kernel.Add(scope, Row{Key: k, ID: id, Generation: g}); err != nil { + return err + } + } + if have[k] == nil { + have[k] = map[string]uint64{} + } + have[k][id] = g + } + } + return nil +} + +// Read implements module.Module. +func (m *Module) Read(_ context.Context, scope module.Scope) (module.State, error) { + m.ReadCalls.Add(1) + m.mu.Lock() + ready := m.ready + m.mu.Unlock() + m.Host.mu.Lock() + defer m.Host.mu.Unlock() + out := module.State{} + for k, dests := range m.Host.rows[m.ModuleName+"/"+scope.VPC] { + out[k] = map[string]module.DestinationState{} + for id, g := range dests { + st := module.DestinationState{Generation: g, Programmed: true, Ready: true} + if ready != nil { + ok, reason := ready(k, id) + st.Ready, st.Reason = ok, reason + } + out[k][id] = st + } + } + return out, nil +} + +func sortedKeys[V any](m map[module.ClaimKey]V) []module.ClaimKey { + out := make([]module.ClaimKey, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Slice(out, func(i, j int) bool { return out[i].String() < out[j].String() }) + return out +} + +func sortedIDs(m map[string]uint64) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/internal/worker/module/module.go b/internal/worker/module/module.go new file mode 100644 index 00000000..8ae8da32 --- /dev/null +++ b/internal/worker/module/module.go @@ -0,0 +1,216 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package module is the boundary between Worker's reconciliation and the +// delivery modules that program the host: Local (#1435) for destinations on +// this node and Fabric (#1436) for destinations on other nodes. +// +// Worker resolves every destination, splits the claims of one VPC scope by +// mode and hands each module the complete desired state of the scope for its +// mode. A module never reads requests or decides authorization. +package module + +import ( + "context" + "fmt" + "net/netip" + "slices" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + "go.datum.net/galactic/internal/edge/ownership" +) + +// Scope is one VPC on this node: the unit of ownership and reconciliation. +type Scope struct { + // VPC is the base62 VPC identifier. + VPC string +} + +// Name is the scope's name in the ownership fence. +func (s Scope) Name() string { return ownership.VPCScope(s.VPC) } + +// ClaimKey names a claim. The owner is not part of it: one attachment has +// one owner, so node, VPC and attachment UID identify the claim. +type ClaimKey struct { + Node string + VPC string + AttachmentUID string +} + +func (k ClaimKey) String() string { return k.Node + "/" + k.VPC + "/" + k.AttachmentUID } + +// Destination is one resolved destination. +type Destination struct { + // ID identifies the destination within its claim, independent of its + // position in the spec. DestinationID computes it. + ID string + // Ref is the destination as the client named it. + Ref *workerv1alpha1.Destination + // Mode is MODE_LOCAL or MODE_FABRIC. + Mode workerv1alpha1.Mode + // VPC is the destination's VPC, from its trusted source. + VPC string + // Node is the node the destination is on. + Node string + // Locator is the SRv6 locator of the destination's node. + Locator netip.Prefix +} + +// DestinationID is the identity of a destination within a claim. +func DestinationID(d *workerv1alpha1.Destination) string { + switch t := d.GetTarget().(type) { + case *workerv1alpha1.Destination_Attachment: + return "attachment/" + t.Attachment.GetUid() + case *workerv1alpha1.Destination_Shard: + return fmt.Sprintf("shard/%s/%d", t.Shard.GetGroup(), t.Shard.GetIndex()) + case *workerv1alpha1.Destination_Backend: + return fmt.Sprintf("backend/%s/%d", t.Backend.GetNode(), t.Backend.GetSlot()) + default: + return "" + } +} + +// Claim is the desired state of one claim, restricted to the destinations a +// module programs. +type Claim struct { + Key ClaimKey + Owner string + Generation uint64 + Handoff workerv1alpha1.Handoff + // Destinations are the claim's resolved destinations for the module's + // mode, each with an available capability. + Destinations []Destination +} + +// DestinationState is what a module read back for one destination. +type DestinationState struct { + // Generation is the claim generation the programmed state belongs to. + Generation uint64 + // Programmed is true when the module's writes for the destination are on + // the host, confirmed by read-back. + Programmed bool + // Ready is true when observed state, which the module did not write, + // shows the handoff can carry traffic. + Ready bool + // Reason explains Programmed or Ready being false, with a contract + // reason such as HandoffInterfaceMissing. + Reason string + Message string +} + +// State is a module's read-back for a scope, by claim and destination ID. +// A claim with no programmed state for any destination is absent. +type State map[ClaimKey]map[string]DestinationState + +// Change is a module's prepared change for a scope. +type Change interface { + // Empty reports that applying the change would write nothing. + Empty() bool +} + +// Capability is the state of one capability a module serves. +type Capability struct { + Name string + Available bool + // Reason is a contract reason when unavailable, such as + // DatapathNotLoaded or HookUnsupported. + Reason string + Message string +} + +// Module programs the host for one delivery mode. +// +// Plan and Read must not write. Apply runs inside the scope's fence commit +// (internal/edge/ownership): it must finish every write before it returns, +// must not call the fence, and must leave the host matching desired even +// when the host held rows from an earlier incarnation or a partial commit, +// because Worker calls it with the complete desired state of the scope. +type Module interface { + // Name names the module, such as "local" or "fabric". + Name() string + // Mode is the destination mode the module programs. + Mode() workerv1alpha1.Mode + // Capabilities reports the capabilities the module serves. + Capabilities() []Capability + // Plan prepares the change that makes the scope match desired. + Plan(ctx context.Context, scope Scope, desired []Claim) (Change, error) + // Apply writes a change Plan prepared. + Apply(ctx context.Context, scope Scope, change Change) error + // Read reads the scope's programmed and observed state from the host. + Read(ctx context.Context, scope Scope) (State, error) +} + +// Registry is the set of modules Worker runs and the capability status it +// reports. +type Registry struct { + modules []Module +} + +// NewRegistry returns a registry over modules. Two modules for one mode are +// refused. +func NewRegistry(modules ...Module) (*Registry, error) { + seen := map[workerv1alpha1.Mode]string{} + for _, m := range modules { + if other, ok := seen[m.Mode()]; ok { + return nil, fmt.Errorf("module: %s and %s both serve %s", other, m.Name(), m.Mode()) + } + seen[m.Mode()] = m.Name() + } + return &Registry{modules: slices.Clone(modules)}, nil +} + +// Modules returns the registered modules. +func (r *Registry) Modules() []Module { return slices.Clone(r.modules) } + +// ForMode returns the module serving mode. +func (r *Registry) ForMode(mode workerv1alpha1.Mode) (Module, bool) { + for _, m := range r.modules { + if m.Mode() == mode { + return m, true + } + } + return nil, false +} + +// Capabilities returns one row per contract capability, in contract order. +// A capability no running module serves is Unavailable with +// ModuleNotRunning. +func (r *Registry) Capabilities() []Capability { + served := map[string]Capability{} + for _, m := range r.modules { + for _, c := range m.Capabilities() { + served[c.Name] = c + } + } + out := make([]Capability, 0, len(workerv1alpha1.Capabilities)) + for _, name := range workerv1alpha1.Capabilities { + c, ok := served[name] + if !ok { + c = Capability{Name: name, Reason: workerv1alpha1.ReasonModuleNotRunning, + Message: "no running module serves " + name} + } + out = append(out, c) + } + return out +} + +// Capability returns the status of one capability. +func (r *Registry) Capability(name string) Capability { + for _, c := range r.Capabilities() { + if c.Name == name { + return c + } + } + return Capability{Name: name, Reason: workerv1alpha1.ReasonModuleNotRunning} +} + +// RequiredCapability is the capability a destination of mode needs for a +// claim with handoff: the handoff's Local capability for a Local +// destination, fabric.srv6 for a Fabric one. +func RequiredCapability(handoff workerv1alpha1.Handoff, mode workerv1alpha1.Mode) string { + if mode == workerv1alpha1.Mode_MODE_FABRIC { + return workerv1alpha1.CapabilityFabricSRv6 + } + return workerv1alpha1.HandoffCapability(handoff) +} diff --git a/internal/worker/reconcile.go b/internal/worker/reconcile.go new file mode 100644 index 00000000..354ecd4a --- /dev/null +++ b/internal/worker/reconcile.go @@ -0,0 +1,491 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package worker + +import ( + "context" + "errors" + "fmt" + "net/netip" + "sort" + "strings" + "time" + + "google.golang.org/protobuf/proto" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + "go.datum.net/galactic/internal/edge/ownership" + "go.datum.net/galactic/internal/worker/module" + "go.datum.net/galactic/internal/worker/store" +) + +// errOwnedElsewhere is a scope another writer owns. Worker does not take it: +// moving a legacy writer's scope to Worker is that writer's transfer step. +var errOwnedElsewhere = errors.New("worker: another writer owns the scope") + +func (w *Worker) runReconciler(ctx context.Context) { + for { + vpc, quit := w.queue.Get() + if quit { + return + } + start := time.Now() + err := w.Reconcile(ctx, vpc) + observeReconcile(err, time.Since(start)) + if err != nil { + w.log.Info("reconcile failed; retrying", "vpc", vpc, "error", err) + w.queue.AddRateLimited(vpc) + } else { + w.queue.Forget(vpc) + } + w.queue.Done(vpc) + } +} + +// destEval is the resolution of one destination. +type destEval struct { + ref *workerv1alpha1.Destination + id string + mode workerv1alpha1.Mode + resolved bool + // capReason is set when the destination resolved but its capability is + // unavailable. + capReason, capMsg string + reason, msg string +} + +// claimEval is one claim's evaluation in a reconcile pass. +type claimEval struct { + key module.ClaimKey + owner string + gen uint64 + releasing bool + dests []destEval +} + +// snapshotScope copies the claims of a scope. +func (w *Worker) snapshotScope(vpc string) []claim { + w.mu.Lock() + defer w.mu.Unlock() + var out []claim + for _, c := range w.claims { + if c.rec.VPC == vpc { + out = append(out, claim{rec: c.rec, spec: proto.Clone(c.spec).(*workerv1alpha1.ClaimSpec)}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].key().String() < out[j].key().String() }) + return out +} + +// Reconcile makes the host match the desired state of one VPC scope and +// reports the result on every claim of the scope. +func (w *Worker) Reconcile(ctx context.Context, vpc string) error { + scope := module.Scope{VPC: vpc} + claims := w.snapshotScope(vpc) + if len(claims) == 0 { + w.mu.Lock() + delete(w.scopes, vpc) + w.mu.Unlock() + return nil + } + + evals, desired, err := w.resolve(ctx, vpc, claims) + if err != nil { + return err + } + + t, err := w.acquire(scope) + if err != nil { + reason := workerv1alpha1.ReasonProgrammingFailed + if errors.Is(err, errOwnedElsewhere) || errors.Is(err, ownership.ErrTransitioning) || + errors.Is(err, ownership.ErrNotOwner) { + reason = workerv1alpha1.ReasonScopeOwnedElsewhere + } + w.reportFailure(evals, reason, err) + return err + } + + mods := w.d.Modules.Modules() + changes := make([]module.Change, len(mods)) + empty := true + for i, m := range mods { + ch, err := m.Plan(ctx, scope, desired[m.Mode()]) + if err != nil { + w.reportFailure(evals, workerv1alpha1.ReasonProgrammingFailed, err) + return fmt.Errorf("plan %s: %w", m.Name(), err) + } + changes[i] = ch + empty = empty && ch.Empty() + } + if !empty || t.Resync { + mutate := func() error { + for i, m := range mods { + if changes[i].Empty() { + continue + } + if err := m.Apply(ctx, scope, changes[i]); err != nil { + return fmt.Errorf("apply %s: %w", m.Name(), err) + } + } + return nil + } + // Every change carries the complete desired state of the scope, so + // a snapshot commit is always correct; an incremental commit is used + // when the scope does not require one. + commit := w.d.Fence.Commit + if t.Resync { + commit = w.d.Fence.CommitSnapshot + } + if _, err := commit(t, mutate); err != nil { + if !errors.Is(err, ownership.ErrStale) && !errors.Is(err, ownership.ErrTransitioning) && + !errors.Is(err, ownership.ErrNotOwner) && !errors.Is(err, ownership.ErrResyncRequired) { + // The mutation failed part-way: the fence now requires a + // snapshot, which the retry commits. + w.mu.Lock() + w.scope(vpc).repair = true + w.mu.Unlock() + } + w.reportFailure(evals, workerv1alpha1.ReasonProgrammingFailed, err) + return fmt.Errorf("commit %s: %w", scope.Name(), err) + } + } + w.mu.Lock() + s := w.scope(vpc) + s.resyncing, s.repair = false, false + w.mu.Unlock() + + state := map[workerv1alpha1.Mode]module.State{} + for _, m := range mods { + st, err := m.Read(ctx, scope) + if err != nil { + w.reportFailure(evals, workerv1alpha1.ReasonProgrammingFailed, err) + return fmt.Errorf("read %s: %w", m.Name(), err) + } + state[m.Mode()] = st + } + return w.report(evals, state) +} + +// resolve evaluates every claim of a scope against trusted sources and +// capability status, withdraws claims whose authority was revoked, and +// returns the desired state of each mode. +func (w *Worker) resolve(ctx context.Context, vpc string, claims []claim) ( + []*claimEval, map[workerv1alpha1.Mode][]module.Claim, error) { + own, err := w.d.Sources.Locator(ctx, w.cfg.Node) + if err != nil { + return nil, nil, fmt.Errorf("read this node's locator: %w", err) + } + locators := map[string]netip.Prefix{w.cfg.Node: own} + locator := func(node string) (netip.Prefix, error) { + if p, ok := locators[node]; ok { + return p, nil + } + p, err := w.d.Sources.Locator(ctx, node) + if err != nil { + return netip.Prefix{}, err + } + locators[node] = p + return p, nil + } + + desired := map[workerv1alpha1.Mode][]module.Claim{} + var evals []*claimEval + for _, c := range claims { + e := &claimEval{key: c.key(), owner: c.rec.Owner, gen: c.rec.Generation, releasing: c.rec.Releasing} + evals = append(evals, e) + if e.releasing { + continue + } + att, ok, err := w.d.Sources.Attachment(ctx, e.key.AttachmentUID) + if err != nil { + return nil, nil, fmt.Errorf("read attachment %s: %w", e.key.AttachmentUID, err) + } + if !ok || att.VPC != vpc || att.Node != w.cfg.Node || !att.Admitted { + if err := w.withdraw(e.key, e.gen, store.WithdrawClaim, workerv1alpha1.ReasonAuthorizationRevoked); err != nil { + return nil, nil, err + } + e.releasing = true + continue + } + perMode := map[workerv1alpha1.Mode][]module.Destination{} + for _, d := range c.spec.GetDestinations() { + de := destEval{ref: d, id: module.DestinationID(d)} + dst, err := w.resolveDestination(ctx, vpc, own, c.spec.GetMode(), d, locator, &de) + if err != nil { + return nil, nil, err + } + if de.resolved { + capName := module.RequiredCapability(c.spec.GetHandoff(), de.mode) + if cp := w.d.Modules.Capability(capName); !cp.Available { + de.capReason, de.capMsg = cp.Reason, cp.Message + if de.capMsg == "" { + de.capMsg = capName + " is unavailable" + } + } else { + perMode[de.mode] = append(perMode[de.mode], dst) + } + } + e.dests = append(e.dests, de) + } + for mode, dests := range perMode { + desired[mode] = append(desired[mode], module.Claim{Key: e.key, Owner: e.owner, Generation: e.gen, + Handoff: c.spec.GetHandoff(), Destinations: dests}) + } + } + return evals, desired, nil +} + +// resolveDestination resolves one destination to Local or Fabric. A +// destination on this node's locator is Local; any other is Fabric. Worker +// never falls back from Local to Fabric. +func (w *Worker) resolveDestination(ctx context.Context, vpc string, own netip.Prefix, + pref workerv1alpha1.Mode, d *workerv1alpha1.Destination, + locator func(string) (netip.Prefix, error), de *destEval) (module.Destination, error) { + t, ok := d.GetTarget().(*workerv1alpha1.Destination_Attachment) + if !ok { + de.reason = workerv1alpha1.ReasonDestinationKindUnsupported + de.msg = "shard and backend destinations are resolved by later Tasks (#1437, #1438)" + return module.Destination{}, nil + } + att, found, err := w.d.Sources.Attachment(ctx, t.Attachment.GetUid()) + if err != nil { + return module.Destination{}, fmt.Errorf("read destination %s: %w", t.Attachment.GetUid(), err) + } + switch { + case !found: + de.reason, de.msg = workerv1alpha1.ReasonDestinationNotFound, "the destination attachment does not exist" + return module.Destination{}, nil + case att.VPC != vpc: + de.reason, de.msg = workerv1alpha1.ReasonVPCNotAuthorized, "the destination is in another VPC" + return module.Destination{}, nil + case !own.IsValid(): + de.reason, de.msg = workerv1alpha1.ReasonLocatorUnknown, "this node has no SRv6 locator yet" + return module.Destination{}, nil + } + loc, err := locator(att.Node) + if err != nil { + return module.Destination{}, fmt.Errorf("read locator of %s: %w", att.Node, err) + } + if !loc.IsValid() { + de.reason, de.msg = workerv1alpha1.ReasonLocatorUnknown, "the destination's node has no SRv6 locator yet" + return module.Destination{}, nil + } + mode := workerv1alpha1.Mode_MODE_FABRIC + if loc == own { + mode = workerv1alpha1.Mode_MODE_LOCAL + } + if (pref == workerv1alpha1.Mode_MODE_LOCAL || pref == workerv1alpha1.Mode_MODE_FABRIC) && pref != mode { + de.mode = mode + de.reason, de.msg = workerv1alpha1.ReasonModeUnavailable, + "the destination resolves to "+modeName(mode)+", not the requested mode" + return module.Destination{}, nil + } + de.mode, de.resolved = mode, true + return module.Destination{ID: de.id, Ref: d, Mode: mode, VPC: att.VPC, Node: att.Node, Locator: loc}, nil +} + +func modeName(m workerv1alpha1.Mode) string { + return strings.ToLower(strings.TrimPrefix(m.String(), "MODE_")) +} + +// acquire returns a ticket for this incarnation's next commit to scope: +// Begin for a scope it owns, Claim for an unowned one, and Fence then Adopt +// for one an earlier Worker incarnation owned or this one fenced. +func (w *Worker) acquire(scope module.Scope) (ownership.Ticket, error) { + name := scope.Name() + rec, err := w.d.Fence.Read(name) + if err != nil { + return ownership.Ticket{}, err + } + w.mu.Lock() + fenced := w.scope(scope.VPC).fenced + w.mu.Unlock() + workerOwned := rec.Owner == "" || strings.HasPrefix(rec.Owner, ownerPrefix) + switch { + case rec.Transitioning && fenced == rec.Generation: + case rec.Transitioning && workerOwned, !rec.Transitioning && rec.Owner != w.owner && rec.Owner != "" && workerOwned: + if fenced, err = w.d.Fence.Fence(name); err != nil { + return ownership.Ticket{}, err + } + case rec.Transitioning: + return ownership.Ticket{}, fmt.Errorf("%w: %s is moving to another owner", ownership.ErrTransitioning, name) + case rec.Owner == "": + return w.d.Fence.Claim(name, w.owner) + case rec.Owner == w.owner: + return w.d.Fence.Begin(name, w.owner) + default: + return ownership.Ticket{}, fmt.Errorf("%w: %q owns %s", errOwnedElsewhere, rec.Owner, name) + } + t, err := w.d.Fence.Adopt(name, w.owner, fenced) + w.mu.Lock() + w.scope(scope.VPC).fenced = 0 + w.mu.Unlock() + return t, err +} + +// reportFailure sets Programmed=False with reason on every live claim of +// the pass. +func (w *Worker) reportFailure(evals []*claimEval, reason string, err error) { + w.mu.Lock() + defer w.mu.Unlock() + for _, e := range evals { + c := w.claims[e.key] + if c == nil || c.rec.Generation != e.gen || c.rec.Releasing { + continue + } + setCond(c.status, cond(workerv1alpha1.ConditionProgrammed, condFalse, reason, err.Error())) + setCond(c.status, cond(workerv1alpha1.ConditionReady, condFalse, reason, "")) + c.status.ObservedGeneration = e.gen + w.publish(c) + } +} + +// report turns read-back into status, and deletes released claims whose +// programmed state is gone. +func (w *Worker) report(evals []*claimEval, state map[workerv1alpha1.Mode]module.State) error { + present := func(k module.ClaimKey) bool { + for _, st := range state { + if len(st[k]) > 0 { + return true + } + } + return false + } + var done []keyGen + w.mu.Lock() + for _, e := range evals { + c := w.claims[e.key] + if c == nil || c.rec.Generation != e.gen { + continue + } + if c.rec.Releasing { + if !present(e.key) { + done = append(done, keyGen{e.key, e.gen}) + } + continue + } + if e.releasing { + continue + } + w.applyStatus(c, e, state) + w.publish(c) + } + w.mu.Unlock() + + var errs []error + for _, d := range done { + if err := w.forget(d.key, d.gen); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +// applyStatus computes a live claim's status from its evaluation and the +// read-back. w.mu must be held. +func (w *Worker) applyStatus(c *claim, e *claimEval, state map[workerv1alpha1.Mode]module.State) { + st := c.status + st.ObservedGeneration = e.gen + st.Destinations = nil + resolved, programmed, ready := true, true, true + var rReason, rMsg, pReason, pMsg, yReason, yMsg string + first := func(r, m *string, reason, msg string) { + if *r == "" { + *r, *m = reason, msg + } + } + for _, de := range e.dests { + ds := &workerv1alpha1.DestinationStatus{Destination: de.ref, Mode: de.mode, Resolved: de.resolved} + switch { + case !de.resolved: + ds.Reason, ds.Message = de.reason, de.msg + resolved = false + first(&rReason, &rMsg, de.reason, de.id+": "+de.msg) + case de.capReason != "": + ds.Reason, ds.Message = de.capReason, de.capMsg + programmed = false + first(&pReason, &pMsg, de.capReason, de.id+": "+de.capMsg) + default: + got, ok := state[de.mode][e.key][de.id] + switch { + case !ok || got.Generation != e.gen: + ds.Reason, ds.Message = workerv1alpha1.ReasonPending, "not yet programmed" + if got.Reason != "" { + ds.Reason, ds.Message = got.Reason, got.Message + } + programmed = false + first(&pReason, &pMsg, ds.Reason, de.id+": "+ds.Message) + case !got.Programmed: + ds.Reason, ds.Message = got.Reason, got.Message + programmed = false + first(&pReason, &pMsg, got.Reason, de.id+": "+got.Message) + default: + ds.Programmed = true + ds.Ready = got.Ready + if !got.Ready { + ds.Reason, ds.Message = got.Reason, got.Message + ready = false + first(&yReason, &yMsg, got.Reason, de.id+": "+got.Message) + } + } + } + st.Destinations = append(st.Destinations, ds) + } + set := func(t string, ok bool, reason, msg string) { + if ok { + setCond(st, cond(t, condTrue, workerv1alpha1.ReasonAsExpected, "")) + return + } + setCond(st, cond(t, condFalse, reason, msg)) + } + set(workerv1alpha1.ConditionAdmitted, true, "", "") + set(workerv1alpha1.ConditionResolved, resolved, rReason, rMsg) + // Programmed covers every resolved destination; an unresolved one has + // nothing to program and is reported by Resolved. + set(workerv1alpha1.ConditionProgrammed, programmed, pReason, pMsg) + readyOK := resolved && programmed && ready + switch { + case !resolved: + set(workerv1alpha1.ConditionReady, false, rReason, rMsg) + case !programmed: + set(workerv1alpha1.ConditionReady, false, pReason, pMsg) + default: + set(workerv1alpha1.ConditionReady, readyOK, yReason, yMsg) + } + set(workerv1alpha1.ConditionWithdrawing, false, workerv1alpha1.ReasonAsExpected, "") + st.WithdrawalScope = workerv1alpha1.WithdrawalScope_WITHDRAWAL_SCOPE_UNSPECIFIED +} + +// forget deletes a released claim once its programmed state is confirmed +// gone, and sends its last snapshot. +func (w *Worker) forget(k module.ClaimKey, gen uint64) error { + lock := w.keyLock(k) + lock.Lock() + defer lock.Unlock() + w.mu.Lock() + c := w.claims[k] + if c == nil || c.rec.Generation != gen || !c.rec.Releasing { + w.mu.Unlock() + return nil + } + w.mu.Unlock() + if err := w.d.Store.Delete(k.Node, k.VPC, k.AttachmentUID); err != nil { + return fmt.Errorf("delete released claim: %w", err) + } + w.mu.Lock() + delete(w.claims, k) + for _, t := range []string{workerv1alpha1.ConditionProgrammed, workerv1alpha1.ConditionReady, + workerv1alpha1.ConditionWithdrawing} { + setCond(c.status, cond(t, condFalse, workerv1alpha1.ReasonReleased, "programmed state removed")) + } + c.status.Released = true + c.status.ObservedGeneration = gen + c.status.Destinations = nil + w.publish(c) + w.mu.Unlock() + updateClaimGauge(w) + w.log.Info("claim released", "owner", c.rec.Owner, "vpc", k.VPC, "attachment", k.AttachmentUID, "generation", gen) + return nil +} diff --git a/internal/worker/recovery_test.go b/internal/worker/recovery_test.go new file mode 100644 index 00000000..f4ae7157 --- /dev/null +++ b/internal/worker/recovery_test.go @@ -0,0 +1,352 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package worker + +import ( + "slices" + "strings" + "testing" + "time" + + "google.golang.org/grpc/codes" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + "go.datum.net/galactic/internal/worker/module" +) + +// TestRestartWithPersistedState restarts Worker over programmed state. +// The new incarnation fences and adopts the scope, commits a snapshot that +// writes nothing because the host already matches, and a client that +// repeats its last request gets a duplicate answer. +func TestRestartWithPersistedState(t *testing.T) { + e := newEnv(t) + e.start() + k := mkey(vpcA, "svc-a") + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 2, veth("guest-a1", "guest-a2"))) + e.waitReady(k, 2) + rows := e.host.Rows(vpcA) + oldOwner := e.w.Owner() + + e.crash() + e.start() + e.waitReady(k, 2) + if got := e.local.Writes.Load() + e.fabric.Writes.Load(); got != 0 { + t.Fatalf("restart rewrote %d rows of matching host state", got) + } + if !slices.Equal(rows, e.host.Rows(vpcA)) { + t.Fatal("restart changed host state") + } + rec, err := e.fence.Read(module.Scope{VPC: vpcA}.Name()) + e.mustOK(err) + if rec.Owner != e.w.Owner() || rec.Owner == oldOwner || rec.Pending { + t.Fatalf("fence record = %+v, want adopted by the new incarnation", rec) + } + // The client restarts too and repeats its last request. + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 2, veth("guest-a1", "guest-a2"))) +} + +// TestCrashBeforeProgramming crashes after the claim is recorded and while +// reconciliation holds a ticket but has written nothing. The restart +// programs the claim from the persisted desired state. +func TestCrashBeforeProgramming(t *testing.T) { + e := newEnv(t) + e.start() + entered := e.local.PausePlan() + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1"))) + <-entered + e.crash() + if rows := e.host.Rows(vpcA); len(rows) != 0 { + t.Fatalf("rows before restart = %v", rowIDs(rows)) + } + e.start() + e.waitReady(mkey(vpcA, "svc-a"), 1) +} + +// TestCrashMidApply leaves a partly applied commit behind. The fence marks +// the scope resync-required, and the next incarnation's snapshot finishes +// the change instead of assuming it either done or undone. +func TestCrashMidApply(t *testing.T) { + e := newEnv(t) + e.cfg.RetryMin, e.cfg.RetryMax = time.Hour, time.Hour + e.start() + e.local.FailApply(1, 1) + k := mkey(vpcA, "svc-a") + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1", "guest-a3"))) + e.eventually("partial commit", func() bool { + return condOf(e.status(k), workerv1alpha1.ConditionProgrammed).GetReason() == workerv1alpha1.ReasonProgrammingFailed + }) + if rows := e.host.Rows(vpcA); len(rows) != 1 { + t.Fatalf("rows after the failed commit = %v, want one of two", rowIDs(rows)) + } + rec, err := e.fence.Read(module.Scope{VPC: vpcA}.Name()) + e.mustOK(err) + if !rec.Pending { + t.Fatal("a failed commit did not leave the scope resync-required") + } + + e.crash() + e.cfg.RetryMin, e.cfg.RetryMax = 5*time.Millisecond, 50*time.Millisecond + e.start() + e.waitReady(k, 1) + if got := rowIDs(e.host.Rows(vpcA)); len(got) != 2 { + t.Fatalf("rows after repair = %v", got) + } +} + +// TestCrashBeforeReleaseRecorded removes a released claim's programmed +// state, then crashes before the record is deleted. The restart confirms the +// removal from the host and finishes the release. +func TestCrashBeforeReleaseRecorded(t *testing.T) { + e := newEnv(t) + e.cfg.RetryMin, e.cfg.RetryMax = time.Hour, time.Hour + e.start() + k := mkey(vpcA, "svc-a") + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1"))) + e.waitReady(k, 1) + e.store.failDelete.Store(1) + e.mustOK(e.release(natSA, key(vpcA, "svc-a"), 1)) + e.eventually("rows removed", func() bool { return len(e.host.Rows(vpcA)) == 0 }) + e.idle() + if recs, _ := e.store.List(); len(recs) != 1 || !recs[0].Releasing { + t.Fatalf("records = %+v, want the releasing record kept", recs) + } + + e.crash() + e.cfg.RetryMin, e.cfg.RetryMax = 5*time.Millisecond, 50*time.Millisecond + e.start() + if st := e.status(k); st == nil || condOf(st, workerv1alpha1.ConditionWithdrawing).GetStatus() != condTrue { + t.Fatalf("restored status = %v, want Withdrawing", st) + } + e.waitGone(k) + if recs, _ := e.store.List(); len(recs) != 0 { + t.Fatalf("records after restart = %+v", recs) + } +} + +// TestStoreFailureRefusesWithoutChange fails the durable write: the call +// returns UNAVAILABLE and Worker holds nothing it did not record. +func TestStoreFailureRefusesWithoutChange(t *testing.T) { + e := newEnv(t) + e.start() + e.store.failPut.Store(1) + wantRefusal(t, e.request(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1")), codes.Unavailable, + workerv1alpha1.ReasonResyncing) + if st := e.status(mkey(vpcA, "svc-a")); st != nil { + t.Fatalf("an unrecorded claim is held: %v", st) + } + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1"))) + e.waitReady(mkey(vpcA, "svc-a"), 1) +} + +// TestResyncingRefusesChanges refuses claim changes for a scope until the +// restart's snapshot commits, but still answers duplicates. +func TestResyncingRefusesChanges(t *testing.T) { + e := newEnv(t) + e.start() + k := mkey(vpcA, "svc-a") + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1"))) + e.waitReady(k, 1) + e.crash() + + e.pauseOnStart = true + e.start() + <-e.startPaused + if !e.isResyncing(vpcA) { + t.Fatal("scope is not resyncing before the restart's snapshot") + } + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1"))) + wantRefusal(t, e.update(natSA, key(vpcA, "svc-a"), 2, veth("guest-a3")), codes.Unavailable, + workerv1alpha1.ReasonResyncing) + wantRefusal(t, e.release(natSA, key(vpcA, "svc-a"), 1), codes.Unavailable, workerv1alpha1.ReasonResyncing) + e.local.ResumePlan() + e.waitReady(k, 1) + e.mustOK(e.update(natSA, key(vpcA, "svc-a"), 2, veth("guest-a3"))) + e.waitReady(k, 2) +} + +func (e *env) isResyncing(vpc string) bool { + e.w.mu.Lock() + defer e.w.mu.Unlock() + s := e.w.scopes[vpc] + return s != nil && s.resyncing +} + +// TestDuplicateWatchEvents floods the queue with change events. They +// coalesce, and a scope that already matches is neither committed nor +// written again. +func TestDuplicateWatchEvents(t *testing.T) { + e := newEnv(t) + e.start() + k := mkey(vpcA, "svc-a") + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1", "guest-a2"))) + e.waitReady(k, 1) + e.idle() + before := e.persisted() + writes := e.local.Writes.Load() + e.fabric.Writes.Load() + applies := e.local.ApplyCalls.Load() + e.fabric.ApplyCalls.Load() + for range 200 { + e.w.Changed(vpcA) + e.w.Changed("") + } + e.idle() + if e.local.Writes.Load()+e.fabric.Writes.Load() != writes || + e.local.ApplyCalls.Load()+e.fabric.ApplyCalls.Load() != applies { + t.Fatal("duplicate events wrote host state") + } + // Plan and Read run, but no commit advances the fence revision. + after := e.persisted() + for name, b := range before { + if strings.HasPrefix(name, "ownership/") && after[name] != b { + t.Fatalf("duplicate events changed %s", name) + } + } + e.waitReady(k, 1) +} + +// TestLeaseExpiryWithdrawsCallerClaims lets a caller's sessions lapse: every +// claim of that caller is withdrawn with scope "every claim of the caller", +// and other callers' claims stay. +func TestLeaseExpiryWithdrawsCallerClaims(t *testing.T) { + e := newEnv(t) + e.start() + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1"))) + e.mustOK(e.request(natSA, key(vpcB, "svc-b"), 1, veth("guest-b1"))) + e.mustOK(e.request(otherSA, key(vpcA, "svc-a2"), 1, veth("guest-a1"))) + e.waitReady(mkey(vpcA, "svc-a"), 1) + e.waitReady(mkey(vpcB, "svc-b"), 1) + e.waitReady(mkey(vpcA, "svc-a2"), 1) + + e.clock.Advance(e.cfg.Lease / 2) + e.ctx(otherSA) // otherSA's session is live. + e.mustOK(e.update(otherSA, key(vpcA, "svc-a2"), 2, veth("guest-a1"))) + e.clock.Advance(e.cfg.Lease/2 + time.Second) + e.w.ExpireLeases() + + for _, k := range []module.ClaimKey{mkey(vpcA, "svc-a"), mkey(vpcB, "svc-b")} { + if st := e.status(k); st != nil && st.GetWithdrawalScope() != workerv1alpha1.WithdrawalScope_WITHDRAWAL_SCOPE_CALLER { + t.Fatalf("withdrawal scope = %v, want every claim of the caller", st.GetWithdrawalScope()) + } + e.waitGone(k) + } + e.waitReady(mkey(vpcA, "svc-a2"), 2) + if got := rowIDs(e.host.Rows(vpcA)); !slices.Equal(got, []string{"svc-a2→attachment/guest-a1"}) { + t.Fatalf("rows = %v, want only the live caller's claim", got) + } + if len(e.host.Rows(vpcB)) != 0 { + t.Fatal("expired caller's rows remain in VPC B") + } +} + +// TestAuthorizationRevoked withdraws a claim whose attachment loses its +// authority, reporting Admitted=False with AuthorizationRevoked. +func TestAuthorizationRevoked(t *testing.T) { + e := newEnv(t) + e.start() + k := mkey(vpcA, "svc-a") + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1"))) + e.waitReady(k, 1) + stream, err := e.api.Observe(e.ctx(natSA), &workerv1alpha1.ObserveRequest{Key: key(vpcA, "svc-a")}) + e.mustOK(err) + + a, _, _ := e.src.Attachment(t.Context(), "svc-a") + a.Admitted = false + e.src.SetAttachment(a) + e.w.Changed(vpcA) + + sawRevoked := false + for { + r, err := stream.Recv() + e.mustOK(err) + st := r.GetStatus() + if c := condOf(st, workerv1alpha1.ConditionAdmitted); c.GetStatus() == condFalse && + c.GetReason() == workerv1alpha1.ReasonAuthorizationRevoked { + sawRevoked = true + } + if st.GetReleased() { + break + } + } + if !sawRevoked { + t.Fatal("never observed Admitted=False/AuthorizationRevoked") + } + if rows := e.host.Rows(vpcA); len(rows) != 0 { + t.Fatalf("rows after revocation = %v", rowIDs(rows)) + } +} + +// TestWithdrawalDuringRepair is the contract's worked transition: a commit +// failed part-way, the repair holds a ticket while it prepares a snapshot, +// and the owner releases the claim. The release fences the scope, the +// repair's late commit is dropped, and the snapshot that lands holds no row +// of the released claim while other claims of the scope stay programmed. +func TestWithdrawalDuringRepair(t *testing.T) { + e := newEnv(t) + e.start() + keep := mkey(vpcA, "svc-a2") + e.mustOK(e.request(otherSA, key(vpcA, "svc-a2"), 1, veth("guest-a3"))) + e.waitReady(keep, 1) + + e.local.FailApply(1, 0) + k := mkey(vpcA, "svc-a") + entered := e.local.PausePlan() + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1"))) + // The first pass pauses at Plan; let it fail at Apply. + <-entered + entered = e.local.RepausePlan() + // The retry is the repair: it holds a ticket while it prepares. + <-entered + if !e.isRepairing(vpcA) { + t.Fatal("scope is not in repair after the failed commit") + } + before, err := e.fence.Read(module.Scope{VPC: vpcA}.Name()) + e.mustOK(err) + + e.mustOK(e.release(natSA, key(vpcA, "svc-a"), 1)) + if st := e.status(k); st.GetWithdrawalScope() != workerv1alpha1.WithdrawalScope_WITHDRAWAL_SCOPE_CLAIM { + t.Fatalf("withdrawal scope = %v, want one claim", st.GetWithdrawalScope()) + } + fenced, err := e.fence.Read(module.Scope{VPC: vpcA}.Name()) + e.mustOK(err) + if !fenced.Transitioning || fenced.Generation <= before.Generation { + t.Fatalf("release during repair did not fence the scope: %+v", fenced) + } + e.local.ResumePlan() + + e.waitGone(k) + e.waitReady(keep, 1) + if got := rowIDs(e.host.Rows(vpcA)); !slices.Equal(got, []string{"svc-a2→attachment/guest-a3"}) { + t.Fatalf("rows = %v, want only the kept claim", got) + } + rec, err := e.fence.Read(module.Scope{VPC: vpcA}.Name()) + e.mustOK(err) + if rec.Transitioning || rec.Pending || rec.Owner != e.w.Owner() { + t.Fatalf("fence record after repair = %+v", rec) + } +} + +func (e *env) isRepairing(vpc string) bool { + e.w.mu.Lock() + defer e.w.mu.Unlock() + s := e.w.scopes[vpc] + return s != nil && s.repair +} + +// TestScopeOwnedElsewhere leaves a scope another writer owns alone and +// reports why its claims are not programmed. +func TestScopeOwnedElsewhere(t *testing.T) { + e := newEnv(t) + if _, err := e.fence.Claim(module.Scope{VPC: vpcA}.Name(), "galactic-edge:node-a:1"); err != nil { + t.Fatal(err) + } + e.start() + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1"))) + st := e.waitObserved(mkey(vpcA, "svc-a")) + if c := condOf(st, workerv1alpha1.ConditionProgrammed); c.GetReason() != workerv1alpha1.ReasonScopeOwnedElsewhere { + t.Fatalf("Programmed = %v, want ScopeOwnedElsewhere", c) + } + if rows := e.host.Rows(vpcA); len(rows) != 0 { + t.Fatalf("rows = %v, want none in a scope another writer owns", rowIDs(rows)) + } +} diff --git a/internal/worker/refusal/refusal.go b/internal/worker/refusal/refusal.go new file mode 100644 index 00000000..3724420a --- /dev/null +++ b/internal/worker/refusal/refusal.go @@ -0,0 +1,56 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package refusal builds the gRPC errors Worker returns: a status code and a +// google.rpc.ErrorInfo carrying a machine reason from the contract +// (docs/worker/contract.md, "Errors and retry"). +package refusal + +import ( + "errors" + + "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" +) + +// New returns a gRPC error with code, reason and optional ErrorInfo metadata. +func New(code codes.Code, reason, msg string, md map[string]string) error { + st := status.New(code, msg) + if d, err := st.WithDetails(&errdetails.ErrorInfo{ + Reason: reason, Domain: workerv1alpha1.ErrorDomain, Metadata: md, + }); err == nil { + st = d + } + return st.Err() +} + +// Reason returns the machine reason of a Worker error, or "" when err +// carries none. +func Reason(err error) string { + st, ok := status.FromError(err) + if !ok { + return "" + } + for _, d := range st.Details() { + if info, ok := d.(*errdetails.ErrorInfo); ok && info.GetDomain() == workerv1alpha1.ErrorDomain { + return info.GetReason() + } + } + return "" +} + +// Is reports whether err is a Worker error with code and reason. +func Is(err error, code codes.Code, reason string) bool { + if err == nil { + return false + } + var se interface{ GRPCStatus() *status.Status } + if !errors.As(err, &se) { + return false + } + return se.GRPCStatus().Code() == code && Reason(err) == reason +} diff --git a/internal/worker/sessions.go b/internal/worker/sessions.go new file mode 100644 index 00000000..dee2a409 --- /dev/null +++ b/internal/worker/sessions.go @@ -0,0 +1,126 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package worker + +import ( + "context" + "crypto/rand" + "encoding/hex" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + "go.datum.net/galactic/internal/worker/auth" + "go.datum.net/galactic/internal/worker/refusal" + "go.datum.net/galactic/internal/worker/store" +) + +// session is one client session. The caller, not the session, owns claims: +// a client that restarts opens a new session and keeps its claims. +type session struct { + id string + caller string + expires time.Time +} + +func sessionID(ctx context.Context) string { + md, _ := metadata.FromIncomingContext(ctx) + if v := md.Get(workerv1alpha1.SessionMetadataKey); len(v) == 1 { + return v[0] + } + return "" +} + +// openSession renews the live session the call names, or starts a new one. +func (w *Worker) openSession(ctx context.Context, caller auth.Caller) (string, error) { + now := w.d.Now() + w.mu.Lock() + defer w.mu.Unlock() + if id := sessionID(ctx); id != "" { + if s, ok := w.sessions[id]; ok && s.caller == caller.ServiceAccount && now.Before(s.expires) { + s.expires = now.Add(w.cfg.Lease) + w.lastSeen[caller.ServiceAccount] = now + return id, nil + } + } + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return "", refusal.New(codes.Unavailable, workerv1alpha1.ReasonShuttingDown, "cannot create a session ID", nil) + } + id := hex.EncodeToString(b) + w.sessions[id] = &session{id: id, caller: caller.ServiceAccount, expires: now.Add(w.cfg.Lease)} + w.lastSeen[caller.ServiceAccount] = now + setSessions(len(w.sessions)) + return id, nil +} + +// touchSession renews the session the call names, which must be a live +// session of caller. +func (w *Worker) touchSession(ctx context.Context, caller auth.Caller) error { + id := sessionID(ctx) + now := w.d.Now() + w.mu.Lock() + defer w.mu.Unlock() + s, ok := w.sessions[id] + if id == "" || !ok || s.caller != caller.ServiceAccount || !now.Before(s.expires) { + return refusal.New(codes.FailedPrecondition, workerv1alpha1.ReasonNoSession, + "no live session; call GetCapabilities and send its session ID in "+workerv1alpha1.SessionMetadataKey, nil) + } + s.expires = now.Add(w.cfg.Lease) + w.lastSeen[caller.ServiceAccount] = now + return nil +} + +// begin authenticates a call other than GetCapabilities and renews its +// session. +func (w *Worker) begin(ctx context.Context) (auth.Caller, error) { + caller, err := w.d.Auth.Authenticate(ctx) + if err != nil { + countRefusal(refusal.Reason(err)) + return auth.Caller{}, err + } + if err := w.touchSession(ctx, caller); err != nil { + countRefusal(workerv1alpha1.ReasonNoSession) + return auth.Caller{}, err + } + return caller, nil +} + +// ExpireLeases withdraws every claim of a caller that has had no live +// session for a lease, with scope "every claim of the caller", and drops +// expired sessions. The lease checker calls it every renew interval. +func (w *Worker) ExpireLeases() { + now := w.d.Now() + w.mu.Lock() + for id, s := range w.sessions { + if !now.Before(s.expires) { + delete(w.sessions, id) + } + } + setSessions(len(w.sessions)) + live := map[string]bool{} + for _, s := range w.sessions { + live[s.caller] = true + } + var expired []keyGen + for k, c := range w.claims { + if c.rec.Releasing || live[c.rec.Owner] { + continue + } + if seen, ok := w.lastSeen[c.rec.Owner]; ok && now.Sub(seen) < w.cfg.Lease { + continue + } + expired = append(expired, keyGen{k, c.rec.Generation}) + } + w.mu.Unlock() + for _, e := range expired { + // A failed write is retried at the next check. + if err := w.withdraw(e.key, e.gen, store.WithdrawCaller, workerv1alpha1.ReasonLeaseExpired); err != nil { + w.log.Error("withdraw claims of an expired caller", "owner", e.key.String(), "error", err) + } + } +} diff --git a/internal/worker/socket.go b/internal/worker/socket.go new file mode 100644 index 00000000..18d46b11 --- /dev/null +++ b/internal/worker/socket.go @@ -0,0 +1,53 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package worker + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "path/filepath" + + "google.golang.org/grpc" + + "go.datum.net/galactic/internal/worker/auth" +) + +// Listen opens Worker's unix socket at path with mode 0660, owned by gid +// when gid is not negative. A stale socket from an earlier process is +// replaced. No TCP listener serves the mutating API. +func Listen(ctx context.Context, path string, gid int) (net.Listener, error) { + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return nil, fmt.Errorf("worker: socket directory: %w", err) + } + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("worker: remove stale socket: %w", err) + } + lis, err := (&net.ListenConfig{}).Listen(ctx, "unix", path) + if err != nil { + return nil, fmt.Errorf("worker: listen on %s: %w", path, err) + } + if err := os.Chmod(path, 0o660); err != nil { + _ = lis.Close() + return nil, fmt.Errorf("worker: chmod socket: %w", err) + } + if gid >= 0 { + if err := os.Chown(path, -1, gid); err != nil { + _ = lis.Close() + return nil, fmt.Errorf("worker: chown socket: %w", err) + } + } + return lis, nil +} + +// NewServer returns a gRPC server whose connections carry the peer's +// SO_PEERCRED, with the Worker service registered. +func (w *Worker) NewServer(opts ...grpc.ServerOption) *grpc.Server { + s := grpc.NewServer(append([]grpc.ServerOption{grpc.Creds(auth.PeerCredentials{})}, opts...)...) + w.Register(s) + return s +} diff --git a/internal/worker/sources/sources.go b/internal/worker/sources/sources.go new file mode 100644 index 00000000..67b94a21 --- /dev/null +++ b/internal/worker/sources/sources.go @@ -0,0 +1,217 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package sources reads the facts Worker takes only from trusted sources, +// never from a request (docs/worker/contract.md, "Trusted sources"): an +// attachment's VPC, node and admission, and a node's physical locator. +package sources + +import ( + "context" + "fmt" + "net/netip" + "sync" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + toolscache "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + + cloudv1alpha1 "go.datum.net/galactic/api/cloud/v1alpha1" + bgpv1alpha1 "go.datum.net/galactic/api/network/v1alpha1" + galacticv1alpha1 "go.datum.net/galactic/api/v1alpha1" + "go.datum.net/galactic/internal/nodelocator" +) + +// Attachment kinds. +const ( + KindVPCAttachment = "VPCAttachment" + KindEdgeAttachment = "EdgeAttachment" +) + +// Attachment is the trusted record of an attachment. +type Attachment struct { + UID string + Kind string + // VPC is the base62 VPC identifier the attachment's authority recorded. + VPC string + // Node is the node the attachment lives on. + Node string + // Admitted is false for an attachment being deleted, and for an + // EdgeAttachment whose Admitted condition is not True. + Admitted bool +} + +// Sources answers Worker's trusted-source lookups. +type Sources interface { + // Attachment returns the attachment with uid, and false when none + // exists. + Attachment(ctx context.Context, uid string) (Attachment, bool, error) + // Locator returns node's SRv6 locator, or the zero prefix while it has + // none. + Locator(ctx context.Context, node string) (netip.Prefix, error) +} + +// uidIndex indexes attachments by metadata.uid. +const uidIndex = "metadata.uid" + +// RegisterIndexes adds the field indexes Kube needs to a cache. +func RegisterIndexes(ctx context.Context, idx client.FieldIndexer) error { + byUID := func(o client.Object) []string { return []string{string(o.GetUID())} } + if err := idx.IndexField(ctx, &cloudv1alpha1.VPCAttachment{}, uidIndex, byUID); err != nil { + return fmt.Errorf("sources: index VPCAttachment: %w", err) + } + if err := idx.IndexField(ctx, &galacticv1alpha1.EdgeAttachment{}, uidIndex, byUID); err != nil { + return fmt.Errorf("sources: index EdgeAttachment: %w", err) + } + return nil +} + +// Kube reads trusted sources through a controller-runtime cache whose +// indexes RegisterIndexes added. +type Kube struct { + Reader client.Reader + // Namespace is where BGPRouters live. + Namespace string +} + +// Attachment implements Sources. +func (k Kube) Attachment(ctx context.Context, uid string) (Attachment, bool, error) { + var vas cloudv1alpha1.VPCAttachmentList + if err := k.Reader.List(ctx, &vas, client.MatchingFields{uidIndex: uid}); err != nil { + return Attachment{}, false, fmt.Errorf("sources: list VPCAttachments: %w", err) + } + if len(vas.Items) == 1 { + a := vas.Items[0] + return Attachment{UID: uid, Kind: KindVPCAttachment, VPC: a.Status.VPC, Node: a.Status.Node, + Admitted: a.DeletionTimestamp.IsZero()}, true, nil + } + var eas galacticv1alpha1.EdgeAttachmentList + if err := k.Reader.List(ctx, &eas, client.MatchingFields{uidIndex: uid}); err != nil { + return Attachment{}, false, fmt.Errorf("sources: list EdgeAttachments: %w", err) + } + if len(eas.Items) == 1 { + a := eas.Items[0] + admitted := a.DeletionTimestamp.IsZero() && + meta.IsStatusConditionTrue(a.Status.Conditions, galacticv1alpha1.EdgeAttachmentAdmitted) + return Attachment{UID: uid, Kind: KindEdgeAttachment, VPC: a.Status.VPC, Node: a.Status.NodeName, + Admitted: admitted}, true, nil + } + return Attachment{}, false, nil +} + +// Locator implements Sources. +func (k Kube) Locator(ctx context.Context, node string) (netip.Prefix, error) { + return nodelocator.Prefix(ctx, k.Reader, k.Namespace, node) +} + +// Watch calls changed with a VPC identifier whenever an attachment in that +// VPC changes, and with "" when a BGPRouter changes, which can move any +// destination between Local and Fabric. Duplicate events are expected; +// Worker's queue coalesces them. +func Watch(ctx context.Context, c cache.Cache, changed func(vpc string)) error { + vpcOf := func(obj any) string { + if t, ok := obj.(toolscache.DeletedFinalStateUnknown); ok { + obj = t.Obj + } + switch o := obj.(type) { + case *cloudv1alpha1.VPCAttachment: + return o.Status.VPC + case *galacticv1alpha1.EdgeAttachment: + return o.Status.VPC + } + return "" + } + notify := func(obj any) { + if v := vpcOf(obj); v != "" { + changed(v) + } + } + for _, obj := range []client.Object{&cloudv1alpha1.VPCAttachment{}, &galacticv1alpha1.EdgeAttachment{}} { + inf, err := c.GetInformer(ctx, obj) + if err != nil { + return fmt.Errorf("sources: informer: %w", err) + } + if _, err := inf.AddEventHandler(toolscache.ResourceEventHandlerFuncs{ + AddFunc: notify, + UpdateFunc: func(oldObj, newObj any) { notify(oldObj); notify(newObj) }, + DeleteFunc: notify, + }); err != nil { + return fmt.Errorf("sources: handler: %w", err) + } + } + inf, err := c.GetInformer(ctx, &bgpv1alpha1.BGPRouter{}) + if err != nil { + return fmt.Errorf("sources: informer: %w", err) + } + all := func(any) { changed("") } + if _, err := inf.AddEventHandler(toolscache.ResourceEventHandlerFuncs{ + AddFunc: all, UpdateFunc: func(_, n any) { all(n) }, DeleteFunc: all, + }); err != nil { + return fmt.Errorf("sources: handler: %w", err) + } + return nil +} + +// AddToScheme adds every type Kube reads to a scheme. +func AddToScheme(s *runtime.Scheme) error { + for _, add := range []func(*runtime.Scheme) error{ + cloudv1alpha1.AddToScheme, galacticv1alpha1.AddToScheme, bgpv1alpha1.AddToScheme, + } { + if err := add(s); err != nil { + return err + } + } + return nil +} + +// Static is an in-memory Sources for tests and for nodes without an API +// server. +type Static struct { + mu sync.Mutex + attachments map[string]Attachment + locators map[string]netip.Prefix +} + +// NewStatic returns an empty Static. +func NewStatic() *Static { + return &Static{attachments: map[string]Attachment{}, locators: map[string]netip.Prefix{}} +} + +// SetAttachment adds or replaces an attachment. +func (s *Static) SetAttachment(a Attachment) { + s.mu.Lock() + defer s.mu.Unlock() + s.attachments[a.UID] = a +} + +// DeleteAttachment removes an attachment. +func (s *Static) DeleteAttachment(uid string) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.attachments, uid) +} + +// SetLocator sets a node's locator. +func (s *Static) SetLocator(node string, p netip.Prefix) { + s.mu.Lock() + defer s.mu.Unlock() + s.locators[node] = p +} + +// Attachment implements Sources. +func (s *Static) Attachment(_ context.Context, uid string) (Attachment, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + a, ok := s.attachments[uid] + return a, ok, nil +} + +// Locator implements Sources. +func (s *Static) Locator(_ context.Context, node string) (netip.Prefix, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.locators[node], nil +} diff --git a/internal/worker/sources/sources_test.go b/internal/worker/sources/sources_test.go new file mode 100644 index 00000000..2f57ef76 --- /dev/null +++ b/internal/worker/sources/sources_test.go @@ -0,0 +1,76 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package sources + +import ( + "context" + "net/netip" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + cloudv1alpha1 "go.datum.net/galactic/api/cloud/v1alpha1" + bgpv1alpha1 "go.datum.net/galactic/api/network/v1alpha1" + galacticv1alpha1 "go.datum.net/galactic/api/v1alpha1" +) + +const ( + uidVA = "uid-va" + uidEA = "uid-ea" + uidEB = "uid-eb" + nodeA = "node-a" + vpcA = "vpcA" +) + +func TestKube(t *testing.T) { + s := runtime.NewScheme() + if err := AddToScheme(s); err != nil { + t.Fatal(err) + } + byUID := func(o client.Object) []string { return []string{string(o.GetUID())} } + va := &cloudv1alpha1.VPCAttachment{ObjectMeta: metav1.ObjectMeta{Name: "va", Namespace: "p", UID: uidVA}, + Status: cloudv1alpha1.VPCAttachmentStatus{VPC: vpcA, Node: nodeA}} + admitted := &galacticv1alpha1.EdgeAttachment{ObjectMeta: metav1.ObjectMeta{Name: "ea", Namespace: "p", UID: uidEA}, + Status: galacticv1alpha1.EdgeAttachmentStatus{VPC: vpcA, NodeName: nodeA, Conditions: []metav1.Condition{{ + Type: galacticv1alpha1.EdgeAttachmentAdmitted, Status: metav1.ConditionTrue, Reason: "Admitted", + LastTransitionTime: metav1.Now()}}}} + pending := &galacticv1alpha1.EdgeAttachment{ObjectMeta: metav1.ObjectMeta{Name: "eb", Namespace: "p", UID: uidEB}, + Status: galacticv1alpha1.EdgeAttachmentStatus{VPC: vpcA, NodeName: nodeA}} + router := &bgpv1alpha1.BGPRouter{ObjectMeta: metav1.ObjectMeta{Name: "r", Namespace: "galactic-system"}} + router.Spec.TargetRef.Name = nodeA + router.Spec.SRv6Locator = "2001:db8:1::/48" + router.Spec.NodeID = 0x10 + c := fake.NewClientBuilder().WithScheme(s). + WithObjects(va, admitted, pending, router). + WithIndex(&cloudv1alpha1.VPCAttachment{}, uidIndex, byUID). + WithIndex(&galacticv1alpha1.EdgeAttachment{}, uidIndex, byUID). + Build() + k := Kube{Reader: c, Namespace: "galactic-system"} + ctx := context.Background() + + for uid, want := range map[string]Attachment{ + uidVA: {UID: uidVA, Kind: KindVPCAttachment, VPC: vpcA, Node: nodeA, Admitted: true}, + uidEA: {UID: uidEA, Kind: KindEdgeAttachment, VPC: vpcA, Node: nodeA, Admitted: true}, + uidEB: {UID: uidEB, Kind: KindEdgeAttachment, VPC: vpcA, Node: nodeA, Admitted: false}, + } { + got, ok, err := k.Attachment(ctx, uid) + if err != nil || !ok || got != want { + t.Errorf("Attachment(%s) = %+v, %t, %v; want %+v", uid, got, ok, err, want) + } + } + if _, ok, err := k.Attachment(ctx, "missing"); ok || err != nil { + t.Errorf("Attachment(missing) = %t, %v", ok, err) + } + loc, err := k.Locator(ctx, nodeA) + if err != nil || loc != netip.MustParsePrefix("2001:db8:1:10::/64") { + t.Errorf("Locator(node-a) = %s, %v", loc, err) + } + if loc, err := k.Locator(ctx, "node-b"); err != nil || loc.IsValid() { + t.Errorf("Locator(node-b) = %s, %v; want none", loc, err) + } +} diff --git a/internal/worker/status.go b/internal/worker/status.go new file mode 100644 index 00000000..d9f7e9fc --- /dev/null +++ b/internal/worker/status.go @@ -0,0 +1,112 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package worker + +import ( + "google.golang.org/protobuf/proto" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + "go.datum.net/galactic/internal/worker/module" + "go.datum.net/galactic/internal/worker/store" +) + +var ( + condTrue = workerv1alpha1.ConditionStatus_CONDITION_STATUS_TRUE + condFalse = workerv1alpha1.ConditionStatus_CONDITION_STATUS_FALSE + condUnknown = workerv1alpha1.ConditionStatus_CONDITION_STATUS_UNKNOWN +) + +func cond(t string, s workerv1alpha1.ConditionStatus, reason, msg string) *workerv1alpha1.Condition { + return &workerv1alpha1.Condition{Type: t, Status: s, Reason: reason, Message: msg} +} + +func protoKey(k module.ClaimKey) *workerv1alpha1.ClaimKey { + return &workerv1alpha1.ClaimKey{Node: k.Node, Vpc: k.VPC, AttachmentUid: k.AttachmentUID} +} + +// initialStatus is the status of an accepted claim before reconciliation +// reports on its generation. +func (w *Worker) initialStatus(c *claim, observed uint64) *workerv1alpha1.ClaimStatus { + st := &workerv1alpha1.ClaimStatus{ + Key: protoKey(c.key()), + Generation: c.rec.Generation, + ObservedGeneration: observed, + Conditions: []*workerv1alpha1.Condition{ + cond(workerv1alpha1.ConditionAdmitted, condTrue, workerv1alpha1.ReasonAsExpected, ""), + cond(workerv1alpha1.ConditionResolved, condUnknown, workerv1alpha1.ReasonPending, ""), + cond(workerv1alpha1.ConditionProgrammed, condUnknown, workerv1alpha1.ReasonPending, ""), + cond(workerv1alpha1.ConditionReady, condUnknown, workerv1alpha1.ReasonPending, ""), + cond(workerv1alpha1.ConditionWithdrawing, condFalse, workerv1alpha1.ReasonAsExpected, ""), + }, + } + c.status = st + if c.rec.Releasing { + w.setWithdrawing(c) + } + return c.status +} + +func withdrawalScope(s string) workerv1alpha1.WithdrawalScope { + if s == store.WithdrawCaller { + return workerv1alpha1.WithdrawalScope_WITHDRAWAL_SCOPE_CALLER + } + return workerv1alpha1.WithdrawalScope_WITHDRAWAL_SCOPE_CLAIM +} + +// setWithdrawing marks a releasing claim's status Withdrawing. +func (w *Worker) setWithdrawing(c *claim) { + st := c.status + st.WithdrawalScope = withdrawalScope(c.rec.Withdrawal) + setCond(st, cond(workerv1alpha1.ConditionWithdrawing, condTrue, c.rec.WithdrawReason, "removing programmed state")) + setCond(st, cond(workerv1alpha1.ConditionReady, condFalse, c.rec.WithdrawReason, "the claim is being withdrawn")) + if c.rec.WithdrawReason == workerv1alpha1.ReasonAuthorizationRevoked { + setCond(st, cond(workerv1alpha1.ConditionAdmitted, condFalse, workerv1alpha1.ReasonAuthorizationRevoked, + "the claim's attachment is no longer authorized")) + } +} + +func setCond(st *workerv1alpha1.ClaimStatus, c *workerv1alpha1.Condition) { + for i, cur := range st.Conditions { + if cur.GetType() == c.GetType() { + st.Conditions[i] = c + return + } + } + st.Conditions = append(st.Conditions, c) +} + +// subscriber is one Observe stream. +type subscriber struct { + caller string + // key selects one claim; nil selects every claim of the caller. + key *module.ClaimKey + notify chan struct{} + pending map[module.ClaimKey]*workerv1alpha1.ClaimStatus +} + +func (s *subscriber) wants(owner string, k module.ClaimKey) bool { + return owner == s.caller && (s.key == nil || *s.key == k) +} + +// publish stamps a new revision on c's status and queues a copy for every +// matching subscriber. w.mu must be held. +func (w *Worker) publish(c *claim) { + c.status.Revision = w.nextRevision() + w.fanout(c.rec.Owner, c.key(), c.status) +} + +// fanout queues st for subscribers. w.mu must be held. +func (w *Worker) fanout(owner string, k module.ClaimKey, st *workerv1alpha1.ClaimStatus) { + for s := range w.subs { + if !s.wants(owner, k) { + continue + } + s.pending[k] = proto.Clone(st).(*workerv1alpha1.ClaimStatus) + select { + case s.notify <- struct{}{}: + default: + } + } +} diff --git a/internal/worker/store/store.go b/internal/worker/store/store.go new file mode 100644 index 00000000..6b58d3f5 --- /dev/null +++ b/internal/worker/store/store.go @@ -0,0 +1,151 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package store keeps Worker's desired state durably: one file per claim, +// replaced atomically and synced before a mutating call returns. Programmed +// and observed state are never stored; Worker rebuilds them from the host. +package store + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +// DefaultDir is where Worker keeps desired state on the host. +const DefaultDir = "/var/lib/galactic-worker/claims" + +// Withdrawal causes recorded with a releasing claim. +const ( + WithdrawClaim = "claim" + WithdrawCaller = "caller" +) + +// Record is the durable desired state of one claim. +type Record struct { + Node string `json:"node"` + VPC string `json:"vpc"` + AttachmentUID string `json:"attachmentUID"` + // Owner is the ServiceAccount, "namespace/name", that created the claim. + Owner string `json:"owner"` + Generation uint64 `json:"generation"` + // SpecHash is the hash of the deterministic encoding of Spec. + SpecHash string `json:"specHash"` + // Spec is the claim spec in protobuf JSON. + Spec json.RawMessage `json:"spec"` + // Releasing is set once a release, a revocation or a lease expiry is + // accepted. The record is deleted when read-back confirms that no + // programmed state of the claim remains. + Releasing bool `json:"releasing,omitempty"` + // Withdrawal is the scope of a releasing claim's withdrawal. + Withdrawal string `json:"withdrawal,omitempty"` + // WithdrawReason is the reason the claim is being withdrawn. + WithdrawReason string `json:"withdrawReason,omitempty"` +} + +// Store is the durable desired-state store. +type Store interface { + // Put replaces the record durably. + Put(r Record) error + // Delete removes the record durably. A missing record is not an error. + Delete(node, vpc, attachmentUID string) error + // List returns every record. + List() ([]Record, error) +} + +// Dir is a Store over a directory. +type Dir struct { + dir string +} + +// Open returns a store over dir, creating it if needed. +func Open(dir string) (*Dir, error) { + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, fmt.Errorf("store: create %s: %w", dir, err) + } + return &Dir{dir: dir}, nil +} + +func (d *Dir) path(node, vpc, att string) string { + sum := sha256.Sum256([]byte(node + "\x00" + vpc + "\x00" + att)) + return filepath.Join(d.dir, hex.EncodeToString(sum[:])+".json") +} + +// Put implements Store. +func (d *Dir) Put(r Record) error { + b, err := json.Marshal(r) + if err != nil { + return fmt.Errorf("store: encode: %w", err) + } + path := d.path(r.Node, r.VPC, r.AttachmentUID) + tmp, err := os.CreateTemp(d.dir, ".claim-*") + if err != nil { + return fmt.Errorf("store: %w", err) + } + defer func() { _ = os.Remove(tmp.Name()) }() + if _, err := tmp.Write(b); err != nil { + _ = tmp.Close() + return fmt.Errorf("store: write: %w", err) + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + return fmt.Errorf("store: sync: %w", err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("store: close: %w", err) + } + if err := os.Rename(tmp.Name(), path); err != nil { + return fmt.Errorf("store: rename: %w", err) + } + return d.syncDir() +} + +// Delete implements Store. +func (d *Dir) Delete(node, vpc, att string) error { + if err := os.Remove(d.path(node, vpc, att)); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("store: delete: %w", err) + } + return d.syncDir() +} + +// List implements Store. +func (d *Dir) List() ([]Record, error) { + entries, err := os.ReadDir(d.dir) + if err != nil { + return nil, fmt.Errorf("store: list: %w", err) + } + var out []Record + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".json") { + continue + } + b, err := os.ReadFile(filepath.Join(d.dir, e.Name())) + if err != nil { + return nil, fmt.Errorf("store: read %s: %w", e.Name(), err) + } + var r Record + if err := json.Unmarshal(b, &r); err != nil { + return nil, fmt.Errorf("store: decode %s: %w", e.Name(), err) + } + out = append(out, r) + } + return out, nil +} + +func (d *Dir) syncDir() error { + f, err := os.Open(d.dir) + if err != nil { + return fmt.Errorf("store: open dir: %w", err) + } + defer func() { _ = f.Close() }() + if err := f.Sync(); err != nil { + return fmt.Errorf("store: sync dir: %w", err) + } + return nil +} diff --git a/internal/worker/store/store_test.go b/internal/worker/store/store_test.go new file mode 100644 index 00000000..e9471ee0 --- /dev/null +++ b/internal/worker/store/store_test.go @@ -0,0 +1,61 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package store + +import ( + "encoding/json" + "os" + "testing" +) + +func TestDirRoundTrip(t *testing.T) { + dir := t.TempDir() + d, err := Open(dir) + if err != nil { + t.Fatal(err) + } + a := Record{Node: "n", VPC: "v1", AttachmentUID: "a", Owner: "ns/sa", Generation: 3, SpecHash: "h", + Spec: json.RawMessage(`{"handoff":"HANDOFF_TAP"}`)} + b := a + b.AttachmentUID, b.Releasing, b.Withdrawal = "b", true, WithdrawCaller + for _, r := range []Record{a, b} { + if err := d.Put(r); err != nil { + t.Fatal(err) + } + } + a.Generation = 4 + if err := d.Put(a); err != nil { + t.Fatal(err) + } + // A fresh handle reads what the first wrote. + d2, err := Open(dir) + if err != nil { + t.Fatal(err) + } + recs, err := d2.List() + if err != nil { + t.Fatal(err) + } + got := map[string]Record{} + for _, r := range recs { + got[r.AttachmentUID] = r + } + if len(got) != 2 || got["a"].Generation != 4 || !got["b"].Releasing || got["b"].Withdrawal != WithdrawCaller { + t.Fatalf("records = %+v", recs) + } + if err := d2.Delete("n", "v1", "a"); err != nil { + t.Fatal(err) + } + if err := d2.Delete("n", "v1", "a"); err != nil { + t.Fatalf("deleting a missing record: %v", err) + } + if recs, _ := d2.List(); len(recs) != 1 { + t.Fatalf("records after delete = %+v", recs) + } + entries, _ := os.ReadDir(dir) + if len(entries) != 1 { + t.Fatalf("directory holds %d entries, want 1 and no temporary files", len(entries)) + } +} diff --git a/internal/worker/worker.go b/internal/worker/worker.go new file mode 100644 index 00000000..bfc5f8e9 --- /dev/null +++ b/internal/worker/worker.go @@ -0,0 +1,313 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package worker is Galactic Worker's reconciliation core and gRPC service, +// implementing docs/worker/contract.md. +// +// A mutating call is validated, authenticated, authorized against trusted +// sources and recorded durably before it returns; nothing is programmed on +// the call path. Reconciliation runs per VPC scope from a deduplicating +// queue. It resolves every destination, takes the scope's ticket from the +// ownership fence (internal/edge/ownership), commits the complete desired +// state of the scope through the delivery modules and reads the host back to +// report programmed and observed state. Desired state is durable; +// programmed and observed state are rebuilt from the host after a restart. +package worker + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "log/slog" + "sync" + "sync/atomic" + "time" + + "google.golang.org/protobuf/encoding/protojson" + "k8s.io/client-go/util/workqueue" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + "go.datum.net/galactic/internal/edge/ownership" + "go.datum.net/galactic/internal/worker/auth" + "go.datum.net/galactic/internal/worker/module" + "go.datum.net/galactic/internal/worker/sources" + "go.datum.net/galactic/internal/worker/store" +) + +// Defaults from the contract. +const ( + // DefaultLease and DefaultRenewInterval are the XDP dispatcher's slot + // lease values (D5). + DefaultLease = 120 * time.Second + DefaultRenewInterval = 10 * time.Second + // DefaultClaimLimit is the number of claims one caller may hold (D18). + DefaultClaimLimit = 1024 + // DefaultRetryMin and DefaultRetryMax bound Worker's own programming + // retries. + DefaultRetryMin = time.Second + DefaultRetryMax = 60 * time.Second + // DefaultResyncInterval is how often every scope is reconciled again to + // refresh observed state. + DefaultResyncInterval = 30 * time.Second +) + +// ownerPrefix starts the fence owner ID of every Worker incarnation, so a +// new incarnation can tell a scope an earlier one owned from a scope another +// writer owns. +const ownerPrefix = "galactic-worker:" + +// Authenticator authenticates the caller of a call. +type Authenticator interface { + Authenticate(ctx context.Context) (auth.Caller, error) +} + +// Config configures a Worker. +type Config struct { + // Node is this node's name. + Node string + Lease time.Duration + RenewInterval time.Duration + ClaimLimit int + RetryMin time.Duration + RetryMax time.Duration + ResyncInterval time.Duration + // Reconcilers is the number of scopes reconciled at once. + Reconcilers int +} + +func (c *Config) defaults() { + if c.Lease == 0 { + c.Lease = DefaultLease + } + if c.RenewInterval == 0 { + c.RenewInterval = DefaultRenewInterval + } + if c.ClaimLimit == 0 { + c.ClaimLimit = DefaultClaimLimit + } + if c.RetryMin == 0 { + c.RetryMin = DefaultRetryMin + } + if c.RetryMax == 0 { + c.RetryMax = DefaultRetryMax + } + if c.ResyncInterval == 0 { + c.ResyncInterval = DefaultResyncInterval + } + if c.Reconcilers == 0 { + c.Reconcilers = 4 + } +} + +// Deps are a Worker's collaborators. +type Deps struct { + Store store.Store + Fence *ownership.Authority + Sources sources.Sources + Modules *module.Registry + Auth Authenticator + Log *slog.Logger + // Now is the clock, time.Now when nil. + Now func() time.Time +} + +// claim is one claim's desired state and last status. +type claim struct { + rec store.Record + spec *workerv1alpha1.ClaimSpec + status *workerv1alpha1.ClaimStatus +} + +func (c *claim) key() module.ClaimKey { + return module.ClaimKey{Node: c.rec.Node, VPC: c.rec.VPC, AttachmentUID: c.rec.AttachmentUID} +} + +// scopeState is Worker's view of one VPC scope. +type scopeState struct { + // resyncing is set for a scope whose desired state was loaded at start, + // until the first snapshot commit. Claim changes are refused meanwhile. + resyncing bool + // repair is set after a failed commit, until a commit succeeds. + repair bool + // fenced is the fence generation of a Fence this incarnation started and + // has not yet adopted. + fenced uint64 +} + +// Worker is the reconciliation core and gRPC service. +type Worker struct { + workerv1alpha1.UnimplementedWorkerServer + + cfg Config + d Deps + owner string + log *slog.Logger + queue workqueue.TypedRateLimitingInterface[string] + + keyLocks [256]sync.Mutex + + mu sync.Mutex + claims map[module.ClaimKey]*claim + scopes map[string]*scopeState + sessions map[string]*session + lastSeen map[string]time.Time + subs map[*subscriber]struct{} + revision uint64 + + shutting atomic.Bool + started atomic.Bool +} + +// New returns a Worker. Start loads desired state and starts reconciling. +func New(cfg Config, d Deps) (*Worker, error) { + cfg.defaults() + if cfg.Node == "" { + return nil, errors.New("worker: node name is required") + } + if d.Store == nil || d.Fence == nil || d.Sources == nil || d.Modules == nil || d.Auth == nil { + return nil, errors.New("worker: store, fence, sources, modules and auth are required") + } + if d.Now == nil { + d.Now = time.Now + } + if d.Log == nil { + d.Log = slog.Default() + } + id := make([]byte, 8) + if _, err := rand.Read(id); err != nil { + return nil, fmt.Errorf("worker: owner ID: %w", err) + } + w := &Worker{ + cfg: cfg, + d: d, + owner: ownerPrefix + cfg.Node + ":" + hex.EncodeToString(id), + log: d.Log.With("component", "galactic-worker"), + claims: map[module.ClaimKey]*claim{}, + scopes: map[string]*scopeState{}, + sessions: map[string]*session{}, + lastSeen: map[string]time.Time{}, + subs: map[*subscriber]struct{}{}, + revision: uint64(d.Now().UnixNano()), + queue: workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.NewTypedItemExponentialFailureRateLimiter[string](cfg.RetryMin, cfg.RetryMax), + workqueue.TypedRateLimitingQueueConfig[string]{}), + } + return w, nil +} + +// Owner is this incarnation's fence owner ID. +func (w *Worker) Owner() string { return w.owner } + +// Start loads desired state from the store, marks every scope that has +// claims as resyncing and starts the reconcilers, the lease checker and the +// periodic resync. They run until ctx is done. +func (w *Worker) Start(ctx context.Context) error { + if !w.started.CompareAndSwap(false, true) { + return errors.New("worker: already started") + } + recs, err := w.d.Store.List() + if err != nil { + return err + } + now := w.d.Now() + w.mu.Lock() + for _, r := range recs { + spec := &workerv1alpha1.ClaimSpec{} + if err := protojson.Unmarshal(r.Spec, spec); err != nil { + w.mu.Unlock() + return fmt.Errorf("worker: decode stored spec of %s/%s/%s: %w", r.Node, r.VPC, r.AttachmentUID, err) + } + c := &claim{rec: r, spec: spec} + c.status = w.initialStatus(c, 0) + w.claims[c.key()] = c + w.scope(r.VPC).resyncing = true + w.lastSeen[r.Owner] = now + } + scopes := w.scopeNames() + w.mu.Unlock() + updateClaimGauge(w) + for _, s := range scopes { + w.queue.Add(s) + } + + for range w.cfg.Reconcilers { + go w.runReconciler(ctx) + } + go func() { + <-ctx.Done() + w.queue.ShutDown() + }() + go w.every(ctx, w.cfg.RenewInterval, w.ExpireLeases) + go w.every(ctx, w.cfg.ResyncInterval, func() { w.Changed("") }) + return nil +} + +// Shutdown makes every later mutating call fail with UNAVAILABLE and +// ShuttingDown. Forwarding continues: the datapath does not call Worker. +func (w *Worker) Shutdown() { w.shutting.Store(true) } + +// Changed requeues the scope of vpc, or every scope when vpc is "". Trusted +// source watches call it; duplicate calls coalesce in the queue. +func (w *Worker) Changed(vpc string) { + w.mu.Lock() + var names []string + if vpc == "" { + names = w.scopeNames() + } else if _, ok := w.scopes[vpc]; ok { + names = []string{vpc} + } + w.mu.Unlock() + for _, n := range names { + w.queue.Add(n) + } +} + +func (w *Worker) every(ctx context.Context, d time.Duration, fn func()) { + t := time.NewTicker(d) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + fn() + } + } +} + +// scope returns the state of a VPC scope, creating it. w.mu must be held. +func (w *Worker) scope(vpc string) *scopeState { + s, ok := w.scopes[vpc] + if !ok { + s = &scopeState{} + w.scopes[vpc] = s + } + return s +} + +// scopeNames lists scopes that have claims. w.mu must be held. +func (w *Worker) scopeNames() []string { + out := make([]string, 0, len(w.scopes)) + for n := range w.scopes { + out = append(out, n) + } + return out +} + +// keyLock serializes calls on one claim key. +func (w *Worker) keyLock(k module.ClaimKey) *sync.Mutex { + h := uint32(2166136261) + for _, b := range []byte(k.String()) { + h = (h ^ uint32(b)) * 16777619 + } + return &w.keyLocks[h%uint32(len(w.keyLocks))] +} + +// nextRevision returns the next status revision. w.mu must be held. +func (w *Worker) nextRevision() uint64 { + w.revision++ + return w.revision +} diff --git a/internal/worker/worker_test.go b/internal/worker/worker_test.go new file mode 100644 index 00000000..35c36468 --- /dev/null +++ b/internal/worker/worker_test.go @@ -0,0 +1,493 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package worker + +import ( + "context" + "fmt" + "maps" + "slices" + "sync" + "testing" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + "go.datum.net/galactic/internal/worker/module" + "go.datum.net/galactic/internal/worker/module/fake" + "go.datum.net/galactic/internal/worker/refusal" + "go.datum.net/galactic/internal/worker/sources" +) + +// TestLifecycleResolvesLocalAndFabric requests a claim with a destination +// on this node and one on another, checks each reaches its module, then +// releases it and checks every row and the record are gone. +func TestLifecycleResolvesLocalAndFabric(t *testing.T) { + e := newEnv(t) + e.start() + k := mkey(vpcA, "svc-a") + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1", "guest-a2"))) + st := e.waitReady(k, 1) + modes := map[string]workerv1alpha1.Mode{} + for _, d := range st.GetDestinations() { + modes[module.DestinationID(d.GetDestination())] = d.GetMode() + } + if modes["attachment/guest-a1"] != workerv1alpha1.Mode_MODE_LOCAL || + modes["attachment/guest-a2"] != workerv1alpha1.Mode_MODE_FABRIC { + t.Fatalf("modes = %v, want guest-a1 Local and guest-a2 Fabric", modes) + } + if got := rowIDs(e.host.Rows(vpcA)); !slices.Equal(got, + []string{"svc-a→attachment/guest-a1", "svc-a→attachment/guest-a2"}) { + t.Fatalf("rows = %v", got) + } + + e.mustOK(e.release(natSA, key(vpcA, "svc-a"), 1)) + e.waitGone(k) + if rows := e.host.Rows(vpcA); len(rows) != 0 { + t.Fatalf("rows after release = %v", rowIDs(rows)) + } + if recs, _ := e.store.List(); len(recs) != 0 { + t.Fatalf("records after release = %v", recs) + } + // A retried release of a claim that no longer exists succeeds. + e.mustOK(e.release(natSA, key(vpcA, "svc-a"), 1)) +} + +// TestDuplicateRequestChangesNothing repeats a request whose response was +// lost: the same generation and spec returns the current status and writes +// nothing to the store, the fence or the host. +func TestDuplicateRequestChangesNothing(t *testing.T) { + e := newEnv(t) + e.start() + k := mkey(vpcA, "svc-a") + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 4, veth("guest-a1"))) + e.waitReady(k, 4) + e.idle() + before, rows := e.persisted(), e.host.Rows(vpcA) + writes, plans := e.local.Writes.Load(), e.local.PlanCalls.Load() + + resp, err := e.api.Request(e.ctx(natSA), &workerv1alpha1.RequestRequest{Key: key(vpcA, "svc-a"), + Generation: 4, Spec: veth("guest-a1")}) + e.mustOK(err) + if !isTrue(resp.GetStatus(), workerv1alpha1.ConditionReady) || resp.GetStatus().GetObservedGeneration() != 4 { + t.Fatalf("duplicate returned %v, want the current Ready status", resp.GetStatus()) + } + wantRefusal(t, e.request(natSA, key(vpcA, "svc-a"), 4, veth("guest-a3")), + codes.FailedPrecondition, workerv1alpha1.ReasonGenerationConflict) + wantRefusal(t, e.request(natSA, key(vpcA, "svc-a"), 5, veth("guest-a1")), + codes.FailedPrecondition, workerv1alpha1.ReasonClaimExists) + e.idle() + + if after := e.persisted(); !maps.Equal(before, after) { + t.Fatalf("store or fence changed:\nbefore %v\nafter %v", before, after) + } + if !slices.Equal(rows, e.host.Rows(vpcA)) || e.local.Writes.Load() != writes || e.local.PlanCalls.Load() != plans { + t.Fatal("host state changed or a reconcile ran for a duplicate") + } +} + +// TestOutOfOrderRequests delivers delayed calls after newer ones: each is +// refused with the current generation and changes nothing. +func TestOutOfOrderRequests(t *testing.T) { + e := newEnv(t) + e.start() + k := mkey(vpcA, "svc-a") + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1"))) + e.mustOK(e.update(natSA, key(vpcA, "svc-a"), 3, veth("guest-a1", "guest-a3"))) + e.waitReady(k, 3) + e.idle() + before := e.persisted() + + err := e.update(natSA, key(vpcA, "svc-a"), 2, veth("guest-a1")) + wantRefusal(t, err, codes.Aborted, workerv1alpha1.ReasonStaleGeneration) + if md := errorMD(err); md[workerv1alpha1.MetadataCurrentGeneration] != "3" { + t.Fatalf("stale update metadata = %v, want current generation 3", md) + } + wantRefusal(t, e.release(natSA, key(vpcA, "svc-a"), 2), codes.Aborted, workerv1alpha1.ReasonStaleGeneration) + wantRefusal(t, e.release(natSA, key(vpcA, "svc-a"), 4), codes.FailedPrecondition, + workerv1alpha1.ReasonGenerationConflict) + wantRefusal(t, e.update(natSA, key(vpcA, "svc-a"), 3, veth("guest-a1")), codes.FailedPrecondition, + workerv1alpha1.ReasonGenerationConflict) + e.mustOK(e.update(natSA, key(vpcA, "svc-a"), 3, veth("guest-a1", "guest-a3"))) + e.idle() + if !maps.Equal(before, e.persisted()) { + t.Fatal("an out-of-order call changed durable state") + } + if got := rowIDs(e.host.Rows(vpcA)); len(got) != 2 { + t.Fatalf("rows = %v, want generation 3's two destinations", got) + } +} + +func errorMD(err error) map[string]string { + for _, d := range status.Convert(err).Details() { + if info, ok := d.(interface{ GetMetadata() map[string]string }); ok { + return info.GetMetadata() + } + } + return nil +} + +// TestConcurrentClients runs several callers at once, each with its own +// claims, and two callers racing for one attachment. Exactly one of the +// racers wins and every claim converges. +func TestConcurrentClients(t *testing.T) { + e := newEnv(t) + for i := range 8 { + e.src.SetAttachment(srcAttachment(fmt.Sprintf("svc-%d", i), vpcA)) + } + e.start() + var wg sync.WaitGroup + errs := make(chan error, 64) + for i := range 8 { + caller := natSA + if i%2 == 1 { + caller = otherSA + } + wg.Add(1) + go func() { + defer wg.Done() + k := key(vpcA, fmt.Sprintf("svc-%d", i)) + for gen := uint64(1); gen <= 3; gen++ { + var err error + if gen == 1 { + err = e.request(caller, k, gen, veth("guest-a1")) + } else { + err = e.update(caller, k, gen, veth("guest-a1", "guest-a2")) + } + if err != nil { + errs <- err + } + } + }() + } + results := make(chan error, 2) + for _, caller := range []string{natSA, otherSA} { + wg.Add(1) + go func() { + defer wg.Done() + results <- e.request(caller, key(vpcA, "svc-a"), 1, veth("guest-a3")) + }() + } + wg.Wait() + close(errs) + close(results) + for err := range errs { + t.Errorf("concurrent call failed: %v", err) + } + var won, lost int + for err := range results { + switch { + case err == nil: + won++ + case refusal.Is(err, codes.PermissionDenied, workerv1alpha1.ReasonAttachmentClaimedByAnotherCaller): + lost++ + default: + t.Errorf("racing request: %v", err) + } + } + if won != 1 || lost != 1 { + t.Fatalf("racing requests: %d won, %d refused; want exactly one of each", won, lost) + } + for i := range 8 { + e.waitReady(mkey(vpcA, fmt.Sprintf("svc-%d", i)), 3) + } + e.waitReady(mkey(vpcA, "svc-a"), 1) + if got := len(e.host.Rows(vpcA)); got != 8*2+1 { + t.Fatalf("rows = %d, want %d", got, 8*2+1) + } +} + +// TestCrossVPCRefusedWithoutMutation is the unprivileged twin of the +// zero-mutation root test: a destination in another VPC is refused before +// anything is recorded, fenced or planned. +func TestCrossVPCRefusedWithoutMutation(t *testing.T) { + e := newEnv(t) + e.start() + k := mkey(vpcA, "svc-a") + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1"))) + e.waitReady(k, 1) + e.idle() + before, rows, st := e.persisted(), e.host.Rows(vpcA), e.status(k) + plans, applies := e.local.PlanCalls.Load(), e.local.ApplyCalls.Load() + + // guest-b1 has the same guest addresses as guest-a1 in another VPC. + wantRefusal(t, e.update(natSA, key(vpcA, "svc-a"), 2, veth("guest-a1", "guest-b1")), + codes.PermissionDenied, workerv1alpha1.ReasonVPCNotAuthorized) + wantRefusal(t, e.request(natSA, key(vpcA, "svc-a2"), 1, veth("guest-b1")), + codes.PermissionDenied, workerv1alpha1.ReasonVPCNotAuthorized) + // A claim key naming VPC B on an attachment in VPC A is refused too. + wantRefusal(t, e.request(natSA, key(vpcB, "svc-a2"), 1, veth("guest-b1")), + codes.PermissionDenied, workerv1alpha1.ReasonVPCNotAuthorized) + e.idle() + + if !maps.Equal(before, e.persisted()) { + t.Fatal("a refused cross-VPC claim changed the store or the fence") + } + if !slices.Equal(rows, e.host.Rows(vpcA)) || len(e.host.Rows(vpcB)) != 0 { + t.Fatal("a refused cross-VPC claim changed host state") + } + if e.local.PlanCalls.Load() != plans || e.local.ApplyCalls.Load() != applies { + t.Fatal("a refused cross-VPC claim reached a module") + } + if !proto.Equal(st, e.status(k)) { + t.Fatal("the existing generation's status changed") + } +} + +// TestNonOwnerRefused checks that only the owner may release or update a +// claim, and that a second caller cannot claim the same attachment. +func TestNonOwnerRefused(t *testing.T) { + e := newEnv(t) + e.start() + k := mkey(vpcA, "svc-a") + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1"))) + e.waitReady(k, 1) + e.idle() + before, rows := e.persisted(), e.host.Rows(vpcA) + + wantRefusal(t, e.release(otherSA, key(vpcA, "svc-a"), 1), codes.PermissionDenied, workerv1alpha1.ReasonNotOwner) + wantRefusal(t, e.update(otherSA, key(vpcA, "svc-a"), 2, veth("guest-a1")), codes.PermissionDenied, + workerv1alpha1.ReasonNotOwner) + wantRefusal(t, e.request(otherSA, key(vpcA, "svc-a"), 1, veth("guest-a1")), codes.PermissionDenied, + workerv1alpha1.ReasonAttachmentClaimedByAnotherCaller) + e.idle() + if !maps.Equal(before, e.persisted()) || !slices.Equal(rows, e.host.Rows(vpcA)) { + t.Fatal("a non-owner call changed state") + } + if st := e.status(k); !isTrue(st, workerv1alpha1.ConditionReady) { + t.Fatalf("claim status after non-owner calls = %v", st) + } +} + +// TestObserveDeliversCompleteSnapshots follows one claim over a stream and +// checks snapshots arrive in revision order and end with the release. +func TestObserveDeliversCompleteSnapshots(t *testing.T) { + e := newEnv(t) + e.start() + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1"))) + stream, err := e.api.Observe(e.ctx(natSA), &workerv1alpha1.ObserveRequest{Key: key(vpcA, "svc-a")}) + e.mustOK(err) + var last uint64 + sawReady := false + for { + r, err := stream.Recv() + e.mustOK(err) + st := r.GetStatus() + if st.GetRevision() <= last { + t.Fatalf("revision %d after %d", st.GetRevision(), last) + } + last = st.GetRevision() + if len(st.GetConditions()) != 5 { + t.Fatalf("snapshot is not complete: %v", st) + } + if isTrue(st, workerv1alpha1.ConditionReady) && !sawReady { + sawReady = true + e.mustOK(e.release(natSA, key(vpcA, "svc-a"), 1)) + } + if st.GetReleased() { + break + } + } + if !sawReady { + t.Fatal("never observed Ready") + } + // Another caller's stream of all its claims receives nothing of this + // claim, and a stream naming it is refused. + ctx, cancel := context.WithTimeout(e.ctx(otherSA), 300*time.Millisecond) + defer cancel() + all, err := e.api.Observe(ctx, &workerv1alpha1.ObserveRequest{}) + e.mustOK(err) + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 2, veth("guest-a1"))) + if r, err := all.Recv(); status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("another caller's stream got %v, %v; want nothing until the deadline", r, err) + } + named, err := e.api.Observe(e.ctx(otherSA), &workerv1alpha1.ObserveRequest{Key: key(vpcA, "svc-a")}) + e.mustOK(err) + if _, err := named.Recv(); !refusal.Is(err, codes.PermissionDenied, workerv1alpha1.ReasonNotOwner) { + t.Fatalf("observing another caller's claim: %v, want NotOwner", err) + } +} + +// TestSessionsAndValidation covers the refusals that need no claim state. +func TestSessionsAndValidation(t *testing.T) { + e := newEnv(t) + e.start() + noSession := metadata.AppendToOutgoingContext(t.Context(), callerMD, natSA) + _, err := e.api.Request(noSession, &workerv1alpha1.RequestRequest{Key: key(vpcA, "svc-a"), Generation: 1, + Spec: veth("guest-a1")}) + wantRefusal(t, err, codes.FailedPrecondition, workerv1alpha1.ReasonNoSession) + + _, err = e.api.GetCapabilities(noSession, &workerv1alpha1.GetCapabilitiesRequest{Versions: []string{"v9"}}) + wantRefusal(t, err, codes.FailedPrecondition, workerv1alpha1.ReasonUnsupportedVersion) + + _, err = e.api.GetCapabilities(t.Context(), versions()) + wantRefusal(t, err, codes.Unauthenticated, workerv1alpha1.ReasonNoIdentity) + + wrong := &workerv1alpha1.ClaimKey{Node: otherNode, Vpc: vpcA, AttachmentUid: "svc-a"} + wantRefusal(t, e.request(natSA, wrong, 1, veth("guest-a1")), codes.InvalidArgument, workerv1alpha1.ReasonWrongNode) + for name, s := range map[string]*workerv1alpha1.ClaimSpec{ + "no handoff": spec(workerv1alpha1.Handoff_HANDOFF_UNSPECIFIED, "guest-a1"), + "no destinations": veth(), + "duplicate": veth("guest-a1", "guest-a1"), + "empty target": {Handoff: workerv1alpha1.Handoff_HANDOFF_VETH_PAIR, + Destinations: []*workerv1alpha1.Destination{{}}}, + } { + err := e.request(natSA, key(vpcA, "svc-a"), 1, s) + if !refusal.Is(err, codes.InvalidArgument, workerv1alpha1.ReasonMalformedSpec) { + t.Errorf("%s: got %v, want MalformedSpec", name, err) + } + } + wantRefusal(t, e.request(natSA, key(vpcA, "svc-a"), 0, veth("guest-a1")), codes.InvalidArgument, + workerv1alpha1.ReasonMalformedSpec) + wantRefusal(t, e.update(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1")), codes.FailedPrecondition, + workerv1alpha1.ReasonClaimNotFound) + // The gateway may use tap only; a veth pair claim is refused. + wantRefusal(t, e.request(gwSA, key(vpcA, "svc-a"), 1, veth("guest-a1")), codes.PermissionDenied, + workerv1alpha1.ReasonCapabilityNotAllowed) + // A tap claim with a remote destination needs fabric.srv6, which the + // gateway lacks. + wantRefusal(t, e.request(gwSA, key(vpcA, "svc-a"), 1, spec(workerv1alpha1.Handoff_HANDOFF_TAP, "guest-a2")), + codes.PermissionDenied, workerv1alpha1.ReasonCapabilityNotAllowed) + if recs, _ := e.store.List(); len(recs) != 0 { + t.Fatalf("refused calls recorded %v", recs) + } + + // A GetCapabilities that carries a live session renews it. + ctx := e.ctx(natSA) + resp, err := e.api.GetCapabilities(ctx, versions()) + e.mustOK(err) + e.mu.Lock() + same := resp.GetSessionId() == e.sessions[natSA] + e.mu.Unlock() + if !same { + t.Fatal("GetCapabilities with a live session started a new one") + } +} + +// TestDestinationResolution covers destinations that do not resolve: shard +// and backend kinds, which later Tasks resolve, and a mode preference the +// destination cannot meet. None falls back to the other mode. +func TestDestinationResolution(t *testing.T) { + e := newEnv(t) + e.start() + s := veth("guest-a1") + s.Destinations = append(s.Destinations, + &workerv1alpha1.Destination{Target: &workerv1alpha1.Destination_Shard{ + Shard: &workerv1alpha1.ShardRef{Group: "g", Index: 1}}}, + &workerv1alpha1.Destination{Target: &workerv1alpha1.Destination_Backend{ + Backend: &workerv1alpha1.BackendRef{Node: otherNode, Slot: 2}}}) + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, s)) + st := e.waitObserved(mkey(vpcA, "svc-a")) + if c := condOf(st, workerv1alpha1.ConditionResolved); c.GetStatus() != condFalse || + c.GetReason() != workerv1alpha1.ReasonDestinationKindUnsupported { + t.Fatalf("Resolved = %v, want False/DestinationKindUnsupported", c) + } + if !isTrue(st, workerv1alpha1.ConditionProgrammed) || isTrue(st, workerv1alpha1.ConditionReady) { + t.Fatalf("status = %v, want the resolved destination programmed and the claim not Ready", st) + } + + local := veth("guest-a2") + local.Mode = workerv1alpha1.Mode_MODE_LOCAL + e.mustOK(e.request(natSA, key(vpcA, "svc-a2"), 1, local)) + st = e.waitObserved(mkey(vpcA, "svc-a2")) + if c := condOf(st, workerv1alpha1.ConditionResolved); c.GetReason() != workerv1alpha1.ReasonModeUnavailable { + t.Fatalf("Resolved = %v, want ModeUnavailable", c) + } + for _, r := range e.host.Rows(vpcA) { + if r.Key.AttachmentUID == "svc-a2" { + t.Fatalf("a destination that cannot meet its mode was programmed: %v", r) + } + } +} + +// TestUnavailableModuleReportsReason accepts claims that need a module +// that is not running or a capability the platform excludes, and reports +// Programmed=False with the capability's reason. +func TestUnavailableModuleReportsReason(t *testing.T) { + e := newEnv(t) + var local *fake.Module + e.modules = func(h *fake.Host) []module.Module { + local = fake.New("local", workerv1alpha1.Mode_MODE_LOCAL, h) + local.SetCapability(module.Capability{Name: workerv1alpha1.CapabilityLocalTap, + Reason: workerv1alpha1.ReasonHookUnsupported, Message: "XDP mode excluded"}) + return []module.Module{local} + } + e.start() + resp, err := e.api.GetCapabilities(metadata.AppendToOutgoingContext(t.Context(), callerMD, natSA), + versions()) + e.mustOK(err) + got := map[string]string{} + for _, c := range resp.GetCapabilities() { + got[c.GetName()] = c.GetState().String() + "/" + c.GetReason() + } + if got[workerv1alpha1.CapabilityFabricSRv6] != "CAPABILITY_STATE_UNAVAILABLE/ModuleNotRunning" || + got[workerv1alpha1.CapabilityLocalTap] != "CAPABILITY_STATE_UNAVAILABLE/HookUnsupported" || + got[workerv1alpha1.CapabilityLocalVethPair] != "CAPABILITY_STATE_AVAILABLE/" { + t.Fatalf("capabilities = %v", got) + } + + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1", "guest-a2"))) + st := e.waitObserved(mkey(vpcA, "svc-a")) + if c := condOf(st, workerv1alpha1.ConditionProgrammed); c.GetStatus() != condFalse || + c.GetReason() != workerv1alpha1.ReasonModuleNotRunning { + t.Fatalf("Programmed = %v, want False/ModuleNotRunning", c) + } + if got := rowIDs(e.host.Rows(vpcA)); !slices.Equal(got, []string{"svc-a→attachment/guest-a1"}) { + t.Fatalf("rows = %v, want only the Local destination", got) + } + + e.mustOK(e.request(gwSA, key(vpcA, "svc-a2"), 1, spec(workerv1alpha1.Handoff_HANDOFF_TAP, "guest-a1"))) + st = e.waitObserved(mkey(vpcA, "svc-a2")) + if c := condOf(st, workerv1alpha1.ConditionProgrammed); c.GetReason() != workerv1alpha1.ReasonHookUnsupported { + t.Fatalf("Programmed = %v, want HookUnsupported", c) + } +} + +// TestObservedReadinessSeparateFromProgrammed keeps a programmed +// destination not Ready while the module observes its handoff missing. +func TestObservedReadinessSeparateFromProgrammed(t *testing.T) { + e := newEnv(t) + e.start() + e.local.SetReady(func(module.ClaimKey, string) (bool, string) { + return false, workerv1alpha1.ReasonHandoffInterfaceMissing + }) + k := mkey(vpcA, "svc-a") + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1"))) + st := e.waitObserved(k) + if !isTrue(st, workerv1alpha1.ConditionProgrammed) { + t.Fatalf("Programmed = %v, want True", condOf(st, workerv1alpha1.ConditionProgrammed)) + } + if c := condOf(st, workerv1alpha1.ConditionReady); c.GetStatus() != condFalse || + c.GetReason() != workerv1alpha1.ReasonHandoffInterfaceMissing { + t.Fatalf("Ready = %v, want False/HandoffInterfaceMissing", c) + } + e.local.SetReady(nil) + e.w.Changed(vpcA) + e.waitReady(k, 1) +} + +// TestClaimLimit refuses a caller's claim beyond the limit. +func TestClaimLimit(t *testing.T) { + e := newEnv(t) + e.cfg.ClaimLimit = 1 + e.start() + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1"))) + wantRefusal(t, e.request(natSA, key(vpcA, "svc-a2"), 1, veth("guest-a1")), codes.ResourceExhausted, + workerv1alpha1.ReasonClaimLimit) + // Another caller has its own limit. + e.mustOK(e.request(otherSA, key(vpcA, "svc-a2"), 1, veth("guest-a1"))) +} + +func versions() *workerv1alpha1.GetCapabilitiesRequest { + return &workerv1alpha1.GetCapabilitiesRequest{Versions: []string{workerv1alpha1.Version}} +} + +func srcAttachment(uid, vpc string) sources.Attachment { + return sources.Attachment{UID: uid, VPC: vpc, Node: node, Kind: sources.KindVPCAttachment, Admitted: true} +} diff --git a/internal/worker/zeromutation_root_test.go b/internal/worker/zeromutation_root_test.go new file mode 100644 index 00000000..3b329716 --- /dev/null +++ b/internal/worker/zeromutation_root_test.go @@ -0,0 +1,243 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package worker + +import ( + "encoding/binary" + "fmt" + "hash/fnv" + "net" + "os" + "path/filepath" + "slices" + "sort" + "strings" + "testing" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/rlimit" + "github.com/containernetworking/plugins/pkg/ns" + "github.com/containernetworking/plugins/pkg/testutils" + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" + "google.golang.org/grpc/codes" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + "go.datum.net/galactic/internal/worker/module" + "go.datum.net/galactic/internal/worker/module/fake" +) + +func requireRoot(t *testing.T) { + t.Helper() + if os.Getuid() != 0 { + t.Skip("skipping: requires root (CAP_SYS_ADMIN, CAP_NET_ADMIN, CAP_BPF)") + } +} + +const rowTable = 1000 + +// netKernel mirrors each fake module row into real kernel state in a test +// network namespace: a dummy link, a route in a VRF-style table, a policy +// rule and a row in a pinned BPF map. +type netKernel struct { + netns ns.NetNS + rows *ebpf.Map +} + +func rowIndex(scope module.Scope, r fake.Row) uint32 { + h := fnv.New32a() + _, _ = h.Write([]byte(scope.VPC + "/" + r.Key.String() + "/" + r.ID)) + return h.Sum32()%4000 + 1 +} + +func rowPrefix(i uint32) *net.IPNet { + ip := net.ParseIP("2001:db8:ffff::").To16() + binary.BigEndian.PutUint16(ip[6:], uint16(i)) + return &net.IPNet{IP: ip, Mask: net.CIDRMask(64, 128)} +} + +func (k *netKernel) Add(scope module.Scope, r fake.Row) error { + i := rowIndex(scope, r) + if err := k.rows.Put(i, r.Generation); err != nil { + return err + } + return k.netns.Do(func(ns.NetNS) error { + name := fmt.Sprintf("wrow%d", i) + if _, err := netlink.LinkByName(name); err != nil { + if err := netlink.LinkAdd(&netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: name}}); err != nil { + return err + } + } + l, err := netlink.LinkByName(name) + if err != nil { + return err + } + if err := netlink.LinkSetUp(l); err != nil { + return err + } + if err := netlink.RouteReplace(&netlink.Route{LinkIndex: l.Attrs().Index, Table: rowTable, + Dst: rowPrefix(i)}); err != nil { + return err + } + rule := netlink.NewRule() + rule.Family, rule.Table, rule.Priority, rule.Dst = unix.AF_INET6, rowTable, int(i)+100, rowPrefix(i) + if err := netlink.RuleAdd(rule); err != nil && !os.IsExist(err) { + return err + } + return nil + }) +} + +func (k *netKernel) Remove(scope module.Scope, r fake.Row) error { + i := rowIndex(scope, r) + if err := k.rows.Delete(i); err != nil { + return err + } + return k.netns.Do(func(ns.NetNS) error { + rule := netlink.NewRule() + rule.Family, rule.Table, rule.Priority, rule.Dst = unix.AF_INET6, rowTable, int(i)+100, rowPrefix(i) + _ = netlink.RuleDel(rule) + if l, err := netlink.LinkByName(fmt.Sprintf("wrow%d", i)); err == nil { + return netlink.LinkDel(l) + } + return nil + }) +} + +// snapshot renders every link, route, rule and pinned map row the test can +// see in the namespace and the bpffs, sorted. +func snapshot(t *testing.T, netns ns.NetNS, pin string) []string { + t.Helper() + var out []string + err := netns.Do(func(ns.NetNS) error { + links, err := netlink.LinkList() + if err != nil { + return err + } + for _, l := range links { + a := l.Attrs() + out = append(out, fmt.Sprintf("link %d %s %s mtu=%d flags=%s", a.Index, a.Name, l.Type(), a.MTU, a.Flags)) + } + routes, err := netlink.RouteListFiltered(netlink.FAMILY_ALL, &netlink.Route{Table: unix.RT_TABLE_UNSPEC}, + netlink.RT_FILTER_TABLE) + if err != nil { + return err + } + for _, r := range routes { + out = append(out, fmt.Sprintf("route table=%d dst=%s dev=%d type=%d", r.Table, r.Dst, r.LinkIndex, r.Type)) + } + rules, err := netlink.RuleList(netlink.FAMILY_ALL) + if err != nil { + return err + } + for _, r := range rules { + out = append(out, fmt.Sprintf("rule family=%d prio=%d table=%d dst=%s", r.Family, r.Priority, r.Table, r.Dst)) + } + return nil + }) + if err != nil { + t.Fatal(err) + } + m, err := ebpf.LoadPinnedMap(pin, nil) + if err != nil { + t.Fatal(err) + } + defer func() { _ = m.Close() }() + var k uint32 + var v uint64 + it := m.Iterate() + for it.Next(&k, &v) { + out = append(out, fmt.Sprintf("map %d=%d", k, v)) + } + if err := it.Err(); err != nil { + t.Fatal(err) + } + sort.Strings(out) + return out +} + +// TestCrossVPCRefusalLeavesKernelUnchanged is the acceptance test for an +// unauthorized cross-VPC claim: with a claim in VPC A programmed into real +// routes, rules, links and a pinned map, requests that would reach VPC B, +// which uses the same guest addresses, are refused, and a snapshot of the +// kernel state before and after is identical. +func TestCrossVPCRefusalLeavesKernelUnchanged(t *testing.T) { + requireRoot(t) + if err := rlimit.RemoveMemlock(); err != nil { + t.Fatal(err) + } + netns, err := testutils.NewNS() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = netns.Close() + _ = testutils.UnmountNS(netns) + }) + bpffs := t.TempDir() + if err := unix.Mount("bpf", bpffs, "bpf", 0, ""); err != nil { + t.Fatalf("mount bpffs: %v", err) + } + t.Cleanup(func() { _ = unix.Unmount(bpffs, unix.MNT_DETACH) }) + rows, err := ebpf.NewMapWithOptions(&ebpf.MapSpec{Name: "worker_rows", Type: ebpf.Hash, KeySize: 4, + ValueSize: 8, MaxEntries: 4096, Pinning: ebpf.PinByName}, ebpf.MapOptions{PinPath: bpffs}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = rows.Close() }) + pin := filepath.Join(bpffs, "worker_rows") + kernel := &netKernel{netns: netns, rows: rows} + + e := newEnv(t) + var local, fabric *fake.Module + e.modules = func(h *fake.Host) []module.Module { + local = fake.New("local", workerv1alpha1.Mode_MODE_LOCAL, h) + fabric = fake.New("fabric", workerv1alpha1.Mode_MODE_FABRIC, h) + local.Kernel, fabric.Kernel = kernel, kernel + return []module.Module{local, fabric} + } + e.start() + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth("guest-a1", "guest-a2"))) + e.waitReady(mkey(vpcA, "svc-a"), 1) + e.idle() + + before := snapshot(t, netns, pin) + if !slices.ContainsFunc(before, func(s string) bool { return strings.HasPrefix(s, "map ") }) || + !slices.ContainsFunc(before, func(s string) bool { return strings.Contains(s, fmt.Sprintf("table=%d", rowTable)) }) { + t.Fatalf("the programmed claim is not in kernel state: %v", before) + } + persisted := e.persisted() + plans, applies := local.PlanCalls.Load()+fabric.PlanCalls.Load(), local.ApplyCalls.Load()+fabric.ApplyCalls.Load() + + wantRefusal(t, e.update(natSA, key(vpcA, "svc-a"), 2, veth("guest-a1", "guest-b1")), + codes.PermissionDenied, workerv1alpha1.ReasonVPCNotAuthorized) + wantRefusal(t, e.request(natSA, key(vpcA, "svc-a2"), 1, veth("guest-b1")), + codes.PermissionDenied, workerv1alpha1.ReasonVPCNotAuthorized) + wantRefusal(t, e.request(natSA, key(vpcB, "svc-a2"), 1, veth("guest-b1")), + codes.PermissionDenied, workerv1alpha1.ReasonVPCNotAuthorized) + e.idle() + + after := snapshot(t, netns, pin) + if !slices.Equal(before, after) { + t.Fatalf("kernel state changed:\nbefore %v\nafter %v", before, after) + } + if got := e.persisted(); len(got) != len(persisted) { + t.Fatal("durable state changed") + } else { + for k, v := range persisted { + if got[k] != v { + t.Fatalf("%s changed", k) + } + } + } + if local.PlanCalls.Load()+fabric.PlanCalls.Load() != plans || + local.ApplyCalls.Load()+fabric.ApplyCalls.Load() != applies { + t.Fatal("a refused claim reached a module") + } + // The existing generation keeps running. + if st := e.status(mkey(vpcA, "svc-a")); st.GetGeneration() != 1 || !isTrue(st, workerv1alpha1.ConditionReady) { + t.Fatalf("existing claim = %v", st) + } +} From f06134183ee30d7ed8fa1dd1fed5378c5d0c5cea Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Wed, 18 Nov 2026 18:03:19 -0500 Subject: [PATCH 4/6] test(worker): Add an independent Worker client and its task target The contract needs a client that proves the published interface is enough on its own. examples/worker-client is its own module and imports only api/galactic/worker/v1alpha1 and pkg/workerclient; its module path lies outside go.datum.net/galactic, so the compiler refuses internal imports. Its lifecycle subcommand checks the negotiated version and its allowed capabilities, expects a forbidden handoff to be refused, requests, repeats the request, observes, updates, sends a stale release, releases and observes the release. task test:worker-client builds, vets and tests the module and fails if it imports any other Galactic package; test:unit runs it, so task ci does too. TestIndependentClient builds the binary and runs its lifecycle against a real Worker on a unix socket with the real authentication chain, as a ServiceAccount allowed local.veth_pair only. It also checks that another process holding the same token is refused. Co-Authored-By: Claude --- Taskfile.yaml | 19 +- examples/worker-client/README.md | 53 ++++ examples/worker-client/go.mod | 23 ++ examples/worker-client/go.sum | 18 ++ .../internal/lifecycle/lifecycle.go | 170 ++++++++++++ .../internal/lifecycle/lifecycle_test.go | 249 ++++++++++++++++++ examples/worker-client/main.go | 124 +++++++++ internal/worker/independent_client_test.go | 143 ++++++++++ 8 files changed, 797 insertions(+), 2 deletions(-) create mode 100644 examples/worker-client/README.md create mode 100644 examples/worker-client/go.mod create mode 100644 examples/worker-client/go.sum create mode 100644 examples/worker-client/internal/lifecycle/lifecycle.go create mode 100644 examples/worker-client/internal/lifecycle/lifecycle_test.go create mode 100644 examples/worker-client/main.go create mode 100644 internal/worker/independent_client_test.go diff --git a/Taskfile.yaml b/Taskfile.yaml index de160275..14a20f3c 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -53,10 +53,11 @@ tasks: deps: [golangci-lint, yamlfmt, yaml-ext, 'build:ebpf'] cmds: - '{{.GOLANGCI_LINT}} run' - # examples/edge-provider and examples/looking-glass-client are their - # own modules, which the root run skips. + # examples/edge-provider, examples/looking-glass-client and + # examples/worker-client are their own modules, which the root run skips. - cd examples/edge-provider && {{.GOLANGCI_LINT}} run --config ../../.golangci.yaml - cd examples/looking-glass-client && {{.GOLANGCI_LINT}} run --config ../../.golangci.yaml + - cd examples/worker-client && {{.GOLANGCI_LINT}} run --config ../../.golangci.yaml - '{{.YAMLFMT}} -lint' lint-fix: desc: Lint and apply fixes @@ -187,6 +188,7 @@ tasks: - bash scripts/ci.sh unittest - task: test:edge-provider - task: test:looking-glass-client + - task: test:worker-client test:edge-provider: desc: Build, vet and unit-test the reference edge provider (examples/edge-provider, its own module) dir: examples/edge-provider @@ -207,6 +209,19 @@ tasks: echo "the client imports Galactic packages beyond the published service API" >&2 exit 1 fi + test:worker-client: + desc: Build, vet and unit-test the independent Worker client (examples/worker-client, its own module) + dir: examples/worker-client + cmds: + - go build ./... + - go vet ./... + - go test -race ./... + # The client must build from the published Worker API alone. + - | + if go list -deps ./... | grep -E '^go\.datum\.net/galactic/' | grep -vxE 'go\.datum\.net/galactic/(api/galactic/worker/v1alpha1|pkg/workerclient)'; then + echo "the client imports Galactic packages beyond the published Worker API" >&2 + exit 1 + fi test:unit-root: desc: Re-run just the requireRoot(t)-gated unit tests, as root deps: ['build:ebpf'] diff --git a/examples/worker-client/README.md b/examples/worker-client/README.md new file mode 100644 index 00000000..b2e670a9 --- /dev/null +++ b/examples/worker-client/README.md @@ -0,0 +1,53 @@ +# Worker client + +An independent client of the Galactic Worker service API +([docs/worker/contract.md](../../docs/worker/contract.md)). It is built from the +published interface alone: its module imports only +`go.datum.net/galactic/api/galactic/worker/v1alpha1` and +`go.datum.net/galactic/pkg/workerclient`, and its module path lies outside +`go.datum.net/galactic`, so the compiler refuses any import of Galactic's +internal packages. `task test:worker-client` builds and tests it and fails if +it imports any other Galactic package. + +`TestIndependentClient` in `internal/worker` builds this binary and runs its +lifecycle against a real Worker on a unix socket, with the real authentication +chain and a ServiceAccount allowed `local.veth_pair` only. + +## Identity + +The client presents its pod's projected ServiceAccount token, audience +`galactic-worker`, read from `--token-file` on every call. Worker accepts it +only from a process of the pod the token is bound to, on the same node, and +only for the capabilities its allowlist grants the ServiceAccount. + +## Build + +```sh +cd examples/worker-client +CGO_ENABLED=0 go build -o worker-client . +``` + +The module builds against the API of the checkout it sits in +(`replace go.datum.net/galactic => ../..`). + +## Use + +```sh +worker-client capabilities --socket /run/galactic-worker/worker.sock + +worker-client lifecycle --node node-a --vpc --attachment \ + --destination --update-destination \ + --handoff veth_pair --forbidden-handoff tap +``` + +`lifecycle` checks that the negotiated version is `v1alpha1` and that +`--handoff` is allowed. It then: + +1. Requests the forbidden handoff and expects `PERMISSION_DENIED` with + `CapabilityNotAllowed`. +2. Requests the claim at generation 1, then repeats the request as a duplicate. +3. Observes the claim until Worker reports on generation 1. +4. Updates the claim to generation 2 and observes until Worker reports on it. +5. Sends a stale release at generation 1 and expects `ABORTED`. +6. Releases at generation 2 and observes until the release is confirmed. +7. Releases again, which succeeds because the claim no longer exists. diff --git a/examples/worker-client/go.mod b/examples/worker-client/go.mod new file mode 100644 index 00000000..d373d12e --- /dev/null +++ b/examples/worker-client/go.mod @@ -0,0 +1,23 @@ +module github.com/datum-cloud/galactic/examples/worker-client + +go 1.26.4 + +// The client is built against the Worker API in this checkout. It imports +// only go.datum.net/galactic/api/galactic/worker/v1alpha1 and +// go.datum.net/galactic/pkg/workerclient: its module path is outside +// go.datum.net/galactic, so the compiler refuses any import of the core's +// internal packages. +replace go.datum.net/galactic => ../.. + +require ( + go.datum.net/galactic v0.0.0-00010101000000-000000000000 + google.golang.org/genproto/googleapis/rpc v0.0.0-20261026165753-2ffab32bce7e + google.golang.org/grpc v1.85.0 + google.golang.org/protobuf v1.36.12 +) + +require ( + golang.org/x/net v0.62.0 // indirect + golang.org/x/sys v0.50.0 // indirect + golang.org/x/text v0.44.0 // indirect +) diff --git a/examples/worker-client/go.sum b/examples/worker-client/go.sum new file mode 100644 index 00000000..3aa6be9c --- /dev/null +++ b/examples/worker-client/go.sum @@ -0,0 +1,18 @@ +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= +golang.org/x/sys v0.50.0 h1:dkKZEFTEKaLr1sjhs0WiKqoi8Jj5nHbS3Njmx1UoMJs= +golang.org/x/sys v0.50.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.44.0 h1:YItdp/mQNGnsmrgpcdA5yQ3bnRxCfo/Tj/iByGW9GGo= +golang.org/x/text v0.44.0/go.mod h1:S4FR1waQ9LbjQTls97SyRjDaUUVkaM+cXwwuG/G+MwY= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20261026165753-2ffab32bce7e h1:w3fy/G8OrJHYexn+u5y7hFyc8YRRIr7DCevtIOwAcGY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20261026165753-2ffab32bce7e/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= +google.golang.org/grpc v1.85.0 h1:BPBCfuVRQ1qvBhKDOq9uvQllMvO59rKsYhXpp7Dd8vw= +google.golang.org/grpc v1.85.0/go.mod h1:0rQrOD58yjal/0EaVbGBANindixsCZNQPDV3VKtaKuQ= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/examples/worker-client/internal/lifecycle/lifecycle.go b/examples/worker-client/internal/lifecycle/lifecycle.go new file mode 100644 index 00000000..eac18741 --- /dev/null +++ b/examples/worker-client/internal/lifecycle/lifecycle.go @@ -0,0 +1,170 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package lifecycle drives one claim through the documented Worker +// interface: request, observe, update, observe, release, observe the +// release. It also checks that a capability the caller is not allowed is +// refused, so the client proves it works with only its advertised +// privileges. +package lifecycle + +import ( + "context" + "errors" + "fmt" + "io" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + "go.datum.net/galactic/pkg/workerclient" +) + +// Plan is one lifecycle run. +type Plan struct { + Key *workerv1alpha1.ClaimKey + // Handoff is the claim's handoff; its capability must be allowed. + Handoff workerv1alpha1.Handoff + // Destinations and UpdateDestinations are attachment UIDs for the + // request and the update. + Destinations []string + UpdateDestinations []string + // Forbidden, when set, is a handoff whose capability the caller is not + // allowed. Requesting it must be refused with CapabilityNotAllowed. + Forbidden workerv1alpha1.Handoff +} + +// Logf reports progress. +type Logf func(format string, args ...any) + +func spec(h workerv1alpha1.Handoff, uids []string) *workerv1alpha1.ClaimSpec { + s := &workerv1alpha1.ClaimSpec{Handoff: h, Mode: workerv1alpha1.Mode_MODE_AUTO} + for _, u := range uids { + s.Destinations = append(s.Destinations, &workerv1alpha1.Destination{ + Target: &workerv1alpha1.Destination_Attachment{Attachment: &workerv1alpha1.AttachmentRef{Uid: u}}, + }) + } + return s +} + +// CheckCapabilities verifies the session's version and that the plan's +// handoff is allowed and the forbidden one is not, as GetCapabilities +// reports them. +func CheckCapabilities(c *workerclient.Client, p Plan) error { + caps := c.Capabilities() + if caps.GetVersion() != workerv1alpha1.Version { + return fmt.Errorf("negotiated version %q, want %q", caps.GetVersion(), workerv1alpha1.Version) + } + allowed := map[string]bool{} + for _, cp := range caps.GetCapabilities() { + allowed[cp.GetName()] = cp.GetAllowed() + } + if name := workerv1alpha1.HandoffCapability(p.Handoff); !allowed[name] { + return fmt.Errorf("capability %s is not allowed for this caller", name) + } + if p.Forbidden != workerv1alpha1.Handoff_HANDOFF_UNSPECIFIED { + if name := workerv1alpha1.HandoffCapability(p.Forbidden); allowed[name] { + return fmt.Errorf("capability %s is allowed, but the plan expects it refused", name) + } + } + return nil +} + +// Run drives the lifecycle and returns the first deviation from the +// contract. +func Run(ctx context.Context, c *workerclient.Client, p Plan, logf Logf) error { + if err := CheckCapabilities(c, p); err != nil { + return err + } + if p.Forbidden != workerv1alpha1.Handoff_HANDOFF_UNSPECIFIED { + _, err := c.Request(ctx, p.Key, 1, spec(p.Forbidden, p.Destinations)) + if status.Code(err) != codes.PermissionDenied || + workerclient.Reason(err) != workerv1alpha1.ReasonCapabilityNotAllowed { + return fmt.Errorf("request with a forbidden capability: got %w, want PERMISSION_DENIED/%s", + err, workerv1alpha1.ReasonCapabilityNotAllowed) + } + logf("forbidden capability refused: %s", workerclient.Reason(err)) + } + + obsCtx, cancel := context.WithCancel(ctx) + defer cancel() + + st, err := c.Request(ctx, p.Key, 1, spec(p.Handoff, p.Destinations)) + if err != nil { + return fmt.Errorf("request: %w", err) + } + logf("requested generation %d", st.GetGeneration()) + // A repeated request with the same generation and spec is a duplicate. + if _, err := c.Request(ctx, p.Key, 1, spec(p.Handoff, p.Destinations)); err != nil { + return fmt.Errorf("duplicate request: %w", err) + } + + obs, err := c.Observe(obsCtx, p.Key) + if err != nil { + return fmt.Errorf("observe: %w", err) + } + if err := waitFor(obs, logf, func(s *workerv1alpha1.ClaimStatus) bool { + return s.GetObservedGeneration() >= 1 + }); err != nil { + return fmt.Errorf("observe generation 1: %w", err) + } + + if _, err := c.Update(ctx, p.Key, 2, spec(p.Handoff, p.UpdateDestinations)); err != nil { + return fmt.Errorf("update: %w", err) + } + logf("updated to generation 2") + if err := waitFor(obs, logf, func(s *workerv1alpha1.ClaimStatus) bool { + return s.GetObservedGeneration() >= 2 + }); err != nil { + return fmt.Errorf("observe generation 2: %w", err) + } + + // A release at the replaced generation is stale and changes nothing. + if _, err := c.Release(ctx, p.Key, 1); status.Code(err) != codes.Aborted { + return fmt.Errorf("stale release: got %w, want ABORTED/%s", err, workerv1alpha1.ReasonStaleGeneration) + } + if _, err := c.Release(ctx, p.Key, 2); err != nil { + return fmt.Errorf("release: %w", err) + } + logf("released generation 2") + if err := waitFor(obs, logf, func(s *workerv1alpha1.ClaimStatus) bool { return s.GetReleased() }); err != nil { + return fmt.Errorf("observe release: %w", err) + } + // Releasing a claim that no longer exists succeeds, so a retry is safe. + if _, err := c.Release(ctx, p.Key, 2); err != nil { + return fmt.Errorf("repeated release: %w", err) + } + return nil +} + +// waitFor receives snapshots until done holds. +func waitFor(obs *workerclient.Observer, logf Logf, done func(*workerv1alpha1.ClaimStatus) bool) error { + for { + s, err := obs.Recv() + if errors.Is(err, io.EOF) { + return errors.New("stream closed") + } + if err != nil { + return err + } + logf("status generation=%d observed=%d released=%t conditions=%s", + s.GetGeneration(), s.GetObservedGeneration(), s.GetReleased(), conditions(s)) + if done(s) { + return nil + } + } +} + +func conditions(s *workerv1alpha1.ClaimStatus) string { + out := "" + for _, c := range s.GetConditions() { + if out != "" { + out += "," + } + out += fmt.Sprintf("%s=%s/%s", c.GetType(), + c.GetStatus().String()[len("CONDITION_STATUS_"):], c.GetReason()) + } + return out +} diff --git a/examples/worker-client/internal/lifecycle/lifecycle_test.go b/examples/worker-client/internal/lifecycle/lifecycle_test.go new file mode 100644 index 00000000..c8b91446 --- /dev/null +++ b/examples/worker-client/internal/lifecycle/lifecycle_test.go @@ -0,0 +1,249 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package lifecycle + +import ( + "context" + "net" + "path/filepath" + "strconv" + "sync" + "testing" + "time" + + "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/durationpb" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + "go.datum.net/galactic/pkg/workerclient" +) + +// fake is a contract-shaped Worker: sessions, generations and statuses, with +// no host state. The Galactic module's own test runs this client's binary +// against the real Worker. +type fake struct { + workerv1alpha1.UnimplementedWorkerServer + + mu sync.Mutex + session string + claim *workerv1alpha1.ClaimStatus + hash []byte + revision uint64 + changed chan struct{} + calls []string +} + +func refuse(code codes.Code, reason string) error { + st, _ := status.New(code, reason).WithDetails(&errdetails.ErrorInfo{ + Reason: reason, Domain: workerv1alpha1.ErrorDomain}) + return st.Err() +} + +func (f *fake) check(ctx context.Context, needSession bool) error { + md, _ := metadata.FromIncomingContext(ctx) + if len(md.Get("authorization")) != 1 || md.Get("authorization")[0] != "Bearer test-token" { + return refuse(codes.Unauthenticated, workerv1alpha1.ReasonNoIdentity) + } + if needSession { + s := md.Get(workerv1alpha1.SessionMetadataKey) + f.mu.Lock() + defer f.mu.Unlock() + if len(s) != 1 || s[0] != f.session { + return refuse(codes.FailedPrecondition, workerv1alpha1.ReasonNoSession) + } + } + return nil +} + +func (f *fake) GetCapabilities(ctx context.Context, _ *workerv1alpha1.GetCapabilitiesRequest) ( + *workerv1alpha1.GetCapabilitiesResponse, error) { + if err := f.check(ctx, false); err != nil { + return nil, err + } + f.mu.Lock() + defer f.mu.Unlock() + f.session = "s1" + resp := &workerv1alpha1.GetCapabilitiesResponse{Version: workerv1alpha1.Version, SessionId: f.session, + Lease: durationpb.New(2 * time.Minute), RenewInterval: durationpb.New(10 * time.Second)} + for _, n := range workerv1alpha1.Capabilities { + resp.Capabilities = append(resp.Capabilities, &workerv1alpha1.Capability{Name: n, + State: workerv1alpha1.CapabilityState_CAPABILITY_STATE_AVAILABLE, + Allowed: n == workerv1alpha1.CapabilityLocalVethPair}) + } + return resp, nil +} + +func (f *fake) publish(st *workerv1alpha1.ClaimStatus) { + f.revision++ + st.Revision = f.revision + f.claim = st + select { + case f.changed <- struct{}{}: + default: + } +} + +func (f *fake) Request(ctx context.Context, req *workerv1alpha1.RequestRequest) ( + *workerv1alpha1.RequestResponse, error) { + if err := f.check(ctx, true); err != nil { + return nil, err + } + if req.GetSpec().GetHandoff() != workerv1alpha1.Handoff_HANDOFF_VETH_PAIR { + return nil, refuse(codes.PermissionDenied, workerv1alpha1.ReasonCapabilityNotAllowed) + } + h, _ := proto.Marshal(req.GetSpec()) + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, "request/"+strconv.FormatUint(req.GetGeneration(), 10)) + if f.claim != nil && !f.claim.GetReleased() { + if f.claim.GetGeneration() == req.GetGeneration() && string(h) == string(f.hash) { + return &workerv1alpha1.RequestResponse{Status: f.claim}, nil + } + return nil, refuse(codes.FailedPrecondition, workerv1alpha1.ReasonClaimExists) + } + f.hash = h + f.publish(&workerv1alpha1.ClaimStatus{Key: req.GetKey(), Generation: req.GetGeneration(), + ObservedGeneration: req.GetGeneration()}) + return &workerv1alpha1.RequestResponse{Status: f.claim}, nil +} + +func (f *fake) Update(ctx context.Context, req *workerv1alpha1.UpdateRequest) (*workerv1alpha1.UpdateResponse, error) { + if err := f.check(ctx, true); err != nil { + return nil, err + } + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, "update/"+strconv.FormatUint(req.GetGeneration(), 10)) + f.publish(&workerv1alpha1.ClaimStatus{Key: req.GetKey(), Generation: req.GetGeneration(), + ObservedGeneration: req.GetGeneration()}) + return &workerv1alpha1.UpdateResponse{Status: f.claim}, nil +} + +func (f *fake) Release(ctx context.Context, req *workerv1alpha1.ReleaseRequest) ( + *workerv1alpha1.ReleaseResponse, error) { + if err := f.check(ctx, true); err != nil { + return nil, err + } + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, "release/"+strconv.FormatUint(req.GetGeneration(), 10)) + switch { + case f.claim == nil || f.claim.GetReleased(): + return &workerv1alpha1.ReleaseResponse{}, nil + case req.GetGeneration() < f.claim.GetGeneration(): + return nil, refuse(codes.Aborted, workerv1alpha1.ReasonStaleGeneration) + } + f.publish(&workerv1alpha1.ClaimStatus{Key: req.GetKey(), Generation: req.GetGeneration(), + ObservedGeneration: req.GetGeneration(), Released: true}) + return &workerv1alpha1.ReleaseResponse{Status: f.claim}, nil +} + +func (f *fake) Observe(_ *workerv1alpha1.ObserveRequest, + s grpc.ServerStreamingServer[workerv1alpha1.ObserveResponse]) error { + if err := f.check(s.Context(), true); err != nil { + return err + } + if err := s.SendHeader(metadata.MD{}); err != nil { + return err + } + var sent uint64 + for { + f.mu.Lock() + st := f.claim + f.mu.Unlock() + if st != nil && st.GetRevision() > sent { + sent = st.GetRevision() + // Send every snapshot twice: the client drops the duplicate. + for range 2 { + if err := s.Send(&workerv1alpha1.ObserveResponse{Status: st}); err != nil { + return err + } + } + continue + } + select { + case <-s.Context().Done(): + return nil + case <-f.changed: + case <-time.After(50 * time.Millisecond): + } + } +} + +func TestLifecycleAgainstContractFake(t *testing.T) { + sock := filepath.Join(t.TempDir(), "w.sock") + lis, err := (&net.ListenConfig{}).Listen(t.Context(), "unix", sock) + if err != nil { + t.Fatal(err) + } + f := &fake{changed: make(chan struct{}, 1)} + srv := grpc.NewServer() + workerv1alpha1.RegisterWorkerServer(srv, f) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + c, err := workerclient.Dial(ctx, workerclient.Options{SocketPath: sock, + Token: func() (string, error) { return "test-token", nil }, RetryMin: time.Millisecond}) + if err != nil { + t.Fatal(err) + } + defer func() { _ = c.Close() }() + + p := Plan{ + Key: &workerv1alpha1.ClaimKey{Node: "node-a", Vpc: "vpcA", AttachmentUid: "att-1"}, + Handoff: workerv1alpha1.Handoff_HANDOFF_VETH_PAIR, + Destinations: []string{"dst-1"}, + UpdateDestinations: []string{"dst-1", "dst-2"}, + Forbidden: workerv1alpha1.Handoff_HANDOFF_TAP, + } + if err := Run(ctx, c, p, t.Logf); err != nil { + t.Fatalf("Run() = %v", err) + } + want := []string{"request/1", "request/1", "update/2", "release/1", "release/2", "release/2"} + f.mu.Lock() + defer f.mu.Unlock() + if len(f.calls) != len(want) { + t.Fatalf("calls = %v, want %v", f.calls, want) + } + for i := range want { + if f.calls[i] != want[i] { + t.Fatalf("calls = %v, want %v", f.calls, want) + } + } +} + +func TestCheckCapabilitiesRefusesWrongPrivileges(t *testing.T) { + sock := filepath.Join(t.TempDir(), "w.sock") + lis, err := (&net.ListenConfig{}).Listen(t.Context(), "unix", sock) + if err != nil { + t.Fatal(err) + } + srv := grpc.NewServer() + workerv1alpha1.RegisterWorkerServer(srv, &fake{changed: make(chan struct{}, 1)}) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + c, err := workerclient.Dial(context.Background(), workerclient.Options{SocketPath: sock, + Token: func() (string, error) { return "test-token", nil }, NoKeepAlive: true}) + if err != nil { + t.Fatal(err) + } + defer func() { _ = c.Close() }() + // The fake allows only veth_pair; a plan that needs tap must fail before + // any call. + if err := CheckCapabilities(c, Plan{Handoff: workerv1alpha1.Handoff_HANDOFF_TAP}); err == nil { + t.Fatal("CheckCapabilities() = nil, want an error for a capability the caller lacks") + } + if err := CheckCapabilities(c, Plan{Handoff: workerv1alpha1.Handoff_HANDOFF_VETH_PAIR, + Forbidden: workerv1alpha1.Handoff_HANDOFF_VETH_PAIR}); err == nil { + t.Fatal("CheckCapabilities() = nil, want an error when the forbidden capability is allowed") + } +} diff --git a/examples/worker-client/main.go b/examples/worker-client/main.go new file mode 100644 index 00000000..a5e36f17 --- /dev/null +++ b/examples/worker-client/main.go @@ -0,0 +1,124 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Command worker-client is an independent Galactic Worker client. It is +// built from the published Worker API alone +// (go.datum.net/galactic/api/galactic/worker/v1alpha1 and +// go.datum.net/galactic/pkg/workerclient) and drives one claim through the +// documented interface. +// +// Subcommands: +// +// capabilities print GetCapabilities as JSON +// lifecycle request, observe, update, observe and release one claim, +// and check that a forbidden capability is refused +package main + +import ( + "context" + "flag" + "fmt" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/datum-cloud/galactic/examples/worker-client/internal/lifecycle" + "google.golang.org/protobuf/encoding/protojson" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + "go.datum.net/galactic/pkg/workerclient" +) + +const usage = `usage: worker-client [flags] + +Every subcommand takes --socket and --token-file. Run +"worker-client -h" for its flags.` + +type list []string + +func (l *list) String() string { return strings.Join(*l, ",") } +func (l *list) Set(v string) error { *l = append(*l, v); return nil } + +func handoff(name string) (workerv1alpha1.Handoff, error) { + if name == "" { + return workerv1alpha1.Handoff_HANDOFF_UNSPECIFIED, nil + } + v, ok := workerv1alpha1.Handoff_value["HANDOFF_"+strings.ToUpper(name)] + if !ok { + return 0, fmt.Errorf("unknown handoff %q (tap, veth_pair, routed_link, socket)", name) + } + return workerv1alpha1.Handoff(v), nil +} + +func main() { + if len(os.Args) < 2 { + fmt.Fprintln(os.Stderr, usage) + os.Exit(2) + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if err := run(ctx, os.Args[1], os.Args[2:]); err != nil { + fmt.Fprintln(os.Stderr, "worker-client:", err) + os.Exit(1) + } +} + +func run(ctx context.Context, cmd string, args []string) error { + fs := flag.NewFlagSet(cmd, flag.ContinueOnError) + socket := fs.String("socket", workerv1alpha1.DefaultSocketPath, "Worker socket") + tokenFile := fs.String("token-file", workerclient.DefaultTokenFile, "projected ServiceAccount token") + timeout := fs.Duration("timeout", time.Minute, "overall deadline") + var p lifecycle.Plan + var node, vpc, attachment, hName, forbidden string + var dests, updates list + if cmd == "lifecycle" { + fs.StringVar(&node, "node", "", "claim node") + fs.StringVar(&vpc, "vpc", "", "claim VPC identifier") + fs.StringVar(&attachment, "attachment", "", "claim attachment UID") + fs.StringVar(&hName, "handoff", "veth_pair", "claim handoff") + fs.StringVar(&forbidden, "forbidden-handoff", "", "a handoff this caller must be refused") + fs.Var(&dests, "destination", "destination attachment UID for the request (repeatable)") + fs.Var(&updates, "update-destination", "destination attachment UID for the update (repeatable)") + } else if cmd != "capabilities" { + return fmt.Errorf("unknown subcommand %q\n%s", cmd, usage) + } + if err := fs.Parse(args); err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, *timeout) + defer cancel() + + c, err := workerclient.Dial(ctx, workerclient.Options{SocketPath: *socket, TokenFile: *tokenFile}) + if err != nil { + return fmt.Errorf("dial: %w", err) + } + defer func() { _ = c.Close() }() + + if cmd == "capabilities" { + b, err := protojson.Marshal(c.Capabilities()) + if err != nil { + return err + } + fmt.Println(string(b)) + return nil + } + if p.Handoff, err = handoff(hName); err != nil { + return err + } + if p.Forbidden, err = handoff(forbidden); err != nil { + return err + } + if len(updates) == 0 { + updates = dests + } + p.Key = &workerv1alpha1.ClaimKey{Node: node, Vpc: vpc, AttachmentUid: attachment} + p.Destinations, p.UpdateDestinations = dests, updates + if err := lifecycle.Run(ctx, c, p, func(f string, a ...any) { fmt.Printf(f+"\n", a...) }); err != nil { + return err + } + fmt.Println("lifecycle passed") + return nil +} diff --git a/internal/worker/independent_client_test.go b/internal/worker/independent_client_test.go new file mode 100644 index 00000000..7f86338f --- /dev/null +++ b/internal/worker/independent_client_test.go @@ -0,0 +1,143 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package worker + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + authenticationv1 "k8s.io/api/authentication/v1" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + "go.datum.net/galactic/internal/worker/auth" + "go.datum.net/galactic/internal/worker/module" +) + +type tokenReviewer map[string]authenticationv1.UserInfo + +func (r tokenReviewer) Review(_ context.Context, token string) (authenticationv1.UserInfo, error) { + if u, ok := r[token]; ok { + return u, nil + } + return authenticationv1.UserInfo{}, errors.New("not authenticated") +} + +// exePods maps a peer to a pod by its executable: the built client binary +// stands in for a process of the client's pod. +type exePods struct { + exe string + pod string +} + +func (p exePods) PodUID(pid int32) (string, error) { + exe, err := os.Readlink(fmt.Sprintf("/proc/%d/exe", pid)) + if err != nil { + return "", err + } + if exe != p.exe { + return "", fmt.Errorf("peer %d runs %s, not the client", pid, exe) + } + return p.pod, nil +} + +// TestIndependentClient builds examples/worker-client, an independent module +// that imports only the published Worker API, and runs its lifecycle +// against a real Worker on a unix socket with the real authentication +// chain: TokenReview (faked), SO_PEERCRED and the allowlist. The client's +// ServiceAccount may use local.veth_pair only; the lifecycle also checks +// that a tap claim is refused. +func TestIndependentClient(t *testing.T) { + if testing.Short() { + t.Skip("builds the example client") + } + dir, err := os.MkdirTemp("", "wk") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + bin := filepath.Join(dir, "worker-client") + build := exec.CommandContext(t.Context(), "go", "build", "-o", bin, ".") + build.Dir = filepath.Join("..", "..", "examples", "worker-client") + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("build the example client: %v\n%s", err, out) + } + bin, err = filepath.EvalSymlinks(bin) + if err != nil { + t.Fatal(err) + } + tokenFile := filepath.Join(dir, "token") + if err := os.WriteFile(tokenFile, []byte("client-token\n"), 0o600); err != nil { + t.Fatal(err) + } + + e := newEnv(t) + const sa = "galactic-system/worker-client" + authn := &auth.Authenticator{ + Reviewer: tokenReviewer{"client-token": {Username: "system:serviceaccount:galactic-system:worker-client", + Extra: map[string]authenticationv1.ExtraValue{auth.ExtraPodUID: {"pod-client"}, auth.ExtraNodeName: {node}}}}, + Pods: exePods{exe: bin, pod: "pod-client"}, + Allowlist: auth.NewAllowlist(map[string][]string{sa: {workerv1alpha1.CapabilityLocalVethPair}}), + Node: node, + } + e.start() // for the fake modules over the shared host + e.crash() + reg, err := module.NewRegistry(e.local, e.fabric) + if err != nil { + t.Fatal(err) + } + w, err := New(e.cfg, Deps{Store: e.store, Fence: e.fence, Sources: e.src, Modules: reg, Auth: authn}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if err := w.Start(ctx); err != nil { + t.Fatal(err) + } + sock := filepath.Join(dir, "worker.sock") + lis, err := Listen(ctx, sock, -1) + if err != nil { + t.Fatal(err) + } + if st, err := os.Stat(sock); err != nil || st.Mode().Perm() != 0o660 { + t.Fatalf("socket mode = %v, %v; want 0660", st.Mode(), err) + } + srv := w.NewServer() + go func() { _ = srv.Serve(lis) }() + defer srv.Stop() + + cctx, ccancel := context.WithTimeout(ctx, time.Minute) + defer ccancel() + run := exec.CommandContext(cctx, bin, "lifecycle", "--socket", sock, "--token-file", tokenFile, + "--node", node, "--vpc", vpcA, "--attachment", "svc-a", + "--destination", "guest-a1", "--update-destination", "guest-a1", "--update-destination", "guest-a3", + "--handoff", "veth_pair", "--forbidden-handoff", "tap") + out, err := run.CombinedOutput() + t.Logf("client output:\n%s", out) + if err != nil { + t.Fatalf("independent client failed: %v", err) + } + if !strings.Contains(string(out), "lifecycle passed") { + t.Fatal("the client did not complete its lifecycle") + } + if rows := e.host.Rows(vpcA); len(rows) != 0 { + t.Fatalf("rows after the client released its claim = %v", rowIDs(rows)) + } + + // Another process with the client's token is not the client's pod. + other := exec.CommandContext(cctx, "go", "run", ".", "capabilities", "--socket", sock, "--token-file", tokenFile) + other.Dir = build.Dir + if out, err := other.CombinedOutput(); err == nil || !strings.Contains(string(out), workerv1alpha1.ReasonNoIdentity) && + !strings.Contains(string(out), "Unauthenticated") { + t.Fatalf("a process outside the token's pod was accepted: %v\n%s", err, out) + } +} From cbcdc3de37b94c17d2d7c0366e9249642cc540e9 Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Wed, 18 Nov 2026 18:03:26 -0500 Subject: [PATCH 5/6] docs(worker): Record the Worker API decisions made in #1433 Implementing the contract settled six points it left open. The epic coordinator approved each, and the contract now states them: - D19: the session ID travels in x-galactic-worker-session; a call without a live session is FAILED_PRECONDITION/NoSession, and a GetCapabilities with a live session renews it - D20: shard and backend destinations are in the API but report Resolved=False/DestinationKindUnsupported until #1437 and #1438 add their resolvers inside Worker - D21: the peer check maps SO_PEERCRED to a pod by its cgroup path, so Worker needs hostPID and the host's /proc, which #1441 packages - D22: the allowlist is a YAML file passed with --allowlist - D18 now includes the claim limit, 1024 per caller - Worker uses internal/edge/ownership in place; #1434 relocates it The pages also record ClaimNotFound for an Update of a missing claim, the status reasons the implementation reports, which capability a claim needs, how Worker compares VPC identity (the base62 identifier on the attachment's status, which is the VPC's fabric identity in this cell), and that Worker adopts only scopes an earlier Worker owned. Co-Authored-By: Claude --- docs/worker/README.md | 16 +++++-- docs/worker/contract.md | 82 +++++++++++++++++++++++++++++++----- docs/worker/ownership.md | 11 ++++- docs/worker/qualification.md | 1 + 4 files changed, 94 insertions(+), 16 deletions(-) diff --git a/docs/worker/README.md b/docs/worker/README.md index 0e5dfa23..f5d1dcbb 100644 --- a/docs/worker/README.md +++ b/docs/worker/README.md @@ -1,12 +1,18 @@ # Galactic Worker -Galactic Worker is a proposed node-local service. Service agents use it to +Galactic Worker is a node-local service. Service agents use it to deliver tenant traffic locally or across the fabric, and it becomes the single writer of shared host state. These pages are the contract for [epic #1430](https://github.com/datum-cloud/galactic/issues/1430). Every Task of the epic implements against them on the `integration/galactic-worker` branch. -Status: proposed. Nothing described here is implemented at the branch start. +Status: the service API, reconciliation core, client library and module +interface are implemented by +[#1433](https://github.com/datum-cloud/galactic/issues/1433). No delivery module +is built in yet: Local ([#1435](https://github.com/datum-cloud/galactic/issues/1435)) +and Fabric ([#1436](https://github.com/datum-cloud/galactic/issues/1436)) add +them, so every capability reports `ModuleNotRunning` until then. +`examples/worker-client` is an independent client of the published API. | Page | Contents | | -------------------------------------- | ----------------------------------------------------------------------------------------- | @@ -41,4 +47,8 @@ PR. "Confirmed" means the product owner approved the decision on the date given. | D15 | Branch workflow with start commit `451eb40` | Accepted | [Branch workflow](branch-workflow.md) | | D16 | NIC: Broadcom NetXtreme-C/E (`bnxt_en`) in generic mode; native mode on lab veth devices | Confirmed 2026-11-18 | [Qualification](qualification.md#platform) | | D17 | Comparison against the main base on the same host, using the MASQUE benchmark method | Proposed | [Qualification](qualification.md#comparison-method) | -| D18 | Numeric limits | Confirmed 2026-11-18 | [Qualification](qualification.md#limits) | +| D18 | Numeric limits, including 1024 claims per caller | Confirmed 2026-11-18 | [Qualification](qualification.md#limits) | +| D19 | The session ID travels in the `x-galactic-worker-session` metadata key; a call without a live session is `FAILED_PRECONDITION`/`NoSession` | Accepted | [Contract](contract.md#sessions-restart-and-stale-deletion) | +| D20 | Shard and backend destinations report `DestinationKindUnsupported` until #1437 and #1438 add their resolvers inside Worker | Accepted | [Contract](contract.md#claims) | +| D21 | The peer check maps `SO_PEERCRED` to a pod by the cgroup path, so Worker needs `hostPID` and the host `/proc` | Accepted | [Contract](contract.md#transport-identity-and-access) | +| D22 | The allowlist is a YAML file passed with `--allowlist` | Accepted | [Contract](contract.md#transport-identity-and-access) | diff --git a/docs/worker/contract.md b/docs/worker/contract.md index edfbbcf1..63d82587 100644 --- a/docs/worker/contract.md +++ b/docs/worker/contract.md @@ -1,9 +1,12 @@ # Worker service contract -Status: proposed, version `v1alpha1`. Nothing in this page is implemented at -the branch start. [#1433](https://github.com/datum-cloud/galactic/issues/1433) -implements the API, and the Tasks named in each section own its tests. Decision -IDs (D1, D2, …) refer to the [decision log](README.md#decision-log). +Status: version `v1alpha1`. +[#1433](https://github.com/datum-cloud/galactic/issues/1433) implements the +service (`cmd/galactic-worker`, `internal/worker`), its protobuf API +(`api/galactic/worker/v1alpha1`), the client library (`pkg/workerclient`) and +the delivery module interface (`internal/worker/module`). The Tasks named in +each section own the rest and its tests. Decision IDs (D1, D2, …) refer to the +[decision log](README.md#decision-log). Galactic Worker is one process per node. Service agents (NAT, Gateway, the Envoy galactic-vrf sidecar and the provider connector) ask it to deliver tenant @@ -32,6 +35,14 @@ A second caller that names the same attachment gets `PERMISSION_DENIED` with reason `AttachmentClaimedByAnotherCaller`. Fabric state for a VPC is shared: any number of claims may resolve destinations through the same VPC. +A destination names an attachment, a shard or a backend. Worker resolves +attachment destinations. The shard resolver, from NAT's trusted source, is added +inside Worker by [#1437](https://github.com/datum-cloud/galactic/issues/1437), +and the backend resolver by +[#1438](https://github.com/datum-cloud/galactic/issues/1438). Until then a shard +or backend destination is accepted and reports `Resolved=False` with reason +`DestinationKindUnsupported` (D20). + The mode preference is `auto`, `local` or `fabric`. `auto` lets Worker resolve each destination. `local` and `fabric` restrict the result; a destination that cannot resolve to the requested mode reports `Resolved=False` with reason @@ -64,6 +75,12 @@ that needs an unavailable capability is accepted and reports `Programmed=False` with the same reason, so a client can tell a temporary module outage from a refused request. +A claim needs the Local capability of its handoff (`local.tap`, +`local.veth_pair`, `local.routed_link` or `local.socket`) for its Local +destinations, and `fabric.srv6` for its Fabric destinations. The caller must +be allowed the handoff's capability, and `fabric.srv6` when a destination +resolves to Fabric at admission. + ### Version negotiation The client sends the range of API versions it supports. Worker picks the @@ -119,6 +136,12 @@ Worker takes these facts only from the sources below, never from the request (D1 | Backend SID | The backend node's locator plus the slot the gateway encodes in the uSID, as `docs/gateway/architecture.md` describes. | | Interface traffic policy | TrafficPolicyBinding, resolved by the cell controller. Worker applies no policy of its own. | +Worker compares a VPC as the base62 identifier the attachment's status records +(`VPCAttachment.status.vpc`, `EdgeAttachment.status.vpc`). That identifier is +the VPC's fabric identity as this cell received it, and a claim's `vpc` is the +same identifier. An attachment whose identifier is still empty authorizes +nothing. + A claim is cross-VPC when any destination belongs to a VPC other than the claim's attachment VPC. Worker refuses it. Two VPCs may use the same addresses, so Worker compares VPC identity, never addresses. @@ -130,11 +153,29 @@ so Worker compares VPC identity, never addresses. | `INVALID_ARGUMENT` | `MalformedSpec`, `WrongNode` | Fix the request. Do not retry. | | `UNAUTHENTICATED` | `NoIdentity`, `IdentityExpired` | Refresh the credential, then retry. | | `PERMISSION_DENIED` | `VPCNotAuthorized`, `AttachmentClaimedByAnotherCaller`, `NotOwner`, `CapabilityNotAllowed` | Do not retry until authority changes. | -| `FAILED_PRECONDITION` | `UnsupportedVersion`, `ClaimExists`, `GenerationConflict` | Re-read with `Observe`, then use the right operation. | +| `FAILED_PRECONDITION` | `UnsupportedVersion`, `ClaimExists`, `ClaimNotFound`, `GenerationConflict` | Re-read with `Observe`, then use the right operation. | +| `FAILED_PRECONDITION` | `NoSession` | Call `GetCapabilities` for a new session, then retry. | | `ABORTED` | `StaleGeneration` | Re-read with `Observe`; retry only with a newer generation. | | `UNAVAILABLE` | `Resyncing`, `ShuttingDown` | Retry with backoff. | | `RESOURCE_EXHAUSTED` | `ClaimLimit`, `TableRangeExhausted` | Retry with backoff; report if it persists. | +`Update` of a claim that does not exist, or is being withdrawn, gets +`ClaimNotFound`. + +Status carries these reasons besides the ones above: + +| Reason | Where | Meaning | +| -------------------------------------------------- | ------------- | --------------------------------------------------------------------------------------------------------- | +| `Pending` | Any condition | Worker has not yet reconciled the generation | +| `DestinationKindUnsupported` | `Resolved` | A shard or backend destination, until #1437 or #1438 resolves it | +| `DestinationNotFound` | `Resolved` | The destination attachment no longer exists | +| `LocatorUnknown` | `Resolved` | This node or the destination's node has no SRv6 locator yet | +| `ModeUnavailable` | `Resolved` | The destination resolves to a mode the preference excludes | +| `ModuleNotRunning` and other capability reasons | `Programmed` | The capability the destination needs is unavailable | +| `ProgrammingFailed` | `Programmed` | The last commit failed; the message carries the error | +| `ScopeOwnedElsewhere` | `Programmed` | Another writer owns the VPC scope's fence (see [ownership](ownership.md#bootstrap-adoption-and-transfer)) | +| `Released`, `LeaseExpired`, `AuthorizationRevoked` | `Withdrawing` | Why the claim is being withdrawn | + Clients retry with exponential backoff from 200 ms to 30 s with full jitter. Every mutating call is idempotent for a given generation, so a retry after a lost response is safe (D3). Every refusal carries a gRPC code and a machine @@ -143,6 +184,11 @@ reason from this table (D6). Worker retries its own programming from 1 s to ## Sessions, restart and stale deletion +Every call after `GetCapabilities` carries the session ID in the +`x-galactic-worker-session` metadata key. A call without a live session of the +caller gets `FAILED_PRECONDITION`/`NoSession`. A `GetCapabilities` that carries +a live session ID renews that session instead of starting another (D19). + A session starts with `GetCapabilities` and carries a lease of 120 s, renewed by an open `Observe` stream or a call at least every 10 s. These are the XDP dispatcher's slot lease values (`internal/plumbing/ebpf/xdpdispatch`), already @@ -175,12 +221,26 @@ D8 and D9 were confirmed on 2026-11-18. `/run/galactic-worker/worker.sock`, mode `0660`, owned by a dedicated group. No TCP listener serves the mutating API. - **Client identity (D8).** Each call carries a projected ServiceAccount token - with audience `galactic-worker`, bound to the calling pod. Worker validates it - with TokenReview, caches the result until the token expires, and checks that - the bound pod runs on this node. Worker also reads the peer's credentials - (`SO_PEERCRED`) and refuses a peer that is not a process of that pod. A static - allowlist maps each ServiceAccount to the capabilities it may use, for - example the NAT agent to `local.veth_pair` and `fabric.srv6`. + with audience `galactic-worker`, bound to the calling pod, as + `authorization: Bearer `. Worker validates it with TokenReview, caches + the result until the token expires, and checks that the bound pod runs on + this node. Worker also reads the peer's credentials (`SO_PEERCRED`) and + refuses a peer that is not a process of that pod. It maps the peer PID to a + pod through the pod UID in `/proc//cgroup`, so Worker runs in the host + PID namespace and reads the host's `/proc`; in any other PID namespace the + kernel reports PID 0 for a peer in another pod (D21). + [#1441](https://github.com/datum-cloud/galactic/issues/1441) packages + `hostPID` and the `/proc` mount. +- **Allowlist (D8, D22).** A static YAML file, passed with `--allowlist`, maps + each ServiceAccount to the capabilities it may use. A ServiceAccount it does + not list may use none. + + ```yaml + serviceAccounts: + - serviceAccount: galactic-system/galactic-nat + capabilities: [local.veth_pair, fabric.srv6] + ``` + - **Access model (D9).** Host-network agents (NAT, Gateway, galactic-edge) mount `/run/galactic-worker` with a `hostPath` volume. The galactic-vrf sidecar runs in the Envoy pod's network namespace and mounts the same directory; pod diff --git a/docs/worker/ownership.md b/docs/worker/ownership.md index 5f790e15..d45379f2 100644 --- a/docs/worker/ownership.md +++ b/docs/worker/ownership.md @@ -21,7 +21,9 @@ starts. A write that bypasses the fence is a defect. The fence is `internal/edge/ownership`, generalized from its current VPC scope (`internal/edge/ownership/ownership.go:150-317`). Its state directory must -outlive every writer, as the package documentation requires. +outlive every writer, as the package documentation requires. Worker uses the +package in place, with its `VPCScope` names and its shared `DefaultDir`; +relocating it is part of [#1434](https://github.com/datum-cloud/galactic/issues/1434). A resource class has one of two kinds of authority: @@ -72,7 +74,12 @@ Each step uses the fence operations that exist today (D13). incarnation owned. Both return a ticket that requires `CommitSnapshot`, which Worker runs with its complete desired state read from disk, merged with the host's current rows. Until that snapshot commits, Worker refuses claim - changes for the scope with `UNAVAILABLE`/`Resyncing`. + changes for the scope with `UNAVAILABLE`/`Resyncing`. Each Worker + incarnation's owner ID starts with `galactic-worker:`, which is how a new + incarnation recognizes a scope an earlier one owned. Worker never fences a + scope another writer owns: its claims there report `Programmed=False` with + reason `ScopeOwnedElsewhere` until that writer's transfer step (3) hands the + scope over. 3. **Ownership transfer.** A legacy writer moves to Worker in two steps. First it writes with `Exclusive` on the shared fence instead of writing directly. Then Worker adopts the scope, and the legacy writer calls Worker instead of diff --git a/docs/worker/qualification.md b/docs/worker/qualification.md index 4ce6d2d6..c823edb7 100644 --- a/docs/worker/qualification.md +++ b/docs/worker/qualification.md @@ -69,6 +69,7 @@ hardware. | Worker reconcile time, claim change to `Programmed=True` | p99 under 1 s | | Re-convergence after a Worker restart, to every claim `Ready` | Under 5 s | | Worker resident memory | Under 256 MiB | +| Claims one caller holds (`ClaimLimit`, `--claim-limit`) | 1024 | CPU per packet, drops and guest contention are measured and reported against the base without a limit, as the MASQUE benchmark does, because they depend on From d573ade6f0a00b6099a381318db70e5ceaf83625 Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Wed, 18 Nov 2026 18:09:16 -0500 Subject: [PATCH 6/6] fix(worker): Check fabric.srv6 even before a locator is known Admission compared locators to decide whether a claim needs fabric.srv6, ignored lookup errors, and skipped the check while either locator was unknown. During bring-up a caller without fabric.srv6 could be admitted with a destination on another node, and once the locators appeared reconciliation programmed it as Fabric without checking the owner again. A destination on another node now needs fabric.srv6 whatever the locators say, and a lookup error refuses with Resyncing. Worker also records the owner's capabilities with each generation, and reconciliation leaves unprogrammed any destination whose mode they do not cover, reporting CapabilityNotAllowed on Programmed. Co-Authored-By: Claude --- docs/worker/contract.md | 7 +++++-- internal/worker/claims.go | 21 ++++++++++++++++++--- internal/worker/reconcile.go | 9 ++++++++- internal/worker/store/store.go | 8 ++++++-- internal/worker/worker_test.go | 32 ++++++++++++++++++++++++++++++++ 5 files changed, 69 insertions(+), 8 deletions(-) diff --git a/docs/worker/contract.md b/docs/worker/contract.md index 63d82587..e5a51b66 100644 --- a/docs/worker/contract.md +++ b/docs/worker/contract.md @@ -78,8 +78,10 @@ refused request. A claim needs the Local capability of its handoff (`local.tap`, `local.veth_pair`, `local.routed_link` or `local.socket`) for its Local destinations, and `fabric.srv6` for its Fabric destinations. The caller must -be allowed the handoff's capability, and `fabric.srv6` when a destination -resolves to Fabric at admission. +be allowed the handoff's capability, and `fabric.srv6` when a destination is +on another node or resolves to Fabric at admission. Worker records the owner's +capabilities with each generation and does not program a destination that +later resolves to a mode they do not cover. ### Version negotiation @@ -172,6 +174,7 @@ Status carries these reasons besides the ones above: | `LocatorUnknown` | `Resolved` | This node or the destination's node has no SRv6 locator yet | | `ModeUnavailable` | `Resolved` | The destination resolves to a mode the preference excludes | | `ModuleNotRunning` and other capability reasons | `Programmed` | The capability the destination needs is unavailable | +| `CapabilityNotAllowed` | `Programmed` | The destination resolved to a mode the owner was not allowed at admission, so it stays unprogrammed | | `ProgrammingFailed` | `Programmed` | The last commit failed; the message carries the error | | `ScopeOwnedElsewhere` | `Programmed` | Another writer owns the VPC scope's fence (see [ownership](ownership.md#bootstrap-adoption-and-transfer)) | | `Released`, `LeaseExpired`, `AuthorizationRevoked` | `Withdrawing` | Why the claim is being withdrawn | diff --git a/internal/worker/claims.go b/internal/worker/claims.go index d43da96c..35cc854b 100644 --- a/internal/worker/claims.go +++ b/internal/worker/claims.go @@ -177,7 +177,10 @@ func (w *Worker) admit(ctx context.Context, caller auth.Caller, k module.ClaimKe case !att.Admitted: return notAuthorized("the claim's attachment is not admitted") } - own, _ := w.d.Sources.Locator(ctx, w.cfg.Node) + own, err := w.d.Sources.Locator(ctx, w.cfg.Node) + if err != nil { + return refusal.New(codes.Unavailable, workerv1alpha1.ReasonResyncing, "trusted sources unavailable", nil) + } needFabric := spec.GetMode() == workerv1alpha1.Mode_MODE_FABRIC for _, d := range spec.GetDestinations() { t, ok := d.GetTarget().(*workerv1alpha1.Destination_Attachment) @@ -199,7 +202,19 @@ func (w *Worker) admit(ctx context.Context, caller auth.Caller, k module.ClaimKe if spec.GetMode() == workerv1alpha1.Mode_MODE_LOCAL { continue } - if loc, _ := w.d.Sources.Locator(ctx, dst.Node); own.IsValid() && loc.IsValid() && loc != own { + // A destination on another node needs fabric.srv6 even while its + // locator is unknown, so the check cannot be skipped during + // bring-up. Reconciliation checks the recorded capabilities again + // when the destination resolves. + if dst.Node != w.cfg.Node { + needFabric = true + continue + } + loc, err := w.d.Sources.Locator(ctx, dst.Node) + if err != nil { + return refusal.New(codes.Unavailable, workerv1alpha1.ReasonResyncing, "trusted sources unavailable", nil) + } + if own.IsValid() && loc.IsValid() && loc != own { needFabric = true } } @@ -265,7 +280,7 @@ func (w *Worker) put(ctx context.Context, caller auth.Caller, key *workerv1alpha return nil, refusal.New(codes.Internal, workerv1alpha1.ReasonMalformedSpec, "encode spec", nil) } rec := store.Record{Node: k.Node, VPC: k.VPC, AttachmentUID: k.AttachmentUID, Owner: caller.ServiceAccount, - Generation: gen, SpecHash: hash, Spec: raw} + Capabilities: caller.Capabilities, Generation: gen, SpecHash: hash, Spec: raw} if err := w.d.Store.Put(rec); err != nil { w.log.Error("record claim", "error", err) return nil, refusal.New(codes.Unavailable, workerv1alpha1.ReasonResyncing, "cannot record the claim durably", nil) diff --git a/internal/worker/reconcile.go b/internal/worker/reconcile.go index 354ecd4a..04119295 100644 --- a/internal/worker/reconcile.go +++ b/internal/worker/reconcile.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "net/netip" + "slices" "sort" "strings" "time" @@ -218,7 +219,13 @@ func (w *Worker) resolve(ctx context.Context, vpc string, claims []claim) ( } if de.resolved { capName := module.RequiredCapability(c.spec.GetHandoff(), de.mode) - if cp := w.d.Modules.Capability(capName); !cp.Available { + if !slices.Contains(c.rec.Capabilities, capName) { + // The destination resolved to a mode the owner was + // not allowed when it was admitted, for example once a + // locator became known. + de.capReason = workerv1alpha1.ReasonCapabilityNotAllowed + de.capMsg = c.rec.Owner + " may not use " + capName + } else if cp := w.d.Modules.Capability(capName); !cp.Available { de.capReason, de.capMsg = cp.Reason, cp.Message if de.capMsg == "" { de.capMsg = capName + " is unavailable" diff --git a/internal/worker/store/store.go b/internal/worker/store/store.go index 6b58d3f5..03810c58 100644 --- a/internal/worker/store/store.go +++ b/internal/worker/store/store.go @@ -33,8 +33,12 @@ type Record struct { VPC string `json:"vpc"` AttachmentUID string `json:"attachmentUID"` // Owner is the ServiceAccount, "namespace/name", that created the claim. - Owner string `json:"owner"` - Generation uint64 `json:"generation"` + Owner string `json:"owner"` + // Capabilities are the capabilities the owner was allowed when the + // generation was accepted. Reconciliation programs no destination + // whose capability is missing from them. + Capabilities []string `json:"capabilities,omitempty"` + Generation uint64 `json:"generation"` // SpecHash is the hash of the deterministic encoding of Spec. SpecHash string `json:"specHash"` // Spec is the claim spec in protobuf JSON. diff --git a/internal/worker/worker_test.go b/internal/worker/worker_test.go index 35c36468..ce9513e2 100644 --- a/internal/worker/worker_test.go +++ b/internal/worker/worker_test.go @@ -8,6 +8,7 @@ import ( "context" "fmt" "maps" + "net/netip" "slices" "sync" "testing" @@ -491,3 +492,34 @@ func versions() *workerv1alpha1.GetCapabilitiesRequest { func srcAttachment(uid, vpc string) sources.Attachment { return sources.Attachment{UID: uid, VPC: vpc, Node: node, Kind: sources.KindVPCAttachment, Admitted: true} } + +// TestFabricCapabilityFailsClosed refuses a remote destination to a caller +// without fabric.srv6 while the destination's locator is still unknown, and +// leaves unprogrammed a destination that only becomes remote after +// admission. +func TestFabricCapabilityFailsClosed(t *testing.T) { + e := newEnv(t) + e.src.SetLocator(otherNode, netip.Prefix{}) + e.start() + tap := func(uids ...string) *workerv1alpha1.ClaimSpec { + return spec(workerv1alpha1.Handoff_HANDOFF_TAP, uids...) + } + wantRefusal(t, e.request(gwSA, key(vpcA, "svc-a"), 1, tap("guest-a2")), codes.PermissionDenied, + workerv1alpha1.ReasonCapabilityNotAllowed) + + k := mkey(vpcA, "svc-a") + e.mustOK(e.request(gwSA, key(vpcA, "svc-a"), 1, tap("guest-a3"))) + e.waitReady(k, 1) + moved := srcAttachment("guest-a3", vpcA) + moved.Node = otherNode + e.src.SetAttachment(moved) + e.src.SetLocator(otherNode, locB) + e.w.Changed(vpcA) + e.eventually("remote destination refused at reconcile", func() bool { + return condOf(e.status(k), workerv1alpha1.ConditionProgrammed).GetReason() == + workerv1alpha1.ReasonCapabilityNotAllowed + }) + if rows := e.host.Rows(vpcA); len(rows) != 0 { + t.Fatalf("rows = %v, want the destination the owner may not reach unprogrammed", rowIDs(rows)) + } +}