diff --git a/api/galactic/worker/v1alpha1/contract.go b/api/galactic/worker/v1alpha1/contract.go index f1584c5a..333d20b6 100644 --- a/api/galactic/worker/v1alpha1/contract.go +++ b/api/galactic/worker/v1alpha1/contract.go @@ -40,13 +40,16 @@ const ( CapabilityLocalRoutedLink = "local.routed_link" CapabilityLocalSocket = "local.socket" CapabilityFabricSRv6 = "fabric.srv6" + // CapabilityFabricRoutes allows a caller to hand Worker tenant routes + // through the Routes service. Only galactic-router is granted it. + CapabilityFabricRoutes = "fabric.routes" ) // Capabilities lists every capability in the order GetCapabilities reports // them. var Capabilities = []string{ CapabilityLocalTap, CapabilityLocalVethPair, CapabilityLocalRoutedLink, - CapabilityLocalSocket, CapabilityFabricSRv6, + CapabilityLocalSocket, CapabilityFabricSRv6, CapabilityFabricRoutes, } // HandoffCapability is the Local capability a handoff kind needs, or "" for @@ -115,4 +118,32 @@ const ( ReasonAuthorizationRevoked = "AuthorizationRevoked" ReasonLeaseExpired = "LeaseExpired" ReasonReleased = "Released" + // ReasonDestinationAddressUnknown is a destination whose attachment + // records no address, such as a guest that manages its own addressing. + ReasonDestinationAddressUnknown = "DestinationAddressUnknown" + // ReasonForwardRouteMissing is a destination address no tenant route + // covers. + ReasonForwardRouteMissing = "ForwardRouteMissing" + // ReasonRouteWithdrawn is a destination address whose tenant route was + // withdrawn. + ReasonRouteWithdrawn = "RouteWithdrawn" + // ReasonRouteMismatch is a destination address whose covering route + // leads to another node's locator, or does not encapsulate. + ReasonRouteMismatch = "RouteMismatch" + // ReasonUnderlayUnreachable is a destination whose locator has no + // underlay route over an SRv6 uplink. + ReasonUnderlayUnreachable = "UnderlayUnreachable" + // ReasonNeighborUnresolved is a destination whose underlay next hop has + // no resolved neighbor, or whose programmed next hop is out of date. + ReasonNeighborUnresolved = "NeighborUnresolved" + // ReasonReturnPathNotProgrammed is a destination whose return traffic + // this node cannot yet decapsulate into the VPC's table. + ReasonReturnPathNotProgrammed = "ReturnPathNotProgrammed" +) + +// Route rejection reasons, carried in RejectedRoute. +const ( + // ReasonMalformedRoute is a route whose VPC, prefix or SID does not + // parse. + ReasonMalformedRoute = "MalformedRoute" ) diff --git a/api/galactic/worker/v1alpha1/routes.pb.go b/api/galactic/worker/v1alpha1/routes.pb.go new file mode 100644 index 00000000..748eb3ca --- /dev/null +++ b/api/galactic/worker/v1alpha1/routes.pb.go @@ -0,0 +1,427 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// The tenant route handoff from galactic-router to Galactic Worker. +// docs/worker/contract.md describes it under "Tenant route handoff". +// +// galactic-router keeps learning tenant routes from EVPN and hands each one +// to Worker, whose Fabric module writes it into the datapath. The service +// shares the Worker socket, authentication and sessions; only a caller +// allowed the fabric.routes capability may call it. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: galactic/worker/v1alpha1/routes.proto + +package workerv1alpha1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// TenantRoute is one tenant route: a prefix in a VPC's table, encapsulated +// toward a SID. +type TenantRoute struct { + state protoimpl.MessageState `protogen:"open.v1"` + // vpc is the base62 identifier of the VPC whose route target imported the + // route. + Vpc string `protobuf:"bytes,1,opt,name=vpc,proto3" json:"vpc,omitempty"` + // table_id is the kernel VRF table the caller resolved for vpc. Worker + // programs the route only while it equals the VPC's current table. + TableId uint32 `protobuf:"varint,2,opt,name=table_id,json=tableId,proto3" json:"table_id,omitempty"` + // prefix is the route's destination, in CIDR form. + Prefix string `protobuf:"bytes,3,opt,name=prefix,proto3" json:"prefix,omitempty"` + // sid is the IPv6 SID the route encapsulates toward. + Sid string `protobuf:"bytes,4,opt,name=sid,proto3" json:"sid,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TenantRoute) Reset() { + *x = TenantRoute{} + mi := &file_galactic_worker_v1alpha1_routes_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TenantRoute) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TenantRoute) ProtoMessage() {} + +func (x *TenantRoute) ProtoReflect() protoreflect.Message { + mi := &file_galactic_worker_v1alpha1_routes_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TenantRoute.ProtoReflect.Descriptor instead. +func (*TenantRoute) Descriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_routes_proto_rawDescGZIP(), []int{0} +} + +func (x *TenantRoute) GetVpc() string { + if x != nil { + return x.Vpc + } + return "" +} + +func (x *TenantRoute) GetTableId() uint32 { + if x != nil { + return x.TableId + } + return 0 +} + +func (x *TenantRoute) GetPrefix() string { + if x != nil { + return x.Prefix + } + return "" +} + +func (x *TenantRoute) GetSid() string { + if x != nil { + return x.Sid + } + return "" +} + +// TenantRouteKey names a tenant route. +type TenantRouteKey struct { + state protoimpl.MessageState `protogen:"open.v1"` + Vpc string `protobuf:"bytes,1,opt,name=vpc,proto3" json:"vpc,omitempty"` + // prefix is the route's destination, in CIDR form. + Prefix string `protobuf:"bytes,2,opt,name=prefix,proto3" json:"prefix,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TenantRouteKey) Reset() { + *x = TenantRouteKey{} + mi := &file_galactic_worker_v1alpha1_routes_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TenantRouteKey) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TenantRouteKey) ProtoMessage() {} + +func (x *TenantRouteKey) ProtoReflect() protoreflect.Message { + mi := &file_galactic_worker_v1alpha1_routes_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TenantRouteKey.ProtoReflect.Descriptor instead. +func (*TenantRouteKey) Descriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_routes_proto_rawDescGZIP(), []int{1} +} + +func (x *TenantRouteKey) GetVpc() string { + if x != nil { + return x.Vpc + } + return "" +} + +func (x *TenantRouteKey) GetPrefix() string { + if x != nil { + return x.Prefix + } + return "" +} + +type RoutesSyncRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // sequence is chosen by the client and echoed in the response. + Sequence uint64 `protobuf:"varint,1,opt,name=sequence,proto3" json:"sequence,omitempty"` + // upserts add routes or replace the route with the same key. + Upserts []*TenantRoute `protobuf:"bytes,2,rep,name=upserts,proto3" json:"upserts,omitempty"` + // withdrawals remove routes. Withdrawing an unknown route succeeds. + Withdrawals []*TenantRouteKey `protobuf:"bytes,3,rep,name=withdrawals,proto3" json:"withdrawals,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RoutesSyncRequest) Reset() { + *x = RoutesSyncRequest{} + mi := &file_galactic_worker_v1alpha1_routes_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RoutesSyncRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RoutesSyncRequest) ProtoMessage() {} + +func (x *RoutesSyncRequest) ProtoReflect() protoreflect.Message { + mi := &file_galactic_worker_v1alpha1_routes_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RoutesSyncRequest.ProtoReflect.Descriptor instead. +func (*RoutesSyncRequest) Descriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_routes_proto_rawDescGZIP(), []int{2} +} + +func (x *RoutesSyncRequest) GetSequence() uint64 { + if x != nil { + return x.Sequence + } + return 0 +} + +func (x *RoutesSyncRequest) GetUpserts() []*TenantRoute { + if x != nil { + return x.Upserts + } + return nil +} + +func (x *RoutesSyncRequest) GetWithdrawals() []*TenantRouteKey { + if x != nil { + return x.Withdrawals + } + return nil +} + +type RoutesSyncResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // sequence is the request's sequence. + Sequence uint64 `protobuf:"varint,1,opt,name=sequence,proto3" json:"sequence,omitempty"` + // rejected lists the routes of the request Worker did not record, with a + // machine reason. Every other route of the request is recorded. + Rejected []*RejectedRoute `protobuf:"bytes,2,rep,name=rejected,proto3" json:"rejected,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RoutesSyncResponse) Reset() { + *x = RoutesSyncResponse{} + mi := &file_galactic_worker_v1alpha1_routes_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RoutesSyncResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RoutesSyncResponse) ProtoMessage() {} + +func (x *RoutesSyncResponse) ProtoReflect() protoreflect.Message { + mi := &file_galactic_worker_v1alpha1_routes_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RoutesSyncResponse.ProtoReflect.Descriptor instead. +func (*RoutesSyncResponse) Descriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_routes_proto_rawDescGZIP(), []int{3} +} + +func (x *RoutesSyncResponse) GetSequence() uint64 { + if x != nil { + return x.Sequence + } + return 0 +} + +func (x *RoutesSyncResponse) GetRejected() []*RejectedRoute { + if x != nil { + return x.Rejected + } + return nil +} + +type RejectedRoute struct { + state protoimpl.MessageState `protogen:"open.v1"` + Key *TenantRouteKey `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Reason string `protobuf:"bytes,2,opt,name=reason,proto3" json:"reason,omitempty"` + Message string `protobuf:"bytes,3,opt,name=message,proto3" json:"message,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RejectedRoute) Reset() { + *x = RejectedRoute{} + mi := &file_galactic_worker_v1alpha1_routes_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RejectedRoute) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RejectedRoute) ProtoMessage() {} + +func (x *RejectedRoute) ProtoReflect() protoreflect.Message { + mi := &file_galactic_worker_v1alpha1_routes_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RejectedRoute.ProtoReflect.Descriptor instead. +func (*RejectedRoute) Descriptor() ([]byte, []int) { + return file_galactic_worker_v1alpha1_routes_proto_rawDescGZIP(), []int{4} +} + +func (x *RejectedRoute) GetKey() *TenantRouteKey { + if x != nil { + return x.Key + } + return nil +} + +func (x *RejectedRoute) GetReason() string { + if x != nil { + return x.Reason + } + return "" +} + +func (x *RejectedRoute) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +var File_galactic_worker_v1alpha1_routes_proto protoreflect.FileDescriptor + +const file_galactic_worker_v1alpha1_routes_proto_rawDesc = "" + + "\n" + + "%galactic/worker/v1alpha1/routes.proto\x12\x18galactic.worker.v1alpha1\"d\n" + + "\vTenantRoute\x12\x10\n" + + "\x03vpc\x18\x01 \x01(\tR\x03vpc\x12\x19\n" + + "\btable_id\x18\x02 \x01(\rR\atableId\x12\x16\n" + + "\x06prefix\x18\x03 \x01(\tR\x06prefix\x12\x10\n" + + "\x03sid\x18\x04 \x01(\tR\x03sid\":\n" + + "\x0eTenantRouteKey\x12\x10\n" + + "\x03vpc\x18\x01 \x01(\tR\x03vpc\x12\x16\n" + + "\x06prefix\x18\x02 \x01(\tR\x06prefix\"\xbc\x01\n" + + "\x11RoutesSyncRequest\x12\x1a\n" + + "\bsequence\x18\x01 \x01(\x04R\bsequence\x12?\n" + + "\aupserts\x18\x02 \x03(\v2%.galactic.worker.v1alpha1.TenantRouteR\aupserts\x12J\n" + + "\vwithdrawals\x18\x03 \x03(\v2(.galactic.worker.v1alpha1.TenantRouteKeyR\vwithdrawals\"u\n" + + "\x12RoutesSyncResponse\x12\x1a\n" + + "\bsequence\x18\x01 \x01(\x04R\bsequence\x12C\n" + + "\brejected\x18\x02 \x03(\v2'.galactic.worker.v1alpha1.RejectedRouteR\brejected\"}\n" + + "\rRejectedRoute\x12:\n" + + "\x03key\x18\x01 \x01(\v2(.galactic.worker.v1alpha1.TenantRouteKeyR\x03key\x12\x16\n" + + "\x06reason\x18\x02 \x01(\tR\x06reason\x12\x18\n" + + "\amessage\x18\x03 \x01(\tR\amessage2o\n" + + "\x06Routes\x12e\n" + + "\x04Sync\x12+.galactic.worker.v1alpha1.RoutesSyncRequest\x1a,.galactic.worker.v1alpha1.RoutesSyncResponse(\x010\x01BCZAgo.datum.net/galactic/api/galactic/worker/v1alpha1;workerv1alpha1b\x06proto3" + +var ( + file_galactic_worker_v1alpha1_routes_proto_rawDescOnce sync.Once + file_galactic_worker_v1alpha1_routes_proto_rawDescData []byte +) + +func file_galactic_worker_v1alpha1_routes_proto_rawDescGZIP() []byte { + file_galactic_worker_v1alpha1_routes_proto_rawDescOnce.Do(func() { + file_galactic_worker_v1alpha1_routes_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_galactic_worker_v1alpha1_routes_proto_rawDesc), len(file_galactic_worker_v1alpha1_routes_proto_rawDesc))) + }) + return file_galactic_worker_v1alpha1_routes_proto_rawDescData +} + +var file_galactic_worker_v1alpha1_routes_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_galactic_worker_v1alpha1_routes_proto_goTypes = []any{ + (*TenantRoute)(nil), // 0: galactic.worker.v1alpha1.TenantRoute + (*TenantRouteKey)(nil), // 1: galactic.worker.v1alpha1.TenantRouteKey + (*RoutesSyncRequest)(nil), // 2: galactic.worker.v1alpha1.RoutesSyncRequest + (*RoutesSyncResponse)(nil), // 3: galactic.worker.v1alpha1.RoutesSyncResponse + (*RejectedRoute)(nil), // 4: galactic.worker.v1alpha1.RejectedRoute +} +var file_galactic_worker_v1alpha1_routes_proto_depIdxs = []int32{ + 0, // 0: galactic.worker.v1alpha1.RoutesSyncRequest.upserts:type_name -> galactic.worker.v1alpha1.TenantRoute + 1, // 1: galactic.worker.v1alpha1.RoutesSyncRequest.withdrawals:type_name -> galactic.worker.v1alpha1.TenantRouteKey + 4, // 2: galactic.worker.v1alpha1.RoutesSyncResponse.rejected:type_name -> galactic.worker.v1alpha1.RejectedRoute + 1, // 3: galactic.worker.v1alpha1.RejectedRoute.key:type_name -> galactic.worker.v1alpha1.TenantRouteKey + 2, // 4: galactic.worker.v1alpha1.Routes.Sync:input_type -> galactic.worker.v1alpha1.RoutesSyncRequest + 3, // 5: galactic.worker.v1alpha1.Routes.Sync:output_type -> galactic.worker.v1alpha1.RoutesSyncResponse + 5, // [5:6] is the sub-list for method output_type + 4, // [4:5] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name +} + +func init() { file_galactic_worker_v1alpha1_routes_proto_init() } +func file_galactic_worker_v1alpha1_routes_proto_init() { + if File_galactic_worker_v1alpha1_routes_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_galactic_worker_v1alpha1_routes_proto_rawDesc), len(file_galactic_worker_v1alpha1_routes_proto_rawDesc)), + NumEnums: 0, + NumMessages: 5, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_galactic_worker_v1alpha1_routes_proto_goTypes, + DependencyIndexes: file_galactic_worker_v1alpha1_routes_proto_depIdxs, + MessageInfos: file_galactic_worker_v1alpha1_routes_proto_msgTypes, + }.Build() + File_galactic_worker_v1alpha1_routes_proto = out.File + file_galactic_worker_v1alpha1_routes_proto_goTypes = nil + file_galactic_worker_v1alpha1_routes_proto_depIdxs = nil +} diff --git a/api/galactic/worker/v1alpha1/routes.proto b/api/galactic/worker/v1alpha1/routes.proto new file mode 100644 index 00000000..c78f3992 --- /dev/null +++ b/api/galactic/worker/v1alpha1/routes.proto @@ -0,0 +1,73 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// The tenant route handoff from galactic-router to Galactic Worker. +// docs/worker/contract.md describes it under "Tenant route handoff". +// +// galactic-router keeps learning tenant routes from EVPN and hands each one +// to Worker, whose Fabric module writes it into the datapath. The service +// shares the Worker socket, authentication and sessions; only a caller +// allowed the fabric.routes capability may call it. + +syntax = "proto3"; + +package galactic.worker.v1alpha1; + +option go_package = "go.datum.net/galactic/api/galactic/worker/v1alpha1;workerv1alpha1"; + +// Routes takes the tenant routes galactic-router learns. The contract names +// the service Routes. +// buf:lint:ignore SERVICE_SUFFIX +service Routes { + // Sync is a stream of route changes. Worker answers each request once it + // has recorded it. An open stream renews the session lease. A client sends + // every route it holds after it opens a stream, because Worker keeps routes + // only in memory. + rpc Sync(stream RoutesSyncRequest) returns (stream RoutesSyncResponse); +} + +// TenantRoute is one tenant route: a prefix in a VPC's table, encapsulated +// toward a SID. +message TenantRoute { + // vpc is the base62 identifier of the VPC whose route target imported the + // route. + string vpc = 1; + // table_id is the kernel VRF table the caller resolved for vpc. Worker + // programs the route only while it equals the VPC's current table. + uint32 table_id = 2; + // prefix is the route's destination, in CIDR form. + string prefix = 3; + // sid is the IPv6 SID the route encapsulates toward. + string sid = 4; +} + +// TenantRouteKey names a tenant route. +message TenantRouteKey { + string vpc = 1; + // prefix is the route's destination, in CIDR form. + string prefix = 2; +} + +message RoutesSyncRequest { + // sequence is chosen by the client and echoed in the response. + uint64 sequence = 1; + // upserts add routes or replace the route with the same key. + repeated TenantRoute upserts = 2; + // withdrawals remove routes. Withdrawing an unknown route succeeds. + repeated TenantRouteKey withdrawals = 3; +} + +message RoutesSyncResponse { + // sequence is the request's sequence. + uint64 sequence = 1; + // rejected lists the routes of the request Worker did not record, with a + // machine reason. Every other route of the request is recorded. + repeated RejectedRoute rejected = 2; +} + +message RejectedRoute { + TenantRouteKey key = 1; + string reason = 2; + string message = 3; +} diff --git a/api/galactic/worker/v1alpha1/routes_grpc.pb.go b/api/galactic/worker/v1alpha1/routes_grpc.pb.go new file mode 100644 index 00000000..18105536 --- /dev/null +++ b/api/galactic/worker/v1alpha1/routes_grpc.pb.go @@ -0,0 +1,143 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// The tenant route handoff from galactic-router to Galactic Worker. +// docs/worker/contract.md describes it under "Tenant route handoff". +// +// galactic-router keeps learning tenant routes from EVPN and hands each one +// to Worker, whose Fabric module writes it into the datapath. The service +// shares the Worker socket, authentication and sessions; only a caller +// allowed the fabric.routes capability may call it. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc (unknown) +// source: galactic/worker/v1alpha1/routes.proto + +package workerv1alpha1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + Routes_Sync_FullMethodName = "/galactic.worker.v1alpha1.Routes/Sync" +) + +// RoutesClient is the client API for Routes service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Routes takes the tenant routes galactic-router learns. The contract names +// the service Routes. +// buf:lint:ignore SERVICE_SUFFIX +type RoutesClient interface { + // Sync is a stream of route changes. Worker answers each request once it + // has recorded it. An open stream renews the session lease. A client sends + // every route it holds after it opens a stream, because Worker keeps routes + // only in memory. + Sync(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[RoutesSyncRequest, RoutesSyncResponse], error) +} + +type routesClient struct { + cc grpc.ClientConnInterface +} + +func NewRoutesClient(cc grpc.ClientConnInterface) RoutesClient { + return &routesClient{cc} +} + +func (c *routesClient) Sync(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[RoutesSyncRequest, RoutesSyncResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &Routes_ServiceDesc.Streams[0], Routes_Sync_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[RoutesSyncRequest, RoutesSyncResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Routes_SyncClient = grpc.BidiStreamingClient[RoutesSyncRequest, RoutesSyncResponse] + +// RoutesServer is the server API for Routes service. +// All implementations must embed UnimplementedRoutesServer +// for forward compatibility. +// +// Routes takes the tenant routes galactic-router learns. The contract names +// the service Routes. +// buf:lint:ignore SERVICE_SUFFIX +type RoutesServer interface { + // Sync is a stream of route changes. Worker answers each request once it + // has recorded it. An open stream renews the session lease. A client sends + // every route it holds after it opens a stream, because Worker keeps routes + // only in memory. + Sync(grpc.BidiStreamingServer[RoutesSyncRequest, RoutesSyncResponse]) error + mustEmbedUnimplementedRoutesServer() +} + +// UnimplementedRoutesServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedRoutesServer struct{} + +func (UnimplementedRoutesServer) Sync(grpc.BidiStreamingServer[RoutesSyncRequest, RoutesSyncResponse]) error { + return status.Error(codes.Unimplemented, "method Sync not implemented") +} +func (UnimplementedRoutesServer) mustEmbedUnimplementedRoutesServer() {} +func (UnimplementedRoutesServer) testEmbeddedByValue() {} + +// UnsafeRoutesServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to RoutesServer will +// result in compilation errors. +type UnsafeRoutesServer interface { + mustEmbedUnimplementedRoutesServer() +} + +func RegisterRoutesServer(s grpc.ServiceRegistrar, srv RoutesServer) { + // If the following call panics, it indicates UnimplementedRoutesServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&Routes_ServiceDesc, srv) +} + +func _Routes_Sync_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(RoutesServer).Sync(&grpc.GenericServerStream[RoutesSyncRequest, RoutesSyncResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Routes_SyncServer = grpc.BidiStreamingServer[RoutesSyncRequest, RoutesSyncResponse] + +// Routes_ServiceDesc is the grpc.ServiceDesc for Routes service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Routes_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "galactic.worker.v1alpha1.Routes", + HandlerType: (*RoutesServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "Sync", + Handler: _Routes_Sync_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "galactic/worker/v1alpha1/routes.proto", +} diff --git a/api/galactic/worker/v1alpha1/worker.pb.go b/api/galactic/worker/v1alpha1/worker.pb.go index cafcc3f2..95f9024a 100644 --- a/api/galactic/worker/v1alpha1/worker.pb.go +++ b/api/galactic/worker/v1alpha1/worker.pb.go @@ -449,7 +449,7 @@ func (x *GetCapabilitiesResponse) GetNode() string { type Capability struct { state protoimpl.MessageState `protogen:"open.v1"` // name is one of local.tap, local.veth_pair, local.routed_link, - // local.socket and fabric.srv6. + // local.socket, fabric.srv6 and fabric.routes. Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` State CapabilityState `protobuf:"varint,2,opt,name=state,proto3,enum=galactic.worker.v1alpha1.CapabilityState" json:"state,omitempty"` // reason is a machine reason when the capability is unavailable, such as @@ -1381,11 +1381,20 @@ type DestinationStatus struct { Mode Mode `protobuf:"varint,2,opt,name=mode,proto3,enum=galactic.worker.v1alpha1.Mode" json:"mode,omitempty"` Resolved bool `protobuf:"varint,3,opt,name=resolved,proto3" json:"resolved,omitempty"` Programmed bool `protobuf:"varint,4,opt,name=programmed,proto3" json:"programmed,omitempty"` - Ready bool `protobuf:"varint,5,opt,name=ready,proto3" json:"ready,omitempty"` + // ready is true when both forward_ready and return_ready are true. + Ready bool `protobuf:"varint,5,opt,name=ready,proto3" json:"ready,omitempty"` // reason explains the first of resolved, programmed and ready that is // false. - Reason string `protobuf:"bytes,6,opt,name=reason,proto3" json:"reason,omitempty"` - Message string `protobuf:"bytes,7,opt,name=message,proto3" json:"message,omitempty"` + Reason string `protobuf:"bytes,6,opt,name=reason,proto3" json:"reason,omitempty"` + Message string `protobuf:"bytes,7,opt,name=message,proto3" json:"message,omitempty"` + // forward_ready is true when traffic from the handoff can reach the + // destination: for a Fabric destination, a route toward the destination's + // locator covers each of its addresses and the underlay next hop resolves. + ForwardReady bool `protobuf:"varint,8,opt,name=forward_ready,json=forwardReady,proto3" json:"forward_ready,omitempty"` + // return_ready is true when traffic from the destination can come back to + // the handoff: for a Fabric destination, this node decapsulates the VPC's + // SID into the VPC's table. + ReturnReady bool `protobuf:"varint,9,opt,name=return_ready,json=returnReady,proto3" json:"return_ready,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1469,6 +1478,20 @@ func (x *DestinationStatus) GetMessage() string { return "" } +func (x *DestinationStatus) GetForwardReady() bool { + if x != nil { + return x.ForwardReady + } + return false +} + +func (x *DestinationStatus) GetReturnReady() bool { + if x != nil { + return x.ReturnReady + } + return false +} + // ClaimStatus is a complete snapshot of one claim. A client drops a // duplicate or reordered snapshot by comparing observed_generation and // revision. @@ -1654,7 +1677,7 @@ const file_galactic_worker_v1alpha1_worker_proto_rawDesc = "" + "\x04type\x18\x01 \x01(\tR\x04type\x12A\n" + "\x06status\x18\x02 \x01(\x0e2).galactic.worker.v1alpha1.ConditionStatusR\x06status\x12\x16\n" + "\x06reason\x18\x03 \x01(\tR\x06reason\x12\x18\n" + - "\amessage\x18\x04 \x01(\tR\amessage\"\x94\x02\n" + + "\amessage\x18\x04 \x01(\tR\amessage\"\xdc\x02\n" + "\x11DestinationStatus\x12G\n" + "\vdestination\x18\x01 \x01(\v2%.galactic.worker.v1alpha1.DestinationR\vdestination\x122\n" + "\x04mode\x18\x02 \x01(\x0e2\x1e.galactic.worker.v1alpha1.ModeR\x04mode\x12\x1a\n" + @@ -1664,7 +1687,9 @@ const file_galactic_worker_v1alpha1_worker_proto_rawDesc = "" + "programmed\x12\x14\n" + "\x05ready\x18\x05 \x01(\bR\x05ready\x12\x16\n" + "\x06reason\x18\x06 \x01(\tR\x06reason\x12\x18\n" + - "\amessage\x18\a \x01(\tR\amessage\"\xb8\x03\n" + + "\amessage\x18\a \x01(\tR\amessage\x12#\n" + + "\rforward_ready\x18\b \x01(\bR\fforwardReady\x12!\n" + + "\freturn_ready\x18\t \x01(\bR\vreturnReady\"\xb8\x03\n" + "\vClaimStatus\x124\n" + "\x03key\x18\x01 \x01(\v2\".galactic.worker.v1alpha1.ClaimKeyR\x03key\x12\x1e\n" + "\n" + diff --git a/api/galactic/worker/v1alpha1/worker.proto b/api/galactic/worker/v1alpha1/worker.proto index 8bbfd367..46226b9a 100644 --- a/api/galactic/worker/v1alpha1/worker.proto +++ b/api/galactic/worker/v1alpha1/worker.proto @@ -72,7 +72,7 @@ enum CapabilityState { message Capability { // name is one of local.tap, local.veth_pair, local.routed_link, - // local.socket and fabric.srv6. + // local.socket, fabric.srv6 and fabric.routes. string name = 1; CapabilityState state = 2; // reason is a machine reason when the capability is unavailable, such as @@ -224,11 +224,20 @@ message DestinationStatus { Mode mode = 2; bool resolved = 3; bool programmed = 4; + // ready is true when both forward_ready and return_ready are true. bool ready = 5; // reason explains the first of resolved, programmed and ready that is // false. string reason = 6; string message = 7; + // forward_ready is true when traffic from the handoff can reach the + // destination: for a Fabric destination, a route toward the destination's + // locator covers each of its addresses and the underlay next hop resolves. + bool forward_ready = 8; + // return_ready is true when traffic from the destination can come back to + // the handoff: for a Fabric destination, this node decapsulates the VPC's + // SID into the VPC's table. + bool return_ready = 9; } // ClaimStatus is a complete snapshot of one claim. A client drops a diff --git a/cmd/galactic-router/root.go b/cmd/galactic-router/root.go index c2918239..23bb63bb 100644 --- a/cmd/galactic-router/root.go +++ b/cmd/galactic-router/root.go @@ -44,6 +44,7 @@ import ( "go.datum.net/galactic/internal/reconcile" galacticruntime "go.datum.net/galactic/internal/runtime" "go.datum.net/galactic/internal/runtime/gobgp" + "go.datum.net/galactic/internal/runtime/workerroutes" "go.datum.net/galactic/internal/serviceroute" tpnode "go.datum.net/galactic/internal/trafficpolicy/node" "go.datum.net/galactic/pkg/serviceplan" @@ -89,6 +90,27 @@ func prepareGoBGPAPIDir(dir string) error { return nil } +// workerHandoff returns where tenant routes go besides the datapath. Every +// route is written under its VPC's write fence, which leaves a VPC +// galactic-worker owns to Worker; without the fence directory the router +// writes directly, as before Worker existed. With a Worker socket configured, +// every route is also handed to Worker, by a handoff that runs until ctx is +// done. +func workerHandoff(ctx context.Context, cfg *config.RouterConfig) (gobgp.TenantRouteSink, gobgp.VPCFence) { + var fence gobgp.VPCFence + if auth, err := ownership.Open(ownership.DefaultDir); err != nil { + ctrl.Log.Error(err, "VPC write fence unavailable: tenant routes are written without it") + } else { + fence = gobgp.OwnershipFence(auth) + } + if cfg.WorkerSocket == "" { + return nil, fence + } + h := workerroutes.New(cfg.WorkerSocket, cfg.WorkerTokenFile) + go h.Run(ctx) + return h, fence +} + // runCmd contains the application startup logic. It reads configuration from // the provided config and initializes the BGP runtime. func runCmd(cfg *config.RouterConfig) error { @@ -217,7 +239,7 @@ func runCmd(cfg *config.RouterConfig) error { return err } factory := gobgp.NewRuntimeFactory(int32(bgpListenPort), cfg.Reflector, bgpLocalAddr, peerEventEmitter, bmpCfg, - gobgp.WithAPIDir(cfg.GoBGPAPIDir)) + gobgp.WithAPIDir(cfg.GoBGPAPIDir), gobgp.WithWorkerHandoff(workerHandoff(ctx, cfg))) // Create runtime manager. runtimeMgr := galacticruntime.NewRuntimeManager(factory) @@ -587,6 +609,10 @@ func newRootCommand() *cobra.Command { "Interval between BMP statistics reports per peer; 0 disables them") cmd.Flags().String("gobgp-api-dir", "", "Directory to serve the GoBGP API in, read-only, as a unix socket per BGPRouter; empty serves none") + cmd.Flags().String("worker-socket", "", + "Galactic Worker socket to hand tenant routes to; empty hands Worker nothing") + cmd.Flags().String("worker-token-file", config.DefaultRouterWorkerTokenFile, + "Projected ServiceAccount token, audience galactic-worker, presented to Worker") cmd.Flags().Bool("build-info", false, "Print build information and exit") cmd.Flags().BoolP("version", "V", false, "Print version and exit") diff --git a/cmd/galactic-worker/root.go b/cmd/galactic-worker/root.go index 7d804e3b..59e167b0 100644 --- a/cmd/galactic-worker/root.go +++ b/cmd/galactic-worker/root.go @@ -10,6 +10,7 @@ import ( "fmt" "log/slog" "net" + "net/netip" "os" "strings" "time" @@ -31,12 +32,15 @@ import ( "go.datum.net/galactic/internal/gc" "go.datum.net/galactic/internal/logging" "go.datum.net/galactic/internal/metadata" + "go.datum.net/galactic/internal/nodelocator" "go.datum.net/galactic/internal/ownership" "go.datum.net/galactic/internal/plumbing/ebpf/attach" "go.datum.net/galactic/internal/worker" "go.datum.net/galactic/internal/worker/auth" + "go.datum.net/galactic/internal/worker/fabric" "go.datum.net/galactic/internal/worker/hostscope" "go.datum.net/galactic/internal/worker/module" + "go.datum.net/galactic/internal/worker/routes" "go.datum.net/galactic/internal/worker/sources" "go.datum.net/galactic/internal/worker/store" ) @@ -94,10 +98,6 @@ func (o *options) validate() error { return errors.Join(errs...) } -// modules returns the delivery modules this build runs. Local (#1435) and -// Fabric (#1436) add themselves here. -func modules() []module.Module { return nil } - func run(o *options) error { allow, err := auth.LoadAllowlist(o.allowlist) if err != nil { @@ -113,11 +113,6 @@ func run(o *options) error { if err != nil { return err } - reg, err := module.NewRegistry(modules()...) - if err != nil { - return err - } - scheme := runtime.NewScheme() utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(sources.AddToScheme(scheme)) @@ -167,6 +162,28 @@ func run(o *options) error { } else { edge = guard } + // The tenant routes galactic-router hands Worker, which the Fabric + // module writes. + tenantRoutes := routes.New() + fabricDatapath := &fabric.Kernel{PinDir: o.pinDir, K8s: direct, Namespace: o.namespace, Node: o.nodeName} + defer func() { _ = fabricDatapath.Close() }() + fab, err := fabric.New(fabric.Config{ + Routes: tenantRoutes, + Datapath: fabricDatapath, + OwnLocator: func(ctx context.Context) (netip.Prefix, error) { + return nodelocator.Prefix(ctx, mgr.GetClient(), o.namespace, o.nodeName) + }, + Log: slog.Default(), + }) + if err != nil { + return err + } + fabric.InitMetrics() + // The delivery modules this build runs. + reg, err := module.NewRegistry(fab) + if err != nil { + return err + } w, err := worker.New(worker.Config{ Node: o.nodeName, Lease: o.lease, RenewInterval: o.renewInterval, ClaimLimit: o.claimLimit, ResyncInterval: o.resyncInterval, HostGrace: o.hostGrace, @@ -184,7 +201,8 @@ func run(o *options) error { Allowlist: allow, Node: o.nodeName, }, - Log: slog.Default(), + Routes: tenantRoutes, + Log: slog.Default(), }) if err != nil { return err @@ -199,6 +217,14 @@ func run(o *options) error { }); err != nil { return err } + // An underlay route or neighbor change requeues only the VPCs whose + // tenant routes it touches; the periodic resync covers a lost + // subscription. + if err := fab.Watch(ctx, w.RoutesChanged, func(err error) { + slog.Error("route and neighbor event subscription failed; relying on periodic resync", "error", err) + }); err != nil { + return err + } mgrErr := make(chan error, 1) go func() { mgrErr <- mgr.Start(ctx) }() diff --git a/config/galactic-router/base/daemonset.yaml b/config/galactic-router/base/daemonset.yaml index b26468a7..6092a724 100644 --- a/config/galactic-router/base/daemonset.yaml +++ b/config/galactic-router/base/daemonset.yaml @@ -72,6 +72,12 @@ spec: # node pod. See the gobgp-api volume below. - name: GALACTIC_ROUTER_GOBGP_API_DIR value: /run/galactic-router + # Hand every tenant route to galactic-worker, whose Fabric module + # writes the routes of the VPCs it owns. Without a Worker on the + # node the router writes every route itself and keeps retrying + # the connection quietly. + - name: GALACTIC_ROUTER_WORKER_SOCKET + value: /run/galactic-worker/worker.sock ports: - name: metrics containerPort: 9179 @@ -173,6 +179,13 @@ spec: mountPath: /var/lib/galactic-edge/ownership - name: gobgp-api mountPath: /run/galactic-router + # galactic-worker's socket, and the token, audience + # galactic-worker, that identifies this pod to it. + - name: galactic-worker-socket + mountPath: /run/galactic-worker + - name: galactic-worker-token + mountPath: /var/run/secrets/galactic-worker + readOnly: true volumes: - name: netns hostPath: @@ -189,6 +202,17 @@ spec: hostPath: path: /run/galactic-router type: DirectoryOrCreate + - name: galactic-worker-socket + hostPath: + path: /run/galactic-worker + type: DirectoryOrCreate + - name: galactic-worker-token + projected: + sources: + - serviceAccountToken: + audience: galactic-worker + expirationSeconds: 3600 + path: token # Never delete or restore this directory while a writer is alive. - name: edge-ownership hostPath: diff --git a/deploy/containerlab/docs/baseline.md b/deploy/containerlab/docs/baseline.md index 4e53ae5a..6f0f290e 100644 --- a/deploy/containerlab/docs/baseline.md +++ b/deploy/containerlab/docs/baseline.md @@ -149,3 +149,29 @@ After a deliberate change to the fixture or to an expectation, rerun with `BASELINE_WRITE_REFERENCE=1 task baseline:twice`. The reference is replaced only when both runs pass and agree. Commit the new `baseline/reference-results.json` with the change that caused it. + +## Running Galactic Worker on the fixture + +`scripts/worker-fabric.sh start` builds this checkout's `galactic-worker` and +runs it as a process on `dfw-worker` and `sjc-worker`, with the cluster's +admin kubeconfig and an allowlist that grants galactic-router +`fabric.routes`. It is lab-only: Worker has no manifests yet. galactic-router +finds Worker through `GALACTIC_ROUTER_WORKER_SOCKET`, and Worker's Fabric +module then writes the tenant routes of every VPC on the node. + +| Command | Prints or does | +| ------------------------------- | ---------------------------------------------------- | +| `worker-fabric.sh scopes NODE` | The node's VPC fence records, with their owners | +| `worker-fabric.sh table NODE V` | The kernel table of VPC `V`'s VRF on the node | +| `worker-fabric.sh dump NODE V` | VPC `V`'s `egress_route_table` entries, one per line | +| `worker-fabric.sh stop` | Stops Worker on both nodes | + +Worker takes over every VPC scope on its node and never hands one back, so +after `stop` the router leaves those VPCs' tenant routes to a Worker that is +gone. Run `task destroy` and `task deploy` before reusing the lab. + +The fixture's workers belong to different clusters, and Worker resolves a +claim's destinations only within its own cluster. A claim on one worker +toward a guest on the other therefore reports `DestinationNotFound`, so the +fixture shows Fabric's tenant routes and traffic, not claim readiness across +the two workers. diff --git a/deploy/containerlab/scripts/worker-fabric.sh b/deploy/containerlab/scripts/worker-fabric.sh new file mode 100755 index 00000000..90d981e1 --- /dev/null +++ b/deploy/containerlab/scripts/worker-fabric.sh @@ -0,0 +1,141 @@ +#!/bin/bash +# worker-fabric.sh: run this checkout's galactic-worker on the compute nodes +# of the dedicated-placement fixture (docs/baseline.md) and inspect what its +# Fabric module programs. Lab-only: Worker has no manifests yet, so the +# binary runs as a process inside each Kind node with the cluster's admin +# kubeconfig. galactic-router finds it through GALACTIC_ROUTER_WORKER_SOCKET +# and hands it every tenant route. +# +# Worker takes over every VPC scope on the node and never hands one back, so +# after stop the router leaves those VPCs' tenant routes to a Worker that is +# gone. Redeploy the lab (task destroy, task deploy) before reusing it. +# +# Usage: +# worker-fabric.sh start build, copy and start Worker on NODES +# worker-fabric.sh stop stop Worker on NODES +# worker-fabric.sh scopes NODE print the node's VPC fence records +# worker-fabric.sh table NODE VPC print VPC's VRF table ID on NODE +# worker-fabric.sh dump NODE VPC print VPC's egress_route_table entries +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +REPO_DIR=$(cd "${SCRIPT_DIR}/../../.." && pwd) +NODES=${NODES:-"dfw-worker sjc-worker"} +DIR=/opt/galactic-worker +LOG=/var/log/galactic-worker.log +PIN_DIR=/sys/fs/bpf/galactic +FENCE_DIR=/var/lib/galactic-edge/ownership + +site_of() { echo "${1%%-*}"; } + +start() { + local bin + bin=$(mktemp -d) + trap 'rm -rf "${bin}"' RETURN + (cd "${REPO_DIR}" && CGO_ENABLED=0 go build -o "${bin}/galactic-worker" ./cmd/galactic-worker) + cat >"${bin}/allowlist.yaml" <<'EOF' +serviceAccounts: + - serviceAccount: galactic-system/galactic-router + capabilities: [fabric.routes] +EOF + for node in ${NODES}; do + kind get kubeconfig --name "$(site_of "${node}")" --internal >"${bin}/kubeconfig" + docker exec "${node}" mkdir -p "${DIR}" + docker cp "${bin}/galactic-worker" "${node}:${DIR}/galactic-worker" + docker cp "${bin}/allowlist.yaml" "${node}:${DIR}/allowlist.yaml" + docker cp "${bin}/kubeconfig" "${node}:${DIR}/kubeconfig" + docker exec -d -e KUBECONFIG="${DIR}/kubeconfig" "${node}" sh -c \ + "exec ${DIR}/galactic-worker --node-name ${node} --allowlist ${DIR}/allowlist.yaml \ + --metrics-port 19104 --grpc-health-port 19105 --log-level debug >>${LOG} 2>&1" + done + for node in ${NODES}; do + for _ in $(seq 60); do + if docker exec "${node}" test -S /run/galactic-worker/worker.sock; then + echo "${node}: Worker serving" + continue 2 + fi + sleep 2 + done + echo "${node}: Worker did not start; log follows" >&2 + docker exec "${node}" tail -50 "${LOG}" >&2 || true + return 1 + done +} + +stop() { + for node in ${NODES}; do + docker exec "${node}" sh -c "pkill -TERM -f '^${DIR}/galactic-worker' || true" + for _ in $(seq 30); do + docker exec "${node}" pgrep -f "^${DIR}/galactic-worker" >/dev/null || break + sleep 1 + done + docker exec "${node}" sh -c "pkill -KILL -f '^${DIR}/galactic-worker' || true; rm -f /run/galactic-worker/worker.sock" + echo "${node}: Worker stopped" + done +} + +scopes() { + docker exec "$1" sh -c "for f in ${FENCE_DIR}/vpc-*.json; do [ -e \"\$f\" ] || continue; echo \"\$(basename \"\$f\" .json) \$(cat \"\$f\")\"; done" +} + +table() { + local vrf + # The VPC identifier is zero-padded to 9 characters; printf pads strings + # with spaces only. + vrf="G$(printf '%9s' "$2" | tr ' ' 0)V" + docker exec "$1" ip -j -d link show dev "${vrf}" | jq -r '.[0].linkinfo.info_data.table' +} + +# dump prints NODE's egress_route_table entries in VPC's table, one per line: +# prefix, SID, link index and destination MAC. It needs bpftool on the host. +dump() { + local node=$1 vpc=$2 tbl pid + tbl=$(table "${node}" "${vpc}") + pid=$(docker inspect -f '{{.State.Pid}}' "${node}") + # The node's bpffs is reachable through its /proc root; bpftool runs on the + # host, since the node image has none. + sudo bpftool -j map dump pinned "/proc/${pid}/root${PIN_DIR}/egress_route_table" | + python3 -c ' +import ipaddress, json, struct, sys + +table = int(sys.argv[1]) + +def raw(field): + return bytes(int(b, 16) for b in field) + +out = [] +for e in json.load(sys.stdin): + f = e.get("formatted") + if f: + k, v = f["key"], f["value"] + tid, fam, plen = k["table_id"], k["family"], k["prefixlen"] - 40 + addr, sid = bytes(k["addr"]), bytes(v["sid"]) + link, dmac = v["link_ifindex"], bytes(v["dmac"]) + else: + # struct egress_route_key and egress_route_value, packed, little endian. + kb, vb = raw(e["key"]), raw(e["value"]) + plen, tid = struct.unpack_from("&2 + exit 2 + ;; +esac diff --git a/docs/router/architecture.md b/docs/router/architecture.md index c5c21960..cc4f0415 100644 --- a/docs/router/architecture.md +++ b/docs/router/architecture.md @@ -119,6 +119,7 @@ there instead. - **Session state and router readiness are exported as metrics, from the two paths above.** Each BGPRouter reconcile, including one that fails before its status update, sets `galactic_router_peer_established{router,peer}` from the runtime's status and `galactic_router_ready{router}` from the BGPRouter's `Ready` condition (`internal/controller/router_metrics.go`). A peer the runtime no longer holds loses its series, and a stopped or retargeted router loses both gauges. An unhealthy runtime lists no peers, so it sets every peer this process last exported to 0 instead, keeping the session alerts firing; after a galactic-router restart whose runtime never becomes healthy there are no peer series, and only `GalacticRouterNotReady` fires. A router with no peers has no series, so `GalacticRouterNoPeersEstablished` cannot fire for it. The event watcher increments `galactic_router_peer_session_drops_total{router,peer}` on every transition out of Established, so drops that the 30s poll misses are still counted. A peer the runtime itself deletes or updates is marked for 30s (`markSelfReset`), so config churn is not counted as a drop, and a deleted peer's counter is removed. The counter otherwise lives until the runtime stops, which a retarget does not do. `GalacticRouterPeerNotEstablished`, `GalacticRouterNoPeersEstablished` (critical), `GalacticRouterPeerFlapping` and `GalacticRouterNotReady` alert on them. - **Routes for route-target-less EVPN paths defer to the underlay.** A path with no route target (the gateway's anycast VIPs, and each egress shard's SID `/64` and masquerade address) becomes a plain kernel route toward its BGP next hop, with no encapsulation. These routes live in their own table, `srv6.PlainRouteTable` (`0xFFFE0000`), marked `proto 250`, and are reached through two policy rules ahead of the main table's own: `32000: lookup main suppress_prefixlength 0`, then `32001: lookup 0xFFFE0000`. An underlay route for the same destination, or for anything more specific than the default route, therefore always wins. Keeping them out of the main table is required, not cosmetic: zebra imports every main-table route it did not install as a `kernel` route with distance 0, so a plain route there stopped FRR installing its own BGP route for the same prefix (#670). The kernel cannot resolve an IPv6 next hop recursively, so each route is written with the link and next hop the gateway resolves to at the time. `internal/runtime/gobgp/plainroutes.go` re-resolves every route whenever the main table changes and every 30s, and retries one that cannot resolve yet. It removes owned routes no path asks for once a two-minute startup grace has passed, and deletes the unmarked main-table route an earlier version left for each prefix. `ip route show table all proto 250` lists every route it owns. XDP datapaths that call `bpf_fib_lookup` with `BPF_FIB_LOOKUP_DIRECT` (`natprog`, `edgeprog`) read the main table only and do not see these routes. - **A tenant VRF route that fails to install is retried, not dropped.** `processEVPNPath` writes a VRF-scoped path's route into `egress_route_table` through `internal/runtime/gobgp/vrfroutes.go`. `Register` resolves the SID's uplink and neighbor MAC at write time, which fails while an uplink is still coming up after a restart, and BGP never redelivers an unchanged best path, so before #676 such a route stayed missing until the path changed upstream. `vrfRoutes` keeps each failed route in a pending set and retries it whenever the main table changes (1s debounce) and every 30s, until it installs or its path is withdrawn; removing a VRF drops its pending routes. One mutex serializes every write to the map from this runtime, taken per route during a retry so an unresolvable neighbor (up to 2s) does not hold up the RIB watcher for the whole pass. `EgressRouteTable.Refresh` cannot cover this: it walks existing entries only. The pending count, and the plain-route reconciler's failing count, are exported as `galactic_router_route_install_failing{kind="vrf"|"plain"}`, and `GalacticRouterRoutesNotInstalled` fires when either stays above 0 for 10 minutes. +- **Tenant VRF routes are written under the VPC's write fence, and a VPC galactic-worker owns is Worker's to write.** Every `egress_route_table` write `vrfRoutes` makes, and each removal after a VRF moves table, runs under the VPC's scope of the node's write fence (`internal/ownership`) with `RefuseWorkerOwned`. In a VPC no Worker owns the router writes the route as before. In a VPC a galactic-worker incarnation owns, the fence refuses the write and the router counts the route as delivered, because it also hands every route it installs or withdraws to Worker through the Routes service (`internal/runtime/workerroutes`, enabled by `GALACTIC_ROUTER_WORKER_SOCKET`). Worker's Fabric module writes and re-resolves those routes; see [Worker contract](../worker/contract.md#tenant-route-handoff). A withdrawal the fence refuses while ownership moves is retried with the pending installs. Without the fence directory the router writes without it. - **A VRF's routes follow its kernel table.** `applyVRFs` re-reads the table of every VRF it has applied on each reconcile, from one VRF listing. When the kernel VRF was recreated on another table, the VRF is applied again, the route-target index moves to the new table, and `withdrawMovedVRFRoutes` removes the routes of the VRF's best paths from the old table, deleting only entries that still point at the SID each path names (#1270). A VRF that is absent, or a listing that fails, keeps the table it has. A VPC's table normally survives recreation through its node reservation; see [VRF table IDs](../cni/architecture.md#vrf-table-ids). - **`ServiceVIPBinding`s are generated, one per node per backend.** `NetworkServiceBindingReconciler` (`internal/controller/networkservice_binding_controller.go`) runs on every node and writes a binding for each VIP of each accepted `NetworkService`, at most one per family (`ruleTranslatedVIPs`) and, for the IPv4 VIP, only while some gateway node's `/IPv4Served` condition reports it loaded (`ipv4LoadedOnSomeNode`; a status change to that signal passes `ruleBindingInputsChanged`, and the node deletes its IPv4 bindings once no gateway does), and each backend the rule's `backendSelector` places on this node, through the same `selectRuleBackends` the gateway uses, so the gateway never sends a backend's flows to a node with no binding planned for it. Workloads are IPv6 only, so both VIPs' backends are the selected attachments' IPv6 addresses: an IPv4 VIP's binding pairs the IPv4 `vipAddress` with an IPv6 `backendAddress`, which the backend node serves by translating the flows (see [ARCHITECTURE-CNI.md](../cni/architecture.md#key-design-decisions)). `ServiceVIPBindingReconciler` refuses an IPv4 `backendAddress` with `BindFailed`. Each node writes and deletes only its own bindings (labelled `galactic.datum.net/binding-node` and `app.kubernetes.io/managed-by: galactic-router`), so there is one writer per object and no leader election. A binding is owned by its rule, records the rule's `vpcRef` (which names the VRF its rows go in) and takes `egressKind` from the attachment's interface mode. `ServiceVIPBindingReconciler` programs a binding only when a `BGPAdvertisement` on this node's `BGPRouter` with that VRF's ID has a prefix containing `backendAddress`, and otherwise reports `BindFailed`; it watches advertisements on its own router so a binding written before its advertisement converges, and teardown skips the check. A backend serves only the oldest rule (by creation time, then namespace/name) selecting it on the same backend port and protocol, since its node translates the backend's replies back to only one VIP; a claim (`backendClaim`) carries the VIP's family, since a reply to an IPv4 client takes the IPv4 VIP's egress-translated row and any other reply the IPv6 VIP's egress row, so one backend socket answers for one VIP of each family, an IPv4 rule never competes with an IPv6 one, and two rules with IPv4 VIPs on one socket do; both sides compute that owner with `ruleBackendOwners` over every rule in the namespace, so a newer rule gets no binding for the backend and no flows for it from the gateway, and every rule in a namespace is re-queued when one of them changes. Rules compete only within a VPC, so the per-VPC `VPCAttachment` fan-out still reaches every rule an attachment change can affect. The node summarizes its bindings on the rule as `/BackendsBound` (`BackendsClaimed` names a backend an older rule serves); a rule whose spec cannot be built into bindings (a second IPv6 VIP, or a selector that does not parse) keeps the node's existing bindings, the same as a rule that loses `Accepted`, so an invalid edit does not drop the backend nodes' `vip_xlat_table` rows; a second IPv4 VIP does the same for the IPv4 family alone, writing no IPv4 binding and keeping the IPv4 ones it had while the IPv6 VIP's bindings stay current, and both report `InvalidRule`. Only the nodes that hold bindings for such a rule report `InvalidRule`, since the gateways already report the error and a condition from every node would be a status write per node per edit. Each rule reconcile lists only its own VPC's attachments, through the VPCAttachment→`status.vpc` field index (`listVPCAttachmentsInVPC`), and skips the cache's deep copy because nothing it does writes to them. Several backends of one VIP can share a node: the gateway writes each backend's slot into the uSID and `usid.c` keys the ingress row on it (#799). - **Each node probes its own backends and publishes their health.** For a `NetworkService` with `spec.healthCheck`, `BackendHealthReconciler` (`internal/controller/backendhealth_controller.go`) runs one TCP probe loop per backend on its node (`internal/healthprobe`), over the backend's host-side interface (the attachment's `status.hostInterface`) from that interface's link-local address, on `healthCheck.port` or the rule's `backendPort`. The link-local source is what lets the SYN-ACK back: `usid_egress` hands a reply to a link-local destination to the kernel before its VIP rewrite (see [ARCHITECTURE-CNI.md](../cni/architecture.md#key-design-decisions)). A decision moves after `unhealthyThreshold` consecutive failures or `healthyThreshold` consecutive successes; a probe that cannot be sent (no interface, no link-local address) counts toward `Unknown`/`ProbeUnavailable`, never toward `Unhealthy`. The decision goes into `status.health` of each of the backend's bindings, with the attachment UID and probed port, by a merge patch of that field alone and only when it changes, so it never races the `Bound` condition. Every binding of one backend shares one loop. A restarted router seeds each loop from the published decision when its attachment UID and port still match, so a restart changes no gateway's table. A rule losing its `healthCheck` clears `status.health` and stops the loops. The design is [docs/gateway/backend-health.md](../gateway/backend-health.md). diff --git a/docs/router/configuration.md b/docs/router/configuration.md index 15ab5cb3..0d587d42 100644 --- a/docs/router/configuration.md +++ b/docs/router/configuration.md @@ -16,23 +16,25 @@ below. Logging takes `--log-level` (`LOG_LEVEL`), without the `GALACTIC_ROUTER` prefix and the standard `--zap-*` flags; see [Logging](../observability/logging.md). -| Option | Environment Variable | CLI Flag | Default | -| ------------------------ | ------------------------------------------ | ---------------------------- | --------------------------- | -| Node name | `GALACTIC_ROUTER_NODE_NAME` | `--node-name` | _(required)_ | -| Route reflector | `GALACTIC_ROUTER_REFLECTOR` | `--reflector` | `false` | -| BGP listen port | `GALACTIC_ROUTER_BGP_LISTEN_PORT` | `--bgp-listen-port` | `179` | -| BGP local address | `GALACTIC_ROUTER_BGP_LOCAL_ADDRESS` | `--bgp-local-address` | _(auto-detected from `lo`)_ | -| Metrics port | `GALACTIC_ROUTER_METRICS_PORT` | `--metrics-port` | `9179` | -| gRPC health port | `GALACTIC_ROUTER_GRPC_HEALTH_PORT` | `--grpc-health-port` | `5179` | -| Orphan-cleanup namespace | `GALACTIC_ROUTER_GC_NAMESPACE` | `--gc-namespace` | `galactic-system` | -| Orphan-cleanup interval | `GALACTIC_ROUTER_GC_INTERVAL` | `--gc-interval` | `5m` | -| BMP stations | `GALACTIC_ROUTER_BMP_STATIONS` | `--bmp-stations` | _(empty; BMP disabled)_ | -| BMP policy | `GALACTIC_ROUTER_BMP_POLICY` | `--bmp-policy` | `pre-policy` | -| BMP statistics interval | `GALACTIC_ROUTER_BMP_STATISTICS_INTERVAL` | `--bmp-statistics-interval` | `60s` | -| GoBGP API directory | `GALACTIC_ROUTER_GOBGP_API_DIR` | `--gobgp-api-dir` | _(empty; no API)_ | -| Traffic policy | `GALACTIC_ROUTER_TRAFFIC_POLICY` | `--traffic-policy` | `false` | -| Service frontends | `GALACTIC_ROUTER_SERVICE_FRONTEND_ENABLED` | `--service-frontend-enabled` | `false` | -| Spread remote pairing | `GALACTIC_ROUTER_SERVICE_SPREAD_PAIRING` | `--service-spread-pairing` | `first` | +| Option | Environment Variable | CLI Flag | Default | +| ------------------------ | ------------------------------------------ | ---------------------------- | ---------------------------------------- | +| Node name | `GALACTIC_ROUTER_NODE_NAME` | `--node-name` | _(required)_ | +| Route reflector | `GALACTIC_ROUTER_REFLECTOR` | `--reflector` | `false` | +| BGP listen port | `GALACTIC_ROUTER_BGP_LISTEN_PORT` | `--bgp-listen-port` | `179` | +| BGP local address | `GALACTIC_ROUTER_BGP_LOCAL_ADDRESS` | `--bgp-local-address` | _(auto-detected from `lo`)_ | +| Metrics port | `GALACTIC_ROUTER_METRICS_PORT` | `--metrics-port` | `9179` | +| gRPC health port | `GALACTIC_ROUTER_GRPC_HEALTH_PORT` | `--grpc-health-port` | `5179` | +| Orphan-cleanup namespace | `GALACTIC_ROUTER_GC_NAMESPACE` | `--gc-namespace` | `galactic-system` | +| Orphan-cleanup interval | `GALACTIC_ROUTER_GC_INTERVAL` | `--gc-interval` | `5m` | +| BMP stations | `GALACTIC_ROUTER_BMP_STATIONS` | `--bmp-stations` | _(empty; BMP disabled)_ | +| BMP policy | `GALACTIC_ROUTER_BMP_POLICY` | `--bmp-policy` | `pre-policy` | +| BMP statistics interval | `GALACTIC_ROUTER_BMP_STATISTICS_INTERVAL` | `--bmp-statistics-interval` | `60s` | +| GoBGP API directory | `GALACTIC_ROUTER_GOBGP_API_DIR` | `--gobgp-api-dir` | _(empty; no API)_ | +| Worker socket | `GALACTIC_ROUTER_WORKER_SOCKET` | `--worker-socket` | _(empty; no route handoff)_ | +| Worker token file | `GALACTIC_ROUTER_WORKER_TOKEN_FILE` | `--worker-token-file` | `/var/run/secrets/galactic-worker/token` | +| Traffic policy | `GALACTIC_ROUTER_TRAFFIC_POLICY` | `--traffic-policy` | `false` | +| Service frontends | `GALACTIC_ROUTER_SERVICE_FRONTEND_ENABLED` | `--service-frontend-enabled` | `false` | +| Spread remote pairing | `GALACTIC_ROUTER_SERVICE_SPREAD_PAIRING` | `--service-spread-pairing` | `first` | ## Required Options @@ -212,6 +214,33 @@ gobgp --target unix:///host/run/galactic-router/_.sock neighbor **Type:** directory path **Default:** _(empty)_ +### `--worker-socket` / `GALACTIC_ROUTER_WORKER_SOCKET` + +Galactic Worker's socket. With it set, the router hands every tenant route it +installs or withdraws to Worker through the Routes service +([contract](../worker/contract.md#tenant-route-handoff)), and Worker's Fabric +module writes the routes of every VPC Worker owns. The router writes a route +itself only in a VPC Worker does not own, under the VPC's write fence. While +Worker is unreachable the router keeps the changes and retries the connection +with backoff, sending everything it holds once a stream opens. Empty hands +Worker nothing. + +The base DaemonSet sets `/run/galactic-worker/worker.sock` and mounts the +directory from the host. Worker's allowlist must grant the router's +ServiceAccount `fabric.routes`. + +**Type:** socket path +**Default:** _(empty)_ + +### `--worker-token-file` / `GALACTIC_ROUTER_WORKER_TOKEN_FILE` + +The projected ServiceAccount token, audience `galactic-worker`, the router +presents to Worker. It is read again for every connection. The base DaemonSet +projects it at the default path. + +**Type:** file path +**Default:** `/var/run/secrets/galactic-worker/token` + ### `--bmp-policy` / `GALACTIC_ROUTER_BMP_POLICY` Which routes are streamed to each BMP station: diff --git a/docs/worker/README.md b/docs/worker/README.md index 7ece7713..19b95c04 100644 --- a/docs/worker/README.md +++ b/docs/worker/README.md @@ -8,10 +8,12 @@ the epic implements against them on the `integration/galactic-worker` branch. Status: the service API, reconciliation core, client library and module interface are implemented by -[#1433](https://github.com/datum-cloud/galactic/issues/1433). No delivery module -is built in yet: Local ([#1435](https://github.com/datum-cloud/galactic/issues/1435)) -and Fabric ([#1436](https://github.com/datum-cloud/galactic/issues/1436)) add -them, so every capability reports `ModuleNotRunning` until then. +[#1433](https://github.com/datum-cloud/galactic/issues/1433). The Fabric module +([#1436](https://github.com/datum-cloud/galactic/issues/1436)) serves +`fabric.srv6` and `fabric.routes`, and is the only writer of the tenant routes +of every VPC Worker owns. Local +([#1435](https://github.com/datum-cloud/galactic/issues/1435)) is not built in +yet, so the `local.*` capabilities report `ModuleNotRunning`. `examples/worker-client` is an independent client of the published API. [#1434](https://github.com/datum-cloud/galactic/issues/1434) makes Worker the owner of each node and VPC scope, with CNI writing that state under the fence @@ -56,3 +58,6 @@ PR. "Confirmed" means the product owner approved the decision on the date given. | D21 | The peer check maps `SO_PEERCRED` to a pod by the cgroup path, so Worker needs `hostPID` and the host `/proc` | Accepted | [Contract](contract.md#transport-identity-and-access) | | D22 | The allowlist is a YAML file passed with `--allowlist` | Accepted | [Contract](contract.md#transport-identity-and-access) | | D23 | Worker takes a scope over with `Takeover`, which fences and adopts under one lock, so no crash leaves a scope refusing CNI writes | Accepted | [Ownership](ownership.md#bootstrap-adoption-and-transfer) | +| D24 | galactic-router hands Worker its tenant routes through `Routes.Sync` on the Worker socket, behind the `fabric.routes` capability, applying D8 | Accepted | [Contract](contract.md#tenant-route-handoff) | +| D25 | Each destination reports `forward_ready` and `return_ready`; `ready` requires both | Accepted | [Contract](contract.md#state-model) | +| D26 | A destination's addresses come from its VPCAttachment's spec; one with none reports `DestinationAddressUnknown` | Accepted | [Contract](contract.md#trusted-sources) | diff --git a/docs/worker/contract.md b/docs/worker/contract.md index 5662461e..b58863ea 100644 --- a/docs/worker/contract.md +++ b/docs/worker/contract.md @@ -67,8 +67,8 @@ learns the outcome from `Observe`. ### Capabilities `GetCapabilities` reports one row per capability: `local.tap`, -`local.veth_pair`, `local.routed_link`, `local.socket` and `fabric.srv6`. Each -row is `Available` or `Unavailable` with a machine reason, for example +`local.veth_pair`, `local.routed_link`, `local.socket`, `fabric.srv6` and +`fabric.routes`. Each row is `Available` or `Unavailable` with a machine reason, for example `DatapathNotLoaded`, `HookUnsupported` (an XDP mode or device the [qualified platform](qualification.md) excludes) or `ModuleNotRunning`. A claim that needs an unavailable capability is accepted and reports `Programmed=False` @@ -83,6 +83,55 @@ on another node or resolves to Fabric at admission. Worker records the owner's capabilities with each generation and does not program a destination that later resolves to a mode they do not cover. +`fabric.routes` is not a delivery capability. It allows a caller to hand Worker +tenant routes through the Routes service ([tenant route handoff](#tenant-route-handoff)), +and only galactic-router is granted it (D24). + +### Tenant route handoff + +galactic-router keeps learning tenant routes from EVPN, and Worker's Fabric +module writes them (D24). The router hands each route to Worker through a +second service on the Worker socket, `galactic.worker.v1alpha1.Routes`, with +the same transport, identity and sessions as the Worker service (D8). + +| RPC | Semantics | +| ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Sync` | Bidirectional stream. Each request carries upserts (VPC, the VPC's table, prefix, SID) and withdrawals (VPC, prefix). Worker records them and answers with the request's sequence and the routes it rejected with `MalformedRoute`. The open stream renews the session. | + +A caller not allowed `fabric.routes` gets `PERMISSION_DENIED`/`CapabilityNotAllowed`, +and a Worker with no Fabric module `UNAVAILABLE`/`ModuleNotRunning`. Worker +keeps the routes in memory only: the router sends every route it holds and +every recent withdrawal each time it opens a stream, and each change after +that. Worker acknowledges a request once it has recorded it, before the entry +is removed, so the router keeps an acknowledged withdrawal for 10 minutes and +sends it again on each new stream. When a VRF is removed or moves to another +table, the router withdraws every route it handed Worker for the old table. +While Worker is unreachable the router keeps its changes and retries. + +For every VPC scope Worker owns, the Fabric module writes the recorded routes +into `egress_route_table` inside the scope's commit, whether or not the scope +has claims: + +- It writes a route only while the route's table is the VPC's current table, + so a route never lands in another VPC's table, and never toward this node's + own locator, which would loop. +- It re-resolves each route's underlay next hop when a route or neighbor + event touches it, and rewrites the entry when the next hop moved. +- A route whose next hop stops resolving keeps the entry already written for + it, so a shorter entry, such as the VPC's default toward an egress shard, + never catches its traffic. A route that cannot be written yet, because it + has no entry or its SID changed, is retried on the next event or resync; + until then its traffic matches what it matched before, as when the router + wrote it, and its destinations report why. Neither fails the commit, so + other routes of the scope are written. +- It removes an entry only when the router withdraws its route, and never + touches an entry the router did not hand it, such as CNI's pass-through + entries, egress shard entries or Galactic Edge's entries. + +In a VPC no Worker owns, the router writes its routes itself under the VPC's +fence with `Exclusive` (transfer step 1 in +[ownership](ownership.md#bootstrap-adoption-and-transfer)). + ### Version negotiation The client sends the range of API versions it supports. Worker picks the @@ -102,6 +151,17 @@ Worker keeps three states for each claim, and never reports one as another (D2). | Programmed | What Worker wrote for each destination: mode, VRF table, SID, map rows, routes, and the read-back of each write. | Rebuilt from the host after a restart | | Observed | Readiness evidence Worker did not write: the TAP or veth exists and is up, the neighbor resolves, CNI has attached. | Rebuilt from the host; link events in a VPC's VRF requeue its scope | +Each destination's status reports `forward_ready` and `return_ready` +separately, and `ready` only when both are true (D25). For a Fabric +destination, forward readiness means that the `egress_route_table` entry a +packet to each of the destination's addresses matches comes from a tenant route +toward the destination's own locator, and that the route's underlay next hop +resolves as written. Return readiness means this node decapsulates the VPC's +SID into the VPC's table: its locator is in `locator_table`, and its +BGPVRFInstance and `vrf_table` row for the VPC are in place. Each is judged per +destination, so a withdrawal or an unreachable node changes only the +destinations that use it. + Status carries `observed_generation` and these conditions, matching the EdgeAttachment condition names where they mean the same thing: @@ -129,14 +189,16 @@ missing waits, as the [worked transitions](#worked-transitions) show. Worker takes these facts only from the sources below, never from the request (D10). -| Fact | Source | -| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------- | -| Physical locator | The node's BGPRouter Block and Node-ID, the same source `internal/installer/locatorroute.go` uses for the locator's local route. | -| VPC identity | NetworkFabricIdentity for the VPC (`api/cloud/v1alpha1`), written by the fabric identity allocator. | -| Authorized VPC | `VPCAttachment.spec.vpc` for workload and service attachments, written by NSO; for a connector, an EdgeAttachment that is `Admitted`. | -| Owner | The authenticated caller (D8), recorded with the claim's desired state. | -| Backend SID | The backend node's locator plus the slot the gateway encodes in the uSID, as `docs/gateway/architecture.md` describes. | -| Interface traffic policy | TrafficPolicyBinding, resolved by the cell controller. Worker applies no policy of its own. | +| Fact | Source | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Physical locator | The node's BGPRouter Block and Node-ID, the same source `internal/installer/locatorroute.go` uses for the locator's local route. | +| VPC identity | NetworkFabricIdentity for the VPC (`api/cloud/v1alpha1`), written by the fabric identity allocator. | +| Authorized VPC | `VPCAttachment.spec.vpc` for workload and service attachments, written by NSO; for a connector, an EdgeAttachment that is `Admitted`. | +| Owner | The authenticated caller (D8), recorded with the claim's desired state. | +| Backend SID | The backend node's locator plus the slot the gateway encodes in the uSID, as `docs/gateway/architecture.md` describes. | +| Interface traffic policy | TrafficPolicyBinding, resolved by the cell controller. Worker applies no policy of its own. | +| Destination addresses | `VPCAttachment.spec.interface.addresses` (D26). An attachment whose guest manages its own addressing records none, and its destination reports `DestinationAddressUnknown`; taking those addresses from the attachment's advertised routes is follow-up work. | +| Tenant routes | galactic-router, from EVPN, through the [tenant route handoff](#tenant-route-handoff). | Worker compares a VPC as the base62 identifier the attachment's status records (`VPCAttachment.status.vpc`, `EdgeAttachment.status.vpc`). That identifier is @@ -166,18 +228,25 @@ so Worker compares VPC identity, never addresses. Status carries these reasons besides the ones above: -| Reason | Where | Meaning | -| -------------------------------------------------- | ------------- | --------------------------------------------------------------------------------------------------------- | -| `Pending` | Any condition | Worker has not yet reconciled the generation | -| `DestinationKindUnsupported` | `Resolved` | A shard or backend destination, until #1437 or #1438 resolves it | -| `DestinationNotFound` | `Resolved` | The destination attachment no longer exists | -| `LocatorUnknown` | `Resolved` | This node or the destination's node has no SRv6 locator yet | -| `ModeUnavailable` | `Resolved` | The destination resolves to a mode the preference excludes | -| `ModuleNotRunning` and other capability reasons | `Programmed` | The capability the destination needs is unavailable | -| `CapabilityNotAllowed` | `Programmed` | The destination resolved to a mode the owner was not allowed at admission, so it stays unprogrammed | -| `ProgrammingFailed` | `Programmed` | The last commit failed; the message carries the error | -| `ScopeOwnedElsewhere` | `Programmed` | Another writer owns the VPC scope's fence (see [ownership](ownership.md#bootstrap-adoption-and-transfer)) | -| `Released`, `LeaseExpired`, `AuthorizationRevoked` | `Withdrawing` | Why the claim is being withdrawn | +| Reason | Where | Meaning | +| -------------------------------------------------- | --------------------- | --------------------------------------------------------------------------------------------------------- | +| `Pending` | Any condition | Worker has not yet reconciled the generation | +| `DestinationKindUnsupported` | `Resolved` | A shard or backend destination, until #1437 or #1438 resolves it | +| `DestinationNotFound` | `Resolved` | The destination attachment no longer exists | +| `LocatorUnknown` | `Resolved` | This node or the destination's node has no SRv6 locator yet | +| `ModeUnavailable` | `Resolved` | The destination resolves to a mode the preference excludes | +| `ModuleNotRunning` and other capability reasons | `Programmed` | The capability the destination needs is unavailable | +| `CapabilityNotAllowed` | `Programmed` | The destination resolved to a mode the owner was not allowed at admission, so it stays unprogrammed | +| `ProgrammingFailed` | `Programmed` | The last commit failed; the message carries the error | +| `ScopeOwnedElsewhere` | `Programmed` | Another writer owns the VPC scope's fence (see [ownership](ownership.md#bootstrap-adoption-and-transfer)) | +| `Released`, `LeaseExpired`, `AuthorizationRevoked` | `Withdrawing` | Why the claim is being withdrawn | +| `DestinationAddressUnknown` | `Programmed` | The Fabric destination's attachment records no address | +| `ForwardRouteMissing` | `Programmed` | No tenant route covers one of the destination's addresses | +| `RouteWithdrawn` | `Programmed` | The tenant route covering one of the destination's addresses was withdrawn | +| `RouteMismatch` | `Programmed` | The route covering one of the destination's addresses leads to another node's locator | +| `UnderlayUnreachable` | `Programmed`, `Ready` | No route over an SRv6 uplink reaches the destination's locator; the entry is kept | +| `NeighborUnresolved` | `Programmed`, `Ready` | The underlay next hop has no resolved neighbor, or the entry carries an out-of-date one | +| `ReturnPathNotProgrammed` | `Ready` | This node does not yet decapsulate the VPC's SID into the VPC's table | Clients retry with exponential backoff from 200 ms to 30 s with full jitter. Every mutating call is idempotent for a given generation, so a retry after a @@ -242,6 +311,8 @@ D8 and D9 were confirmed on 2026-11-18. serviceAccounts: - serviceAccount: galactic-system/galactic-nat capabilities: [local.veth_pair, fabric.srv6] + - serviceAccount: galactic-system/galactic-router + capabilities: [fabric.routes] ``` - **Access model (D9).** Host-network agents (NAT, Gateway, galactic-edge) mount @@ -260,3 +331,4 @@ Each case names the decision it exercises and the Task that owns its test. | Unauthorized cross-VPC claim | The caller has a claim on an attachment in VPC A. VPC B uses the same guest addresses. | `Update` adds a destination whose attachment is in VPC B. | Resolves the destination's VPC from its attachment record and its identity from NetworkFabricIdentity, finds B ≠ A, and refuses. Records nothing durable, takes no ticket, logs the refusal and counts it in a metric. | `PERMISSION_DENIED`/`VPCNotAuthorized`. The existing generation keeps running. Routes, rules, links and pinned maps are unchanged. | D1, D10 | [#1433](https://github.com/datum-cloud/galactic/issues/1433) zero-mutation snapshot test; [#1442](https://github.com/datum-cloud/galactic/issues/1442) observes the unauthorized destination in the lab | | Local locator with a missing TAP | Claim admitted; one destination is a guest on this node. | Resolution finds the node's own locator, but the guest's TAP does not exist (not yet attached, or being replaced). | Resolves the destination to Local and does not fall back to Fabric. Programs the other destinations. Leaves the missing one unprogrammed and watches CNI attachment events. When CNI ADD creates the TAP, programs it on the new ifindex and reads it back. | Until the TAP appears: `Resolved=True`, `Programmed=False` and `Ready=False`, with reason `HandoffInterfaceMissing` naming the destination. Afterwards: `Ready=True`. | D2, D11 | [#1435](https://github.com/datum-cloud/galactic/issues/1435) root test for Local programming; [#1434](https://github.com/datum-cloud/galactic/issues/1434) for CNI attachment observation | | Withdrawal during repair | A commit failed part-way, so the VPC scope is resync-required and Worker's repair holds a ticket while it prepares a snapshot. | The owner sends `Release` with the current generation. | Records the release durably and answers. Takes the scope over again with `Takeover`, which invalidates the repair's ticket, and commits a snapshot computed from desired state without the released claim. The repair's late commit fails with a transitioning or stale error and is dropped. | `Release` returns `OK`. Status shows `Withdrawing` with scope "one claim" until the read-back confirms removal, then `Observe` reports the claim released. No row of the released claim survives the snapshot. | D3, D12, D23 | [#1433](https://github.com/datum-cloud/galactic/issues/1433) crash and interleaving tests; [#1436](https://github.com/datum-cloud/galactic/issues/1436) for Fabric rows | +| Underlay next hop lost | Claim `Ready=True` with Fabric destinations on nodes B and C. | The underlay route toward node C's locator is withdrawn. | The route event requeues only the VPCs with a route toward that locator. The Fabric module keeps the entries toward node C and rewrites nothing. When the route returns, it rewrites each entry whose next hop moved. | Meanwhile the destination on node C reports `forward_ready=False` with reason `UnderlayUnreachable`, and the one on node B stays `Ready`. Afterwards both are `Ready`. | D11, D25 | [#1436](https://github.com/datum-cloud/galactic/issues/1436) root test for the Fabric datapath and lab captures | diff --git a/docs/worker/ownership.md b/docs/worker/ownership.md index 752764bb..ba859e58 100644 --- a/docs/worker/ownership.md +++ b/docs/worker/ownership.md @@ -37,25 +37,25 @@ second writer appears, the class becomes fenced. ## Writers and authorities -| Resource class | Writers at the branch start | Coordination today | Authority after transfer | Transfer step | Task | -| ----------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------- | ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | -| Host VRF device and table ID (1 to `0xEFFF`) | `vrf.EnsureAndEnslave` (`internal/plumbing/vrf/vrf.go:116`) from galactic-veth and galactic-tap; `vrf.Add` (`vrf.go:93`) from `nodevpc.Ensure` in galactic-edge; `vrf.DeleteIfUnused` (`vrf.go:302`) from galactic-router GC | flock in `/var/lib/cni/galactic-vrf` (`lock.go:28`), reservations in `tables.json` (`reservation.go:35`) | Worker | CNI writes with `Exclusive`; galactic-edge calls Worker. GC moves into Worker. | [#1434](https://github.com/datum-cloud/galactic/issues/1434) | -| Per-node VPC state, SID Argument and BGPVRFInstance | `nodevpc.Ensure` (`internal/nodevpc/ensure.go:72`) from galactic-edge (`internal/edge/agent/vpcstate.go:30`); `ClaimArgument` (`argument.go:109`) and `AllocateArgument` (`argument.go:239`) from galactic-bgp and galactic-edge; the galactic-vrf sidecar writes BGPVRFInstance directly (`internal/ingresssidecar/gateway.go:273`) | Argument flock (`argument.go:33`); the sidecar's write takes no lock | Worker | Route the sidecar's BGPVRFInstance write through Worker. `argument.go:91` records that a writer outside the lock can still collide. | [#1434](https://github.com/datum-cloud/galactic/issues/1434), [#1439](https://github.com/datum-cloud/galactic/issues/1439) for the sidecar | -| `vrf_table` datapath rows | `attachreg.RegisterDatapath` (`internal/plumbing/ebpf/attachreg/register.go:98`) and `ReleaseArgument` (`register.go:223`) from galactic-bgp and `nodevpc`; galactic-cni repair and the `SweepEBPFVRFTable` sweep (`internal/gc/gc.go:749`) | Per-key convention and sweeps; no lock | Worker | galactic-bgp registers with `Exclusive`; Worker commits its own rows as owner. The sweep runs in Worker. | [#1434](https://github.com/datum-cloud/galactic/issues/1434) | -| `egress_route_table` rows | galactic-router (`internal/runtime/gobgp/vrfroutes.go:34`, `vrfmove.go:30`); galactic-bgp through `attachreg` (`register.go:386`, `register.go:460`); the galactic-vrf sidecar (`internal/ingresssidecar/backend.go:144`, `backend.go:159`); galactic-cni installer, repair and prune (`internal/installer/egressgroup.go:203`, `attachreg/fill.go:123`, `internal/plumbing/ebpf/sidecarmap/prune.go:80`); galactic-edge (`attachreg/hostlink.go:204`) | An in-process mutex in galactic-router only (`vrfroutes.go:81`); no cross-process lock | Worker (Fabric module) | Worker's Fabric module writes the rows. galactic-router keeps learning EVPN routes and hands them to Worker. The other writers call Worker, or use `Exclusive` until their Task moves them. | [#1436](https://github.com/datum-cloud/galactic/issues/1436); [#1439](https://github.com/datum-cloud/galactic/issues/1439) for the sidecar | -| Per-VPC kernel routes and rules | VRF unreachable defaults (`vrf.go`); the locator local route (`internal/installer/locatorroute.go`); sidecar-return routes (`internal/installer/sidecarreturn.go`) | None beyond the sidecar-return range from `0xF000` | Worker | Worker commits them as the scope owner. | [#1434](https://github.com/datum-cloud/galactic/issues/1434) | -| Sidecar return path (tables from `0xF000`) and reaping | `ensureSidecarReturnPath` (`internal/installer/sidecarreturn.go:159`); `deleteIngressAdvertisement` and `pruneAllSidecarRows` (`internal/installer/sidecarreap.go:243`, `sidecarreap.go:264`), all in galactic-cni | Evidence from advertisement annotations and the host veth | Worker | The sidecar requests a `socket` claim; Worker programs the return path and withdraws it on release or lease expiry. | [#1439](https://github.com/datum-cloud/galactic/issues/1439) | -| BGPAdvertisement for a VPC on a node | galactic-bgp; the galactic-vrf sidecar (`gateway.go:300`); edge route publication; galactic-cni reaping (`sidecarreap.go:243`); galactic-router GC (`RunGC`, `gc.go:1265`) | CRD optimistic concurrency | Worker | CNI records its containers with `Exclusive`; Worker writes service claims and owns deletion. | [#1434](https://github.com/datum-cloud/galactic/issues/1434), [#1436](https://github.com/datum-cloud/galactic/issues/1436) | -| GC of per-VPC state | `RunGC` (`internal/gc/gc.go:1265`) in galactic-router; `SweepEBPFVRFTable` (`gc.go:749`) and `SweepEBPFNPTv6Table` (`gc.go:1146`) in galactic-cni | VRF lock; the edge fence for edge scopes | Worker | Each sweep runs in Worker as the scope owner, on the same evidence it uses today. | [#1434](https://github.com/datum-cloud/galactic/issues/1434) | -| Edge host links | `RegisterHostLink`, `UnregisterHostLink` and `SyncHostLinkRoutes` (`attachreg/hostlink.go:50`, `:161`, `:203`) from galactic-edge | The edge VPC fence | Worker | galactic-edge calls Worker through its `Datapath` implementation ([connector](connector.md)). | [#1440](https://github.com/datum-cloud/galactic/issues/1440) | -| Attachment interface and its rows | galactic-veth and galactic-tap create and delete the device; `RegisterTenantGateway` (`register.go:632`) from galactic-bgp; `UnregisterInterface` (`attachreg/unregister.go:33`) from CNI DEL and `internal/installer/staletaps.go` | Device private to one attachment | CNI | None. Worker observes the interface and never deletes it. | [#1434](https://github.com/datum-cloud/galactic/issues/1434) for observation | -| Sidecar pod VRFs (tables from `0x10000`) in the pod | `vrf.AddInRange` (`internal/ingresssidecar/backend.go:95`) in the pod's network namespace | Separate network namespace | galactic-vrf sidecar | None. Service-specific state stays in the service agent. | [#1439](https://github.com/datum-cloud/galactic/issues/1439) | -| BGP and EVPN route origination | galactic-router (GoBGP, from CRDs) | One process | galactic-router | None. | None | -| Pinned programs and maps in `/sys/fs/bpf/galactic` | galactic-cni loads and pins them (`internal/plumbing/ebpf/attach/attach.go:31`) | One loader | galactic-cni (loading only; rows follow their class) | None. | None | -| TC attachment on an interface | galactic-cni (`attach.go:499`) | flock `/run/galactic/tc-.lock` | galactic-cni | None. | None | -| XDP dispatcher slots | galactic-gateway (slots 0 and 1) and galactic-nat (slot 2) through `Fill`, `Clear`, `Release`, `SetRole` and `Renew` (`internal/plumbing/ebpf/xdpdispatch/dispatcher.go:678-881`), 8 slots (`dispatcher.go:60`) | Lock on the pin directory and a 120 s lease per slot | The service that holds the slot | None. Worker reads slot state for capability status only. | None | -| `/sys/fs/bpf/galactic-nat`, `galactic-edge`, `galactic-xdp` | galactic-nat; galactic-gateway; both through the dispatcher | One writer each, plus the dispatcher lock | The owning service | None. | None | -| IPAM, FRR configuration | `internal/cni/ipam`; `cmd/fabric-config-agent` | flock each | Unchanged | None. | None | +| Resource class | Writers at the branch start | Coordination today | Authority after transfer | Transfer step | Task | +| ----------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------- | ---------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| Host VRF device and table ID (1 to `0xEFFF`) | `vrf.EnsureAndEnslave` (`internal/plumbing/vrf/vrf.go:116`) from galactic-veth and galactic-tap; `vrf.Add` (`vrf.go:93`) from `nodevpc.Ensure` in galactic-edge; `vrf.DeleteIfUnused` (`vrf.go:302`) from galactic-router GC | flock in `/var/lib/cni/galactic-vrf` (`lock.go:28`), reservations in `tables.json` (`reservation.go:35`) | Worker | CNI writes with `Exclusive`; galactic-edge calls Worker. GC moves into Worker. | [#1434](https://github.com/datum-cloud/galactic/issues/1434) | +| Per-node VPC state, SID Argument and BGPVRFInstance | `nodevpc.Ensure` (`internal/nodevpc/ensure.go:72`) from galactic-edge (`internal/edge/agent/vpcstate.go:30`); `ClaimArgument` (`argument.go:109`) and `AllocateArgument` (`argument.go:239`) from galactic-bgp and galactic-edge; the galactic-vrf sidecar writes BGPVRFInstance directly (`internal/ingresssidecar/gateway.go:273`) | Argument flock (`argument.go:33`); the sidecar's write takes no lock | Worker | Route the sidecar's BGPVRFInstance write through Worker. `argument.go:91` records that a writer outside the lock can still collide. | [#1434](https://github.com/datum-cloud/galactic/issues/1434), [#1439](https://github.com/datum-cloud/galactic/issues/1439) for the sidecar | +| `vrf_table` datapath rows | `attachreg.RegisterDatapath` (`internal/plumbing/ebpf/attachreg/register.go:98`) and `ReleaseArgument` (`register.go:223`) from galactic-bgp and `nodevpc`; galactic-cni repair and the `SweepEBPFVRFTable` sweep (`internal/gc/gc.go:749`) | Per-key convention and sweeps; no lock | Worker | galactic-bgp registers with `Exclusive`; Worker commits its own rows as owner. The sweep runs in Worker. | [#1434](https://github.com/datum-cloud/galactic/issues/1434) | +| `egress_route_table` rows | galactic-router (`internal/runtime/gobgp/vrfroutes.go:34`, `vrfmove.go:30`); galactic-bgp through `attachreg` (`register.go:386`, `register.go:460`); the galactic-vrf sidecar (`internal/ingresssidecar/backend.go:144`, `backend.go:159`); galactic-cni installer, repair and prune (`internal/installer/egressgroup.go:203`, `attachreg/fill.go:123`, `internal/plumbing/ebpf/sidecarmap/prune.go:80`); galactic-edge (`attachreg/hostlink.go:204`) | An in-process mutex in galactic-router only (`vrfroutes.go:81`); no cross-process lock | Worker (Fabric module) | Worker's Fabric module writes the tenant routes of every VPC Worker owns; galactic-router keeps learning EVPN routes and hands them to Worker ([handoff](contract.md#tenant-route-handoff)), writing with `Exclusive` only in a VPC Worker does not own. The installer sweep skips those tenant routes. The other writers call Worker, or use `Exclusive` until their Task moves them. | [#1436](https://github.com/datum-cloud/galactic/issues/1436); [#1439](https://github.com/datum-cloud/galactic/issues/1439) for the sidecar | +| Per-VPC kernel routes and rules | VRF unreachable defaults (`vrf.go`); the locator local route (`internal/installer/locatorroute.go`); sidecar-return routes (`internal/installer/sidecarreturn.go`) | None beyond the sidecar-return range from `0xF000` | Worker | Worker commits them as the scope owner. | [#1434](https://github.com/datum-cloud/galactic/issues/1434) | +| Sidecar return path (tables from `0xF000`) and reaping | `ensureSidecarReturnPath` (`internal/installer/sidecarreturn.go:159`); `deleteIngressAdvertisement` and `pruneAllSidecarRows` (`internal/installer/sidecarreap.go:243`, `sidecarreap.go:264`), all in galactic-cni | Evidence from advertisement annotations and the host veth | Worker | The sidecar requests a `socket` claim; Worker programs the return path and withdraws it on release or lease expiry. | [#1439](https://github.com/datum-cloud/galactic/issues/1439) | +| BGPAdvertisement for a VPC on a node | galactic-bgp; the galactic-vrf sidecar (`gateway.go:300`); edge route publication; galactic-cni reaping (`sidecarreap.go:243`); galactic-router GC (`RunGC`, `gc.go:1265`) | CRD optimistic concurrency | Worker | CNI records its containers with `Exclusive`; Worker writes service claims and owns deletion. | [#1434](https://github.com/datum-cloud/galactic/issues/1434), [#1436](https://github.com/datum-cloud/galactic/issues/1436) | +| GC of per-VPC state | `RunGC` (`internal/gc/gc.go:1265`) in galactic-router; `SweepEBPFVRFTable` (`gc.go:749`) and `SweepEBPFNPTv6Table` (`gc.go:1146`) in galactic-cni | VRF lock; the edge fence for edge scopes | Worker | Each sweep runs in Worker as the scope owner, on the same evidence it uses today. | [#1434](https://github.com/datum-cloud/galactic/issues/1434) | +| Edge host links | `RegisterHostLink`, `UnregisterHostLink` and `SyncHostLinkRoutes` (`attachreg/hostlink.go:50`, `:161`, `:203`) from galactic-edge | The edge VPC fence | Worker | galactic-edge calls Worker through its `Datapath` implementation ([connector](connector.md)). | [#1440](https://github.com/datum-cloud/galactic/issues/1440) | +| Attachment interface and its rows | galactic-veth and galactic-tap create and delete the device; `RegisterTenantGateway` (`register.go:632`) from galactic-bgp; `UnregisterInterface` (`attachreg/unregister.go:33`) from CNI DEL and `internal/installer/staletaps.go` | Device private to one attachment | CNI | None. Worker observes the interface and never deletes it. | [#1434](https://github.com/datum-cloud/galactic/issues/1434) for observation | +| Sidecar pod VRFs (tables from `0x10000`) in the pod | `vrf.AddInRange` (`internal/ingresssidecar/backend.go:95`) in the pod's network namespace | Separate network namespace | galactic-vrf sidecar | None. Service-specific state stays in the service agent. | [#1439](https://github.com/datum-cloud/galactic/issues/1439) | +| BGP and EVPN route origination | galactic-router (GoBGP, from CRDs) | One process | galactic-router | None. | None | +| Pinned programs and maps in `/sys/fs/bpf/galactic` | galactic-cni loads and pins them (`internal/plumbing/ebpf/attach/attach.go:31`) | One loader | galactic-cni (loading only; rows follow their class) | None. | None | +| TC attachment on an interface | galactic-cni (`attach.go:499`) | flock `/run/galactic/tc-.lock` | galactic-cni | None. | None | +| XDP dispatcher slots | galactic-gateway (slots 0 and 1) and galactic-nat (slot 2) through `Fill`, `Clear`, `Release`, `SetRole` and `Renew` (`internal/plumbing/ebpf/xdpdispatch/dispatcher.go:678-881`), 8 slots (`dispatcher.go:60`) | Lock on the pin directory and a 120 s lease per slot | The service that holds the slot | None. Worker reads slot state for capability status only. | None | +| `/sys/fs/bpf/galactic-nat`, `galactic-edge`, `galactic-xdp` | galactic-nat; galactic-gateway; both through the dispatcher | One writer each, plus the dispatcher lock | The owning service | None. | None | +| IPAM, FRR configuration | `internal/cni/ipam`; `cmd/fabric-config-agent` | flock each | Unchanged | None. | None | NAT translations and shard sessions, Gateway VIP selection and Maglev, and Envoy's proxy sockets stay in their service agents. @@ -129,9 +129,25 @@ CNI and garbage collection. reads each attachment's current ifindex and neighbors when a module asks. Worker never creates or deletes an attachment's link. -The router's `egress_route_table` writers (#1436), the galactic-vrf sidecar's -in-pod BGPVRFInstance and backend writes (#1439) and edge host links (#1440) -move in their own Tasks. +[#1436](https://github.com/datum-cloud/galactic/issues/1436) moves the +router's `egress_route_table` writers: + +- **galactic-router writes under the fence.** Every tenant route write, and + each removal after a VRF moves table, runs under the VPC's scope with + `RefuseWorkerOwned`. A VPC no Worker owns is written as before; one a Worker + incarnation owns is left to Worker. +- **Worker's Fabric module writes the tenant routes.** The router hands every + route it installs or withdraws to Worker through the Routes service, and the + Fabric module writes the routes of each VPC Worker owns in that scope's + commit. It never touches an entry the router did not hand it. +- **The installer sweep leaves them alone.** In the table of a VPC a Worker + owns, the `egress_route_table` refresh sweep re-resolves only egress shard + entries and sentinels; the Fabric module keeps the tenant routes' next hops + current. Those shard writes, CNI's pass-through and shard writes and the + repair fill still write without the fence; no Task moves them yet. + +The galactic-vrf sidecar's in-pod BGPVRFInstance and backend writes (#1439) +and edge host links (#1440) move in their own Tasks. ## Defects this transfer removes diff --git a/internal/config/router.go b/internal/config/router.go index c27cb304..4112e0d9 100644 --- a/internal/config/router.go +++ b/internal/config/router.go @@ -88,6 +88,16 @@ const ( // embedded GoBGP server's API in, read-only, as a unix socket for the // gobgp CLI. Empty, the default, serves no API. EnvRouterGoBGPAPIDir = "GALACTIC_ROUTER_GOBGP_API_DIR" + + // EnvRouterWorkerSocket is Galactic Worker's socket, which galactic-router + // hands its tenant routes to. Empty, the default, hands Worker nothing. + EnvRouterWorkerSocket = "GALACTIC_ROUTER_WORKER_SOCKET" + // EnvRouterWorkerTokenFile is the projected ServiceAccount token + // galactic-router presents to Worker. + EnvRouterWorkerTokenFile = "GALACTIC_ROUTER_WORKER_TOKEN_FILE" + // DefaultRouterWorkerTokenFile is EnvRouterWorkerTokenFile's default, the + // path a pod mounts its Worker token at by convention. + DefaultRouterWorkerTokenFile = "/var/run/secrets/galactic-worker/token" ) // --- RouterConfig ---------------------------------------------------------- @@ -130,6 +140,11 @@ type RouterConfig struct { // GoBGPAPIDir is EnvRouterGoBGPAPIDir. GoBGPAPIDir string + + // WorkerSocket is EnvRouterWorkerSocket. + WorkerSocket string + // WorkerTokenFile is EnvRouterWorkerTokenFile. + WorkerTokenFile string } // NewRouterConfig creates a config resolver reading the GALACTIC_ROUTER @@ -155,6 +170,8 @@ func NewRouterConfig() *RouterConfig { v.SetDefault("bmp_policy", DefaultRouterBMPPolicy) v.SetDefault("bmp_statistics_interval", DefaultRouterBMPStatisticsInterval.String()) v.SetDefault("gobgp_api_dir", "") + v.SetDefault("worker_socket", "") + v.SetDefault("worker_token_file", DefaultRouterWorkerTokenFile) cfg := &RouterConfig{ v: v, @@ -186,6 +203,8 @@ func (c *RouterConfig) BindFlags(flags *pflag.FlagSet) { {"bmp-policy", "bmp_policy"}, {"bmp-statistics-interval", "bmp_statistics_interval"}, {"gobgp-api-dir", "gobgp_api_dir"}, + {"worker-socket", "worker_socket"}, + {"worker-token-file", "worker_token_file"}, } for _, b := range bindings { if flags.Changed(b.flag) { @@ -215,6 +234,8 @@ func (c *RouterConfig) readFields() { c.BMPPolicy = c.v.GetString("bmp_policy") c.BMPStatisticsInterval = c.v.GetDuration("bmp_statistics_interval") c.GoBGPAPIDir = c.v.GetString("gobgp_api_dir") + c.WorkerSocket = c.v.GetString("worker_socket") + c.WorkerTokenFile = c.v.GetString("worker_token_file") } // Validate checks that the required configuration fields are set and within diff --git a/internal/config/router_test.go b/internal/config/router_test.go index f2cb70e8..c4fd4c68 100644 --- a/internal/config/router_test.go +++ b/internal/config/router_test.go @@ -58,6 +58,12 @@ func TestRouterConfigDefaults(t *testing.T) { if cfg.GoBGPAPIDir != "" { t.Errorf("GoBGPAPIDir = %q, want empty", cfg.GoBGPAPIDir) } + if cfg.WorkerSocket != "" { + t.Errorf("WorkerSocket = %q, want empty (no route handoff by default)", cfg.WorkerSocket) + } + if cfg.WorkerTokenFile != DefaultRouterWorkerTokenFile { + t.Errorf("WorkerTokenFile = %q, want %q", cfg.WorkerTokenFile, DefaultRouterWorkerTokenFile) + } } func TestRouterConfigEnvOverride(t *testing.T) { @@ -73,8 +79,13 @@ func TestRouterConfigEnvOverride(t *testing.T) { t.Setenv(EnvRouterBMPPolicy, "local-rib") t.Setenv(EnvRouterBMPStatisticsInterval, "0s") t.Setenv(EnvRouterGoBGPAPIDir, "/run/galactic-router") + t.Setenv(EnvRouterWorkerSocket, "/run/galactic-worker/worker.sock") + t.Setenv(EnvRouterWorkerTokenFile, "/token") cfg := NewRouterConfig() + if cfg.WorkerSocket != "/run/galactic-worker/worker.sock" || cfg.WorkerTokenFile != "/token" { + t.Errorf("Worker socket and token = %q, %q", cfg.WorkerSocket, cfg.WorkerTokenFile) + } if cfg.NodeName != testEnvNode { t.Errorf("NodeName = %q, want %q", cfg.NodeName, testEnvNode) diff --git a/internal/installer/egressgroup.go b/internal/installer/egressgroup.go index 99098182..c18af925 100644 --- a/internal/installer/egressgroup.go +++ b/internal/installer/egressgroup.go @@ -697,6 +697,17 @@ func (g *egressGroup) refresh( table *egressroutemap.EgressRouteTable, foreignTableIDs map[uint32]struct{}, shardSIDs []net.IP, classOf map[uint32]egressroutemap.TranslationClass, opts ...egressroutemap.RefreshOption, ) (egressroutemap.RefreshResult, error) { + // Worker's Fabric module keeps the tenant routes of the VPCs it owns + // current. A sweep that cannot tell which those are would rewrite them + // outside their fence, so it is skipped, as for the sidecar's tables. + workerTables, err := workerTenantTablesFn() + if err != nil { + g.ctl.SweepError("refresh") + slog.Error("egress_route_table refresh sweep skipped: could not read which VPCs galactic-worker owns", + "err", err) + return egressroutemap.RefreshResult{}, err + } + opts = append(opts, egressroutemap.WithoutTenantRoutes(workerTables)) result, err := table.Refresh(foreignTableIDs, shardSIDs, opts...) if err != nil { g.ctl.SweepError("refresh") diff --git a/internal/installer/workertables.go b/internal/installer/workertables.go new file mode 100644 index 00000000..b802ce09 --- /dev/null +++ b/internal/installer/workertables.go @@ -0,0 +1,54 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package installer + +import ( + "errors" + "fmt" + "os" + + "go.datum.net/galactic/internal/cni/vpcfence" + "go.datum.net/galactic/internal/gc" + "go.datum.net/galactic/internal/ownership" + "go.datum.net/galactic/internal/plumbing/vrf" +) + +// workerTenantTablesFn returns the VRF tables whose tenant routes a +// galactic-worker incarnation writes. A package var so tests can state them +// without a host VRF or a fence directory. +var workerTenantTablesFn = workerTenantTables + +// workerTenantTables returns the table of every host VRF whose VPC scope a +// galactic-worker incarnation owns. Worker's Fabric module writes those +// VRFs' tenant routes and keeps their next hops current. A node with no fence +// directory has no Worker, and none. +func workerTenantTables() (map[uint32]struct{}, error) { + auth, err := ownership.Open(vpcfence.Dir) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("open VPC write fence: %w", err) + } + links, err := vrf.ListVRFLinks() + if err != nil { + return nil, fmt.Errorf("list VRF links: %w", err) + } + tables := map[uint32]struct{}{} + for _, l := range links { + vpc, ok := gc.VPCFromVRFName(l.Name) + if !ok { + continue + } + rec, err := auth.Read(ownership.VPCScope(vpc)) + if err != nil { + return nil, fmt.Errorf("read VPC %s write fence: %w", vpc, err) + } + if ownership.WorkerOwned(rec.Owner) { + tables[l.Table] = struct{}{} + } + } + return tables, nil +} diff --git a/internal/installer/workertables_test.go b/internal/installer/workertables_test.go new file mode 100644 index 00000000..3083a4fb --- /dev/null +++ b/internal/installer/workertables_test.go @@ -0,0 +1,69 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package installer + +import ( + "errors" + "net" + "testing" + + "go.datum.net/galactic/internal/plumbing/ebpf/egressroutemap" +) + +// workerTables makes the sweep see tables, or err, as the tables whose tenant +// routes galactic-worker owns. +func workerTables(t *testing.T, tables map[uint32]struct{}, err error) { + t.Helper() + prev := workerTenantTablesFn + workerTenantTablesFn = func() (map[uint32]struct{}, error) { return tables, err } + t.Cleanup(func() { workerTenantTablesFn = prev }) +} + +func TestSweepLeavesWorkerTenantRoutesAlone(t *testing.T) { + f := newFixture(t, testOrdered, []string{shardSIDFor(0x10)}, nil) + _, tenant, _ := net.ParseCIDR("2001:db8:a::2/128") + sid := net.ParseIP(shardSIDFor(0x20)) // a remote node, not a shard + for _, table := range []uint32{1, 2} { + if err := f.routes.Register(table, tenant, sid); err != nil { + t.Fatal(err) + } + } + workerTables(t, map[uint32]struct{}{1: {}}, nil) + restore := egressroutemap.OverrideResolverForTest(func(net.IP) (int, net.HardwareAddr, net.HardwareAddr, error) { + return 9, net.HardwareAddr{2, 0, 0, 0, 0, 9}, net.HardwareAddr{2, 0, 0, 0, 0, 2}, nil + }) + t.Cleanup(restore) + + if err := f.sweep(t); err != nil { + t.Fatalf("sweep = %v", err) + } + for table, want := range map[uint32]int{1: 7, 2: 9} { + row, ok, err := f.routes.LookupRow(table, tenant) + if err != nil || !ok || row.LinkIndex != want { + t.Errorf("table %d tenant route = %+v (%t, %v), want link %d", table, row, ok, err, want) + } + } +} + +func TestSweepSkippedWhenWorkerOwnershipIsUnreadable(t *testing.T) { + f := newFixture(t, testOrdered, []string{shardSIDFor(0x10)}, nil) + _, tenant, _ := net.ParseCIDR("2001:db8:a::2/128") + if err := f.routes.Register(1, tenant, net.ParseIP(shardSIDFor(0x20))); err != nil { + t.Fatal(err) + } + errFence := errors.New("fence unreadable") + workerTables(t, nil, errFence) + restore := egressroutemap.OverrideResolverForTest(func(net.IP) (int, net.HardwareAddr, net.HardwareAddr, error) { + return 9, net.HardwareAddr{2, 0, 0, 0, 0, 9}, net.HardwareAddr{2, 0, 0, 0, 0, 2}, nil + }) + t.Cleanup(restore) + + if err := f.sweep(t); !errors.Is(err, errFence) { + t.Fatalf("sweep = %v, want the fence error", err) + } + if row, _, _ := f.routes.LookupRow(1, tenant); row.LinkIndex != 7 { + t.Errorf("tenant route rewritten to link %d by a sweep that could not read ownership", row.LinkIndex) + } +} diff --git a/internal/instrumentation/lint_test.go b/internal/instrumentation/lint_test.go index 1a60f4b6..8b15830b 100644 --- a/internal/instrumentation/lint_test.go +++ b/internal/instrumentation/lint_test.go @@ -38,6 +38,7 @@ import ( _ "go.datum.net/galactic/internal/runtime/gobgp" _ "go.datum.net/galactic/internal/serviceroute" _ "go.datum.net/galactic/internal/worker" + _ "go.datum.net/galactic/internal/worker/fabric" ) const ( diff --git a/internal/plumbing/ebpf/egressroutemap/egressroute.go b/internal/plumbing/ebpf/egressroutemap/egressroute.go index 061e6a2a..8150f556 100644 --- a/internal/plumbing/ebpf/egressroutemap/egressroute.go +++ b/internal/plumbing/ebpf/egressroutemap/egressroute.go @@ -165,27 +165,63 @@ var uplinkIndexesFn = attach.UplinkIndexes // unreachable one to fail: with a default route in place, all of them // "resolve". func resolveLinkAndL2(sid net.IP) (linkIndex int, dmac, smac net.HardwareAddr, err error) { + hop, err := resolveNextHop(sid, true) + if err != nil { + return 0, nil, nil, err + } + return hop.LinkIndex, hop.DMAC, hop.SMAC, nil +} + +// ErrNoUplinkRoute is a SID with no specific route over an SRv6 uplink. +var ErrNoUplinkRoute = errors.New("no route over an SRv6 uplink") + +// ErrNeighborUnresolved is a SID whose underlay next hop has no resolved +// neighbor entry. +var ErrNeighborUnresolved = errors.New("next hop neighbor unresolved") + +// NextHop is where a packet encapsulated toward a SID leaves the node. +type NextHop struct { + // LinkIndex is the SRv6 uplink the route to the SID leaves through. + LinkIndex int + // Gateway is the underlay next hop, the SID itself when it is on-link. + Gateway net.IP + // DMAC and SMAC are the next hop's and the uplink's MAC addresses. + DMAC, SMAC net.HardwareAddr +} + +// ResolveNextHop resolves sid's underlay next hop from the kernel's routes and +// neighbor cache, as Register does, without soliciting the neighbor. The error +// wraps ErrNoUplinkRoute when no specific route over an SRv6 uplink reaches +// sid, and ErrNeighborUnresolved when the next hop has no resolved neighbor; +// with the latter, the returned NextHop still names the link and gateway. +func ResolveNextHop(sid net.IP) (NextHop, error) { + return resolveNextHop(sid, false) +} + +// resolveNextHop resolves sid's next hop, soliciting the neighbor when solicit +// is set. +func resolveNextHop(sid net.IP, solicit bool) (NextHop, error) { uplinks, err := uplinkIndexesFn() if err != nil { - return 0, nil, nil, fmt.Errorf("resolve SRv6 uplinks to validate the route to %s: %w", sid, err) + return NextHop{}, fmt.Errorf("resolve SRv6 uplinks to validate the route to %s: %w", sid, err) } routes, err := netlink.RouteGet(sid) if err != nil { - return 0, nil, nil, fmt.Errorf("no route to %s: %w", sid, err) + return NextHop{}, fmt.Errorf("%w: no route to %s: %w", ErrNoUplinkRoute, sid, err) } if len(routes) == 0 { - return 0, nil, nil, fmt.Errorf("no route to %s", sid) + return NextHop{}, fmt.Errorf("%w: no route to %s", ErrNoUplinkRoute, sid) } - linkIndex = routes[0].LinkIndex + linkIndex := routes[0].LinkIndex if _, ok := uplinks[linkIndex]; !ok { - return 0, nil, nil, fmt.Errorf( - "no route to %s over an SRv6 uplink: it resolves out link %d, which the datapath is not attached to "+ - "(the fabric has most likely not advertised this SID yet and the lookup fell through to a "+ - "default route)", sid, linkIndex) + return NextHop{}, fmt.Errorf( + "%w: no route to %s over an SRv6 uplink: it resolves out link %d, which the datapath is not "+ + "attached to (the fabric has most likely not advertised this SID yet and the lookup fell "+ + "through to a default route)", ErrNoUplinkRoute, sid, linkIndex) } if err := rejectDefaultRoute(sid); err != nil { - return 0, nil, nil, err + return NextHop{}, fmt.Errorf("%w: %w", ErrNoUplinkRoute, err) } nextHop := routes[0].Gw if nextHop == nil { @@ -194,15 +230,23 @@ func resolveLinkAndL2(sid net.IP) (linkIndex int, dmac, smac net.HardwareAddr, e link, err := netlink.LinkByIndex(linkIndex) if err != nil { - return 0, nil, nil, fmt.Errorf("look up link %d: %w", linkIndex, err) + return NextHop{}, fmt.Errorf("look up link %d: %w", linkIndex, err) } - smac = link.Attrs().HardwareAddr + hop := NextHop{LinkIndex: linkIndex, Gateway: nextHop, SMAC: link.Attrs().HardwareAddr} - dmac, err = resolveNeighbor(linkIndex, nextHop) + if solicit { + hop.DMAC, err = resolveNeighbor(linkIndex, nextHop) + } else { + hop.DMAC, err = lookupNeighbor(linkIndex, nextHop) + if err == nil && hop.DMAC == nil { + err = fmt.Errorf("no resolved neighbor entry for %s on link %d", nextHop, linkIndex) + } + } if err != nil { - return 0, nil, nil, fmt.Errorf("%w (next-hop for sid %s)", err, sid) + hop.DMAC = nil + return hop, fmt.Errorf("%w: %w (next-hop for sid %s)", ErrNeighborUnresolved, err, sid) } - return linkIndex, dmac, smac, nil + return hop, nil } // rejectDefaultRoute fails when the route sid matches is a default route, @@ -519,6 +563,72 @@ func (t *EgressRouteTable) Lookup(tableID uint32, prefix *net.IPNet) (sid net.IP return sid, true, nil } +// Row is the egress_route_table entry a lookup answered with. +type Row struct { + // SID is the entry's SID: the destination of an encapsulating entry. + SID net.IP + // LinkIndex, DMAC and SMAC are the next hop stored with an encapsulating + // entry. + LinkIndex int + DMAC, SMAC net.HardwareAddr + // PassThrough is an entry usid_egress defers to the kernel. + PassThrough bool + // Sentinel is an entry naming an egress shard group. + Sentinel bool +} + +// LookupRow is Lookup returning the whole entry: the entry egress_route_table +// resolves (tableID, prefix) to with a longest-prefix match bounded by +// prefix's length, as usid_egress does for a packet to prefix's address. +func (t *EgressRouteTable) LookupRow(tableID uint32, prefix *net.IPNet) (Row, bool, error) { + key, err := buildKey(tableID, prefix) + if err != nil { + return Row{}, false, fmt.Errorf("egressroutemap: egress_route_table: lookup: %w", err) + } + var value prog.UsidEgressRouteValue + if err := t.table.Lookup(key, &value); err != nil { + if errors.Is(err, ebpf.ErrKeyNotExist) { + return Row{}, false, nil + } + return Row{}, false, fmt.Errorf("egressroutemap: egress_route_table: lookup table=%d prefix=%s: %w", + tableID, prefix, err) + } + row := Row{ + SID: append(net.IP(nil), value.Sid[:]...), + LinkIndex: int(value.LinkIfindex), + DMAC: append(net.HardwareAddr(nil), value.Dmac[:]...), + SMAC: append(net.HardwareAddr(nil), value.Smac[:]...), + PassThrough: value.LinkIfindex == 0, + Sentinel: isGroupSentinel(value), + } + return row, true, nil +} + +// UnregisterIfSID removes the entry of its own that (tableID, prefix) has +// when it encapsulates toward sid, or toward any SID when sid is nil. A +// pass-through or sentinel entry is always kept. The entry is read just +// before it is deleted, as the map offers no compare-and-delete. +func (t *EgressRouteTable) UnregisterIfSID(tableID uint32, prefix *net.IPNet, sid net.IP) error { + key, err := buildKey(tableID, prefix) + if err != nil { + return fmt.Errorf("egressroutemap: egress_route_table: unregister: %w", err) + } + var value prog.UsidEgressRouteValue + switch err := t.table.Lookup(key, &value); { + case errors.Is(err, ebpf.ErrKeyNotExist): + return nil + case err != nil: + return fmt.Errorf("egressroutemap: egress_route_table: unregister table=%d prefix=%s: %w", tableID, prefix, err) + case value.LinkIfindex == 0 || isGroupSentinel(value): + return nil + case sid != nil && !net.IP(value.Sid[:]).Equal(sid): + return nil + } + // The lookup is longest-prefix, so the entry read may be a shorter one + // covering prefix; deleting the exact key then finds nothing. + return t.Unregister(tableID, prefix) +} + // Prefixes returns every prefix egress_route_table holds an entry of its own // for, in CIDR string form, keyed by Linux VRF table ID. It walks the whole // map once. Unlike Lookup it never answers for one prefix with a shorter one. @@ -771,6 +881,19 @@ type RefreshOption func(*refreshConfig) type refreshConfig struct { groupFor func(prefix *net.IPNet) (uint32, bool) layout *ClassLayout + // tenantRouteTables are tables whose tenant routes another writer + // re-resolves. + tenantRouteTables map[uint32]struct{} +} + +// WithoutTenantRoutes leaves the tenant routes of tables alone: in each, the +// sweep refreshes only egress shard entries and sentinels, and skips every +// other encapsulating entry. Galactic Worker's Fabric module writes and +// re-resolves the tenant routes of the VPCs it owns. +func WithoutTenantRoutes(tables map[uint32]struct{}) RefreshOption { + return func(c *refreshConfig) { + c.tenantRouteTables = tables + } } // ClassLayout is the hashed-mode assignment of translation classes to shard @@ -938,6 +1061,13 @@ func (t *EgressRouteTable) Refresh( sid := make(net.IP, 16) copy(sid, value.Sid[:]) + if _, tenant := cfg.tenantRouteTables[key.TableId]; tenant { + if fields, err := decodeSID(sid); err != nil || !shards.isShard(fields) { + result.Skipped++ + continue // a tenant route its owner keeps current + } + } + if cfg.groupFor != nil { if fields, err := decodeSID(sid); err == nil && shards.isShard(fields) { if groupID, ok := cfg.groupFor(keyPrefix(key)); ok { diff --git a/internal/plumbing/ebpf/egressroutemap/tenantroutes_test.go b/internal/plumbing/ebpf/egressroutemap/tenantroutes_test.go new file mode 100644 index 00000000..ddb1bb70 --- /dev/null +++ b/internal/plumbing/ebpf/egressroutemap/tenantroutes_test.go @@ -0,0 +1,102 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package egressroutemap + +import ( + "net" + "testing" +) + +// TestRefreshLeavesWorkerTenantRoutesAlone: in a table whose tenant routes +// galactic-worker keeps current, the sweep still refreshes the egress shard +// entry but leaves the tenant route alone; elsewhere it refreshes both. +func TestRefreshLeavesWorkerTenantRoutesAlone(t *testing.T) { + stubResolveLinkAndL2(t) + tbl := NewEgressRouteTable(newFakeTable()) + tenantSID := net.ParseIP("2001:db8:ff02:9:e001::") + shardSID := net.ParseIP("2001:db8:ff02:a:e001::") + _, tenant, _ := net.ParseCIDR("2001:db8:a::2/128") + const workerTable, otherTable = 1, 2 + for _, table := range []uint32{workerTable, otherTable} { + if err := tbl.Register(table, tenant, tenantSID); err != nil { + t.Fatal(err) + } + if err := tbl.Register(table, DefaultPrefix, shardSID); err != nil { + t.Fatal(err) + } + } + moved := answer{link: 1060, dmac: net.HardwareAddr{0xAA, 0, 0, 0, 0, 1}, smac: net.HardwareAddr{0xAA, 0, 0, 0, 0, 2}} + resolverFor(t, map[string]answer{tenantSID.String(): moved, shardSID.String(): moved}) + + result, err := tbl.Refresh(nil, []net.IP{shardSID}, + WithoutTenantRoutes(map[uint32]struct{}{workerTable: {}})) + if err != nil { + t.Fatalf("Refresh() = %v", err) + } + if result.Skipped != 1 || result.Refreshed != 3 { + t.Errorf("Refresh() = %+v, want 1 skipped and 3 refreshed", result) + } + for _, c := range []struct { + table uint32 + dst *net.IPNet + link int + }{ + {workerTable, tenant, fakeLinkIndex}, + {workerTable, DefaultPrefix, moved.link}, + {otherTable, tenant, moved.link}, + {otherTable, DefaultPrefix, moved.link}, + } { + row, ok, err := tbl.LookupRow(c.table, c.dst) + if err != nil || !ok { + t.Fatalf("LookupRow(%d, %s) = %v, %t", c.table, c.dst, err, ok) + } + if row.LinkIndex != c.link { + t.Errorf("table %d %s link = %d, want %d", c.table, c.dst, row.LinkIndex, c.link) + } + } +} + +func TestUnregisterIfSID(t *testing.T) { + stubResolveLinkAndL2(t) + tbl := NewEgressRouteTable(newFakeTable()) + sid := net.ParseIP("2001:db8:ff02:9:e001::") + other := net.ParseIP("2001:db8:ff02:a:e001::") + _, a, _ := net.ParseCIDR("2001:db8:a::2/128") + _, b, _ := net.ParseCIDR("2001:db8:a::3/128") + _, pt, _ := net.ParseCIDR("2001:db8:a::4/128") + for _, p := range []*net.IPNet{a, b} { + if err := tbl.Register(1, p, sid); err != nil { + t.Fatal(err) + } + } + if err := tbl.RegisterPassThrough(1, pt); err != nil { + t.Fatal(err) + } + + if err := tbl.UnregisterIfSID(1, a, other); err != nil { + t.Fatal(err) + } + if _, ok, _ := tbl.LookupRow(1, a); !ok { + t.Error("entry toward another SID removed") + } + if err := tbl.UnregisterIfSID(1, a, sid); err != nil { + t.Fatal(err) + } + if _, ok, _ := tbl.LookupRow(1, a); ok { + t.Error("entry toward the SID kept") + } + if err := tbl.UnregisterIfSID(1, b, nil); err != nil { + t.Fatal(err) + } + if _, ok, _ := tbl.LookupRow(1, b); ok { + t.Error("entry kept by a removal for any SID") + } + if err := tbl.UnregisterIfSID(1, pt, nil); err != nil { + t.Fatal(err) + } + if row, ok, _ := tbl.LookupRow(1, pt); !ok || !row.PassThrough { + t.Error("pass-through entry removed") + } +} diff --git a/internal/runtime/gobgp/runtime.go b/internal/runtime/gobgp/runtime.go index 49518015..116d5360 100644 --- a/internal/runtime/gobgp/runtime.go +++ b/internal/runtime/gobgp/runtime.go @@ -74,6 +74,10 @@ type GoBGPRuntime struct { // it, so the shared watcher dispatches a best-path event in constant time // rather than scanning every VRF. A node can host thousands. rtIndex map[string]uint32 + // tableVPC maps each table in rtIndex to the VPC whose VRF uses it, + // guarded by rtIndexMu. Tenant routes are fenced and handed to + // galactic-worker by VPC. + tableVPC map[uint32]string // appliedAdvertisements holds the EVPN advertisements, by name, whose // routes the last successful applyEVPN converged, for status reporting. advertisementMu sync.Mutex @@ -147,6 +151,15 @@ type FactoryOption func(*factoryOptions) type factoryOptions struct { apiDir string + sink TenantRouteSink + fence VPCFence +} + +// WithWorkerHandoff hands every tenant route each created runtime installs or +// withdraws to sink, and writes each route under fence, leaving a VPC that +// galactic-worker owns to Worker. A nil sink or fence leaves that part out. +func WithWorkerHandoff(sink TenantRouteSink, fence VPCFence) FactoryOption { + return func(o *factoryOptions) { o.sink, o.fence = sink, fence } } // WithAPIDir serves each created runtime's GoBGP API, read-only, on a unix @@ -178,7 +191,7 @@ func NewRuntimeFactory( } return func(key types.NamespacedName) (runtime.RouterRuntime, error) { router := key.String() - return &GoBGPRuntime{ + r := &GoBGPRuntime{ key: key, server: newServer(Config{APISocket: apiSocketPath(fo.apiDir, key)}), listenPort: listenPort, @@ -192,16 +205,26 @@ func NewRuntimeFactory( appliedVRFImportRTs: make(map[string][]string), vrfStatus: make(map[string]model.VRFStatus), rtIndex: make(map[string]uint32), + tableVPC: make(map[uint32]string), appliedAdvertisements: make(map[string]model.DesiredAdvertisement), appliedRoutes: make(map[evpnRouteKey]evpnRoute), plain: newPlainRoutes(installFailingReporter(router, "plain")), vrf: newVRFRoutes(installFailingReporter(router, "vrf")), observer: observer, bmp: newBMPKeeper(bmp, key.String()), - }, nil + } + r.vrf.vpcOf, r.vrf.sink, r.vrf.fence = r.vpcOfTable, fo.sink, fo.fence + return r, nil } } +// vpcOfTable returns the VPC whose VRF uses tableID, or "". +func (r *GoBGPRuntime) vpcOfTable(tableID uint32) string { + r.rtIndexMu.RLock() + defer r.rtIndexMu.RUnlock() + return r.tableVPC[tableID] +} + // Apply converges the running GoBGP instance toward desired. func (r *GoBGPRuntime) Apply(ctx context.Context, desired model.DesiredRouter) error { r.mu.Lock() @@ -396,7 +419,11 @@ func (r *GoBGPRuntime) applyVRFs( if _, ok := desired[name]; !ok { deleteVRF(ctx, b, name) if r.vrf != nil { - r.vrf.forgetTable(r.appliedVRFs[name]) + vpc, err := vrfVPC(name) + if err != nil { + vpc = "" + } + r.vrf.forgetTable(vpc, r.appliedVRFs[name]) } delete(r.appliedVRFs, name) delete(r.appliedVRFImportRTs, name) @@ -410,6 +437,7 @@ func (r *GoBGPRuntime) applyVRFs( } rtIndex := make(map[string]uint32, len(vrfs)) + tableVPC := make(map[uint32]string, len(vrfs)) needsBackfill := false var moved []movedVRFTable @@ -471,10 +499,14 @@ func (r *GoBGPRuntime) applyVRFs( for _, rt := range v.ImportRouteTargets { rtIndex[rt] = tableID } + if vpc, err := vrfVPC(v.Name); err == nil { + tableVPC[tableID] = vpc + } } r.rtIndexMu.Lock() r.rtIndex = rtIndex + r.tableVPC = tableVPC r.rtIndexMu.Unlock() // Only once the index no longer names a moved VRF's old table, so the diff --git a/internal/runtime/gobgp/vrfmove.go b/internal/runtime/gobgp/vrfmove.go index 3fdfc1fd..cd1c56ce 100644 --- a/internal/runtime/gobgp/vrfmove.go +++ b/internal/runtime/gobgp/vrfmove.go @@ -37,7 +37,11 @@ var vrfRouteDelIfSID = srv6.RouteEgressDelIfSID // dropped. func (r *GoBGPRuntime) withdrawMovedVRFRoutes(b *gobgpserver.BgpServer, m movedVRFTable) { if r.vrf != nil { - r.vrf.forgetTable(m.tableID) + vpc, err := vrfVPC(m.name) + if err != nil { + vpc = "" + } + r.vrf.forgetTable(vpc, m.tableID) } rts := make(map[string]struct{}, len(m.importRTs)) for _, rt := range m.importRTs { @@ -61,7 +65,9 @@ func (r *GoBGPRuntime) withdrawMovedVRFRoutes(b *gobgpserver.BgpServer, m movedV continue } prefix := addrToIPNet(ipPrefix.IPPrefix, int(ipPrefix.IPPrefixLength)) - if err := vrfRouteDelIfSID(prefix, m.tableID, evpnGateway(path.Attrs)); err != nil { + if err := r.writeMovedVRF(m.name, func() error { + return vrfRouteDelIfSID(prefix, m.tableID, evpnGateway(path.Attrs)) + }); err != nil { slog.Error("withdrawMovedVRFRoutes: route delete failed", "vrf", m.name, "prefix", prefix, "table", m.tableID, "err", err) continue @@ -92,3 +98,17 @@ func hasRouteTarget(attrs []bgp.PathAttributeInterface, rts map[string]struct{}) } return false } + +// writeMovedVRF runs mutate under the write fence of the VPC of the VRF named +// vrfName. galactic-worker removes the old table's entries of a VPC it owns +// itself, so mutate does not run then. +func (r *GoBGPRuntime) writeMovedVRF(vrfName string, mutate func() error) error { + if r.vrf == nil { + return mutate() + } + vpc, err := vrfVPC(vrfName) + if err != nil { + vpc = "" + } + return r.vrf.write(vpc, mutate) +} diff --git a/internal/runtime/gobgp/vrfroutes.go b/internal/runtime/gobgp/vrfroutes.go index 07664a73..a993a89f 100644 --- a/internal/runtime/gobgp/vrfroutes.go +++ b/internal/runtime/gobgp/vrfroutes.go @@ -6,6 +6,7 @@ package gobgp import ( "context" + "errors" "log/slog" "maps" "net" @@ -16,6 +17,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "go.miloapis.com/instrumentation/metrics" + "go.datum.net/galactic/internal/ownership" "go.datum.net/galactic/internal/plumbing/srv6" ) @@ -57,6 +59,29 @@ func installFailingReporter(router, kind string) func(int) { return func(n int) { gauge.Set(float64(n)) } } +// TenantRouteSink receives every tenant route the router installs or +// withdraws, by VPC. galactic-worker's route handoff implements it. +type TenantRouteSink interface { + Upsert(vpc string, tableID uint32, prefix netip.Prefix, sid net.IP) + Withdraw(vpc string, prefix netip.Prefix) + // ForgetTable withdraws every route of vpc it holds in tableID, for a VRF + // removed or moved to another table. + ForgetTable(vpc string, tableID uint32) +} + +// VPCFence runs mutate under vpc's write fence. It returns an error wrapping +// ownership.ErrWorkerOwned, and runs nothing, when a galactic-worker +// incarnation owns vpc's scope; Worker then writes the VPC's tenant routes +// from what the sink handed it. +type VPCFence func(vpc string, mutate func() error) error + +// OwnershipFence returns the VPCFence of auth, the node's VPC write fence. +func OwnershipFence(auth *ownership.Authority) VPCFence { + return func(vpc string, mutate func() error) error { + return auth.ExclusiveIf(ownership.VPCScope(vpc), ownership.RefuseWorkerOwned, mutate) + } +} + // vrfRouteKey names one tenant route: its prefix in one VRF's table. type vrfRouteKey struct { tableID uint32 @@ -81,6 +106,17 @@ type vrfRoutes struct { mu sync.Mutex // pending maps each route whose last install failed to its gateway. pending map[vrfRouteKey]net.IP + // pendingDel holds each route whose last removal failed. + pendingDel map[vrfRouteKey]struct{} + + // vpcOf names the VPC whose VRF uses a table, or "" when it is unknown. + // sink, when set, is handed every route installed or withdrawn. fence, + // when set, guards every write; a write it refuses because Worker owns + // the VPC is Worker's to make. Each is nil when the router runs without + // Worker or the fence. + vpcOf func(tableID uint32) string + sink TenantRouteSink + fence VPCFence // report, when non-nil, is given len(pending) after every change. report func(int) @@ -93,10 +129,11 @@ type vrfRoutes struct { func newVRFRoutes(report func(int)) *vrfRoutes { return &vrfRoutes{ - pending: make(map[vrfRouteKey]net.IP), - report: report, - retry: vrfRouteRetry, - debounce: vrfRouteDebounce, + pending: make(map[vrfRouteKey]net.IP), + pendingDel: make(map[vrfRouteKey]struct{}), + report: report, + retry: vrfRouteRetry, + debounce: vrfRouteDebounce, } } @@ -107,7 +144,8 @@ func (v *vrfRoutes) install(key vrfRouteKey, gw net.IP) { v.installLocked(key, gw) } -// withdraw stops retrying key and removes its route. +// withdraw stops retrying key and removes its route. A removal that fails is +// retried with the pending installs. func (v *vrfRoutes) withdraw(key vrfRouteKey) error { v.mu.Lock() defer v.mu.Unlock() @@ -115,13 +153,50 @@ func (v *vrfRoutes) withdraw(key vrfRouteKey) error { delete(v.pending, key) v.reportLocked() } - return vrfRouteDel(prefixToIPNet(key.prefix), key.tableID) + vpc := v.vpc(key.tableID) + if v.sink != nil && vpc != "" { + v.sink.Withdraw(vpc, key.prefix) + } + err := v.write(vpc, func() error { return vrfRouteDel(prefixToIPNet(key.prefix), key.tableID) }) + if err != nil { + v.pendingDel[key] = struct{}{} + return err + } + delete(v.pendingDel, key) + return nil +} + +// vpc returns the VPC of tableID, or "". +func (v *vrfRoutes) vpc(tableID uint32) string { + if v.vpcOf == nil { + return "" + } + return v.vpcOf(tableID) } -// forgetTable stops retrying every route in tableID, for a VRF being removed. -func (v *vrfRoutes) forgetTable(tableID uint32) { +// write runs mutate under vpc's fence. A write the fence refuses because +// Worker owns the VPC succeeds without running, since Worker writes it from +// what the sink was handed. +func (v *vrfRoutes) write(vpc string, mutate func() error) error { + if v.fence == nil || vpc == "" { + return mutate() + } + err := v.fence(vpc, mutate) + if errors.Is(err, ownership.ErrWorkerOwned) { + slog.Debug("vrfRoutes: galactic-worker owns the VPC; leaving the route to it", "vpc", vpc) + return nil + } + return err +} + +// forgetTable stops retrying every route in tableID, for a VRF of vpc being +// removed or moved, and withdraws the routes handed to the sink for it. +func (v *vrfRoutes) forgetTable(vpc string, tableID uint32) { v.mu.Lock() defer v.mu.Unlock() + if v.sink != nil && vpc != "" { + v.sink.ForgetTable(vpc, tableID) + } changed := false for key := range v.pending { if key.tableID == tableID { @@ -129,6 +204,11 @@ func (v *vrfRoutes) forgetTable(tableID uint32) { changed = true } } + for key := range v.pendingDel { + if key.tableID == tableID { + delete(v.pendingDel, key) + } + } if changed { v.reportLocked() } @@ -137,7 +217,14 @@ func (v *vrfRoutes) forgetTable(tableID uint32) { // installLocked logs the start and end of a failure rather than every attempt. func (v *vrfRoutes) installLocked(key vrfRouteKey, gw net.IP) { _, wasFailing := v.pending[key] - if err := vrfRouteAdd(prefixToIPNet(key.prefix), gw, key.tableID); err != nil { + delete(v.pendingDel, key) + vpc := v.vpc(key.tableID) + if v.sink != nil && vpc != "" { + v.sink.Upsert(vpc, key.tableID, key.prefix, gw) + } + if err := v.write(vpc, func() error { + return vrfRouteAdd(prefixToIPNet(key.prefix), gw, key.tableID) + }); err != nil { if !wasFailing { slog.Warn("vrfRoutes: route install failed; retrying as the underlay changes", "prefix", key.prefix, "gw", gw, "table", key.tableID, "err", err) @@ -177,6 +264,21 @@ func (v *vrfRoutes) retryPending() { } v.mu.Unlock() } + + v.mu.Lock() + deletes := maps.Clone(v.pendingDel) + v.mu.Unlock() + for key := range deletes { + v.mu.Lock() + if _, ok := v.pendingDel[key]; ok { + if err := v.write(v.vpc(key.tableID), func() error { + return vrfRouteDel(prefixToIPNet(key.prefix), key.tableID) + }); err == nil { + delete(v.pendingDel, key) + } + } + v.mu.Unlock() + } } // run retries pending routes until ctx is done. diff --git a/internal/runtime/gobgp/vrfroutes_test.go b/internal/runtime/gobgp/vrfroutes_test.go index 7fc6aa63..ce5ed19c 100644 --- a/internal/runtime/gobgp/vrfroutes_test.go +++ b/internal/runtime/gobgp/vrfroutes_test.go @@ -168,7 +168,7 @@ func TestVRFRoutes_ForgetTable(t *testing.T) { v.install(tenantRoute, tenantSID) v.install(otherVRF, tenantSID) - v.forgetTable(tenantRoute.tableID) + v.forgetTable("", tenantRoute.tableID) if failing != 1 { t.Errorf("failing = %d after forgetting one of two VRFs, want 1", failing) } diff --git a/internal/runtime/gobgp/vrfroutes_worker_test.go b/internal/runtime/gobgp/vrfroutes_worker_test.go new file mode 100644 index 00000000..ae167c91 --- /dev/null +++ b/internal/runtime/gobgp/vrfroutes_worker_test.go @@ -0,0 +1,164 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package gobgp + +import ( + "errors" + "fmt" + "net" + "net/netip" + "sync" + "testing" + + "go.datum.net/galactic/internal/ownership" +) + +// recordingSink records what the router hands Worker. +type recordingSink struct { + mu sync.Mutex + upserts []string + withdrawals []string + forgotten []string +} + +func (s *recordingSink) ForgetTable(vpc string, tableID uint32) { + s.mu.Lock() + defer s.mu.Unlock() + s.forgotten = append(s.forgotten, fmt.Sprintf("%s %d", vpc, tableID)) +} + +func (s *recordingSink) Upsert(vpc string, tableID uint32, prefix netip.Prefix, sid net.IP) { + s.mu.Lock() + defer s.mu.Unlock() + s.upserts = append(s.upserts, fmt.Sprintf("%s %d %s %s", vpc, tableID, prefix, sid)) +} + +func (s *recordingSink) Withdraw(vpc string, prefix netip.Prefix) { + s.mu.Lock() + defer s.mu.Unlock() + s.withdrawals = append(s.withdrawals, vpc+" "+prefix.String()) +} + +const ( + workerVPC = "vpcW" + otherVPC = "vpcX" + otherRoute = 43 +) + +// fencedRoutes is a vrfRoutes over f whose table 42 belongs to a VPC Worker +// owns and table 43 to one it does not. +func fencedRoutes(f VPCFence, sink TenantRouteSink) (*vrfRoutes, *int) { + failing := 0 + v := newVRFRoutes(func(n int) { failing = n }) + v.vpcOf = func(tableID uint32) string { + switch tableID { + case tenantRoute.tableID: + return workerVPC + case otherRoute: + return otherVPC + } + return "" + } + v.fence, v.sink = f, sink + return v, &failing +} + +func TestVRFRoutes_WorkerOwnedVPCIsLeftToWorker(t *testing.T) { + f := newFakeEgressTable(t) + f.setUsable(tenantSID) + var fenced []string + fence := func(vpc string, mutate func() error) error { + fenced = append(fenced, vpc) + if vpc == workerVPC { + return fmt.Errorf("write: %w", ownership.ErrWorkerOwned) + } + return mutate() + } + sink := &recordingSink{} + v, failing := fencedRoutes(fence, sink) + + v.install(tenantRoute, tenantSID) + if f.addCount() != 0 { + t.Fatal("router wrote a route in a VPC Worker owns") + } + if *failing != 0 { + t.Fatalf("a route Worker writes is counted failing: %d", *failing) + } + if err := v.withdraw(tenantRoute); err != nil { + t.Fatalf("withdraw: %v", err) + } + if len(v.pendingDel) != 0 { + t.Fatal("a withdrawal Worker makes is pending") + } + + other := vrfRouteKey{tableID: otherRoute, prefix: tenantRoute.prefix} + v.install(other, tenantSID) + if _, ok := f.entry(other); !ok { + t.Fatal("route in a VPC Worker does not own was not written") + } + if want := []string{workerVPC, workerVPC, otherVPC}; fmt.Sprint(fenced) != fmt.Sprint(want) { + t.Errorf("fenced writes = %v, want %v", fenced, want) + } + if len(sink.upserts) != 2 || len(sink.withdrawals) != 1 || + sink.upserts[0] != "vpcW 42 2001:db8:100::/64 fc00:0:a:42::" { + t.Errorf("handed Worker upserts %v and withdrawals %v", sink.upserts, sink.withdrawals) + } + + // A VRF removed or moved off its table withdraws what Worker was handed. + v.forgetTable(workerVPC, tenantRoute.tableID) + if fmt.Sprint(sink.forgotten) != "[vpcW 42]" { + t.Errorf("forgotten tables = %v, want [vpcW 42]", sink.forgotten) + } +} + +func TestVRFRoutes_RetriesAFencedWithdrawal(t *testing.T) { + f := newFakeEgressTable(t) + f.setUsable(tenantSID) + moving := true + fence := func(_ string, mutate func() error) error { + if moving { + return ownership.ErrTransitioning + } + return mutate() + } + other := vrfRouteKey{tableID: otherRoute, prefix: tenantRoute.prefix} + v, _ := fencedRoutes(fence, nil) + moving = false + v.install(other, tenantSID) + moving = true + if err := v.withdraw(other); !errors.Is(err, ownership.ErrTransitioning) { + t.Fatalf("withdraw = %v, want ErrTransitioning", err) + } + if _, ok := f.entry(other); !ok { + t.Fatal("entry removed while the fence refused") + } + moving = false + v.retryPending() + if _, ok := f.entry(other); ok { + t.Fatal("refused withdrawal not retried") + } + if len(v.pendingDel) != 0 { + t.Fatal("retried withdrawal still pending") + } +} + +func TestOwnershipFence_RefusesAWorkerOwnedScope(t *testing.T) { + auth, err := ownership.New(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if _, err := auth.Claim(ownership.VPCScope(workerVPC), ownership.WorkerOwnerPrefix+"node-a:1"); err != nil { + t.Fatal(err) + } + fence := OwnershipFence(auth) + ran := false + if err := fence(workerVPC, func() error { ran = true; return nil }); !errors.Is(err, ownership.ErrWorkerOwned) || + ran { + t.Fatalf("fence(worker-owned) = %v, ran %t; want ErrWorkerOwned without running", err, ran) + } + if err := fence(otherVPC, func() error { ran = true; return nil }); err != nil || !ran { + t.Fatalf("fence(unowned) = %v, ran %t; want it run", err, ran) + } +} diff --git a/internal/runtime/workerroutes/workerroutes.go b/internal/runtime/workerroutes/workerroutes.go new file mode 100644 index 00000000..2a5cbe6e --- /dev/null +++ b/internal/runtime/workerroutes/workerroutes.go @@ -0,0 +1,344 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package workerroutes hands galactic-router's tenant routes to Galactic +// Worker through the Routes service, whose Fabric module writes them for +// every VPC Worker owns. +// +// A Handoff keeps the routes the router currently installs and its recent +// withdrawals. Each time it opens a stream it sends all of them, because +// Worker keeps routes only in memory, and then sends each change as it +// happens. While Worker is unreachable changes accumulate and are delivered +// once a stream opens again; a route Worker already wrote stays in the +// datapath meanwhile. +// +// Worker acknowledges a withdrawal once it has recorded it, which is before +// its Fabric module removes the entry. A Worker that restarts in between has +// forgotten the withdrawal, so a Handoff keeps an acknowledged withdrawal for +// withdrawalRetention and sends it again on every new stream. +package workerroutes + +import ( + "context" + "errors" + "io" + "log/slog" + "net" + "net/netip" + "sort" + "sync" + "time" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + "go.datum.net/galactic/pkg/workerclient" +) + +const ( + // maxBatch bounds the routes one request carries. + maxBatch = 1024 + // retryMin and retryMax bound the wait between connection attempts. + retryMin = time.Second + retryMax = 30 * time.Second + // withdrawalRetention is how long an acknowledged withdrawal is kept and + // resent on new streams. + withdrawalRetention = 10 * time.Minute + // maxWithdrawals bounds the withdrawals kept, above egress_route_table's + // 32768 entries. Past it the oldest are dropped, acknowledged ones + // first. + maxWithdrawals = 65536 +) + +// Stream is the part of a Routes.Sync stream a Handoff uses. +type Stream interface { + Send(*workerv1alpha1.RoutesSyncRequest) error + Recv() (*workerv1alpha1.RoutesSyncResponse, error) + CloseSend() error +} + +// Connect opens a stream to Worker and returns a function closing the +// connection. +type Connect func(ctx context.Context) (Stream, func(), error) + +type key struct { + vpc string + prefix netip.Prefix +} + +type route struct { + table uint32 + sid netip.Addr +} + +// withdrawal is one withdrawal the Handoff keeps. +type withdrawal struct { + // seq is the sequence of the request that last carried it, 0 until one + // has. + seq uint64 + // since is when it was withdrawn, and acked when Worker first + // acknowledged it. + since, acked time.Time +} + +// Handoff hands tenant routes to Worker. Its methods are safe for concurrent +// use. +type Handoff struct { + connect Connect + log *slog.Logger + now func() time.Time + + mu sync.Mutex + routes map[key]route + withdrawn map[key]withdrawal + // dirty are the keys changed since they were last sent. + dirty map[key]struct{} + seq uint64 + wake chan struct{} +} + +// New returns a Handoff that connects to Worker at socket, presenting the +// token in tokenFile. +func New(socket, tokenFile string) *Handoff { + return NewWithConnect(func(ctx context.Context) (Stream, func(), error) { + c, err := workerclient.Dial(ctx, workerclient.Options{ + SocketPath: socket, TokenFile: tokenFile, NoKeepAlive: true, + }) + if err != nil { + return nil, nil, err + } + s, err := c.SyncRoutes(ctx) + if err != nil { + _ = c.Close() + return nil, nil, err + } + return s, func() { _ = c.Close() }, nil + }) +} + +// NewWithConnect returns a Handoff that opens streams with connect. +func NewWithConnect(connect Connect) *Handoff { + return &Handoff{ + connect: connect, + log: slog.Default().With("component", "worker-route-handoff"), + now: time.Now, + routes: map[key]route{}, + withdrawn: map[key]withdrawal{}, + dirty: map[key]struct{}{}, + wake: make(chan struct{}, 1), + } +} + +// Upsert records that the router installs prefix in vpc's table tableID +// toward sid. +func (h *Handoff) Upsert(vpc string, tableID uint32, prefix netip.Prefix, sid net.IP) { + addr, ok := netip.AddrFromSlice(sid) + if !ok || vpc == "" { + return + } + k := key{vpc: vpc, prefix: prefix.Masked()} + h.mu.Lock() + h.routes[k] = route{table: tableID, sid: addr.Unmap()} + delete(h.withdrawn, k) + h.dirty[k] = struct{}{} + h.mu.Unlock() + h.signal() +} + +// Withdraw records that the router withdrew prefix from vpc. +func (h *Handoff) Withdraw(vpc string, prefix netip.Prefix) { + if vpc == "" { + return + } + h.mu.Lock() + h.withdrawLocked(key{vpc: vpc, prefix: prefix.Masked()}) + h.mu.Unlock() + h.signal() +} + +// ForgetTable withdraws every route of vpc it holds in tableID. +func (h *Handoff) ForgetTable(vpc string, tableID uint32) { + h.mu.Lock() + for k, r := range h.routes { + if k.vpc == vpc && r.table == tableID { + h.withdrawLocked(k) + } + } + h.mu.Unlock() + h.signal() +} + +func (h *Handoff) withdrawLocked(k key) { + delete(h.routes, k) + h.withdrawn[k] = withdrawal{since: h.now()} + h.dirty[k] = struct{}{} + h.pruneLocked() +} + +// pruneLocked drops acknowledged withdrawals older than withdrawalRetention +// and, past maxWithdrawals, the oldest withdrawals, acknowledged ones first. +func (h *Handoff) pruneLocked() { + now := h.now() + for k, w := range h.withdrawn { + if !w.acked.IsZero() && now.Sub(w.acked) > withdrawalRetention { + delete(h.withdrawn, k) + } + } + if len(h.withdrawn) <= maxWithdrawals { + return + } + keys := make([]key, 0, len(h.withdrawn)) + for k := range h.withdrawn { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { + a, b := h.withdrawn[keys[i]], h.withdrawn[keys[j]] + if a.acked.IsZero() != b.acked.IsZero() { + return !a.acked.IsZero() + } + return a.since.Before(b.since) + }) + drop := len(h.withdrawn) - maxWithdrawals + h.log.Warn("dropping the oldest tenant route withdrawals kept for galactic-worker", "dropped", drop) + for _, k := range keys[:drop] { + delete(h.withdrawn, k) + delete(h.dirty, k) + } +} + +func (h *Handoff) signal() { + select { + case h.wake <- struct{}{}: + default: + } +} + +// Run keeps a stream to Worker open until ctx is done, reconnecting with +// backoff. +func (h *Handoff) Run(ctx context.Context) { + wait := retryMin + connected := false + for ctx.Err() == nil { + stream, closeConn, err := h.connect(ctx) + if err != nil { + if connected { + h.log.Info("cannot reach galactic-worker; tenant route changes wait for it", "error", err) + } else { + h.log.Debug("cannot reach galactic-worker", "error", err) + } + connected = false + } else { + if !connected { + h.log.Info("handing tenant routes to galactic-worker") + } + connected = true + wait = retryMin + err = h.serve(ctx, stream) + closeConn() + if err != nil && ctx.Err() == nil { + h.log.Info("tenant route stream to galactic-worker ended; reconnecting", "error", err) + } + } + select { + case <-ctx.Done(): + return + case <-time.After(wait): + } + wait = min(2*wait, retryMax) + } +} + +// serve sends every held route and withdrawal on stream, then each change, +// until the stream fails or ctx is done. +func (h *Handoff) serve(ctx context.Context, stream Stream) error { + h.mu.Lock() + for k := range h.routes { + h.dirty[k] = struct{}{} + } + for k := range h.withdrawn { + h.dirty[k] = struct{}{} + } + h.mu.Unlock() + + recvErr := make(chan error, 1) + go func() { + for { + resp, err := stream.Recv() + if err != nil { + recvErr <- err + return + } + h.ack(resp) + } + }() + for { + if req := h.next(); req != nil { + if err := stream.Send(req); err != nil { + return err + } + continue + } + select { + case <-ctx.Done(): + _ = stream.CloseSend() + return nil + case err := <-recvErr: + if errors.Is(err, io.EOF) { + return errors.New("galactic-worker closed the stream") + } + return err + case <-h.wake: + } + } +} + +// next builds a request from up to maxBatch changed keys, or returns nil. +func (h *Handoff) next() *workerv1alpha1.RoutesSyncRequest { + h.mu.Lock() + defer h.mu.Unlock() + if len(h.dirty) == 0 { + return nil + } + h.seq++ + req := &workerv1alpha1.RoutesSyncRequest{Sequence: h.seq} + n := 0 + for k := range h.dirty { + if n == maxBatch { + break + } + n++ + delete(h.dirty, k) + if r, ok := h.routes[k]; ok { + req.Upserts = append(req.Upserts, &workerv1alpha1.TenantRoute{ + Vpc: k.vpc, TableId: r.table, Prefix: k.prefix.String(), Sid: r.sid.String(), + }) + continue + } + if w, ok := h.withdrawn[k]; ok { + w.seq = h.seq + h.withdrawn[k] = w + req.Withdrawals = append(req.Withdrawals, &workerv1alpha1.TenantRouteKey{ + Vpc: k.vpc, Prefix: k.prefix.String(), + }) + } + } + return req +} + +// ack marks the withdrawals Worker recorded with resp as acknowledged, drops +// expired ones, and logs the routes it rejected. +func (h *Handoff) ack(resp *workerv1alpha1.RoutesSyncResponse) { + for _, r := range resp.GetRejected() { + h.log.Warn("galactic-worker rejected a tenant route", "vpc", r.GetKey().GetVpc(), + "prefix", r.GetKey().GetPrefix(), "reason", r.GetReason(), "message", r.GetMessage()) + } + h.mu.Lock() + defer h.mu.Unlock() + now := h.now() + for k, w := range h.withdrawn { + if w.seq != 0 && w.seq <= resp.GetSequence() && w.acked.IsZero() { + w.acked = now + h.withdrawn[k] = w + } + } + h.pruneLocked() +} diff --git a/internal/runtime/workerroutes/workerroutes_test.go b/internal/runtime/workerroutes/workerroutes_test.go new file mode 100644 index 00000000..0beb3891 --- /dev/null +++ b/internal/runtime/workerroutes/workerroutes_test.go @@ -0,0 +1,224 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package workerroutes + +import ( + "context" + "errors" + "net" + "net/netip" + "sort" + "sync" + "testing" + "time" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" +) + +// fakeStream records requests and answers each with its sequence unless +// held. +type fakeStream struct { + sent chan *workerv1alpha1.RoutesSyncRequest + resps chan *workerv1alpha1.RoutesSyncResponse + fail chan error + + mu sync.Mutex + hold bool +} + +func newFakeStream() *fakeStream { + return &fakeStream{ + sent: make(chan *workerv1alpha1.RoutesSyncRequest, 64), + resps: make(chan *workerv1alpha1.RoutesSyncResponse, 64), + fail: make(chan error, 1), + } +} + +func (s *fakeStream) Send(req *workerv1alpha1.RoutesSyncRequest) error { + s.sent <- req + s.mu.Lock() + hold := s.hold + s.mu.Unlock() + if !hold { + s.resps <- &workerv1alpha1.RoutesSyncResponse{Sequence: req.GetSequence()} + } + return nil +} + +func (s *fakeStream) Recv() (*workerv1alpha1.RoutesSyncResponse, error) { + select { + case r := <-s.resps: + return r, nil + case err := <-s.fail: + return nil, err + } +} + +func (s *fakeStream) CloseSend() error { return nil } + +// collect reads requests until it has seen one upsert and withdrawals +// withdrawals, or fails after a deadline. +func collect(t *testing.T, s *fakeStream, withdrawals int) ([]string, []string) { + const upserts = 1 + t.Helper() + var ups, wds []string + deadline := time.After(5 * time.Second) + for len(ups) < upserts || len(wds) < withdrawals { + select { + case req := <-s.sent: + for _, u := range req.GetUpserts() { + ups = append(ups, u.GetVpc()+" "+u.GetPrefix()+" "+u.GetSid()) + } + for _, w := range req.GetWithdrawals() { + wds = append(wds, w.GetVpc()+" "+w.GetPrefix()) + } + case <-deadline: + t.Fatalf("got upserts %v and withdrawals %v, want %d and %d", ups, wds, upserts, withdrawals) + } + } + sort.Strings(ups) + sort.Strings(wds) + return ups, wds +} + +var ( + pA = netip.MustParsePrefix("2001:db8:a::1/128") + pB = netip.MustParsePrefix("2001:db8:a::2/128") + sid = net.ParseIP("2001:db8:0:2:e000:1::") +) + +// acked counts the withdrawals Worker acknowledged and the ones it has not. +func (h *Handoff) acked() (acked, pending int) { + h.mu.Lock() + defer h.mu.Unlock() + for _, w := range h.withdrawn { + if w.acked.IsZero() { + pending++ + } else { + acked++ + } + } + return acked, pending +} + +func TestHandoffSendsEverythingThenChanges(t *testing.T) { + stream := newFakeStream() + h := NewWithConnect(func(context.Context) (Stream, func(), error) { return stream, func() {}, nil }) + h.Upsert("vpcA", 100, pA, sid) + h.Upsert("vpcA", 100, pB, sid) + h.Withdraw("vpcA", pB) + h.Withdraw("vpcB", pA) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go h.Run(ctx) + + ups, wds := collect(t, stream, 2) + if len(ups) != 1 || ups[0] != "vpcA 2001:db8:a::1/128 2001:db8:0:2:e000:1::" { + t.Errorf("upserts = %v", ups) + } + if len(wds) != 2 { + t.Errorf("withdrawals = %v", wds) + } + deadline := time.Now().Add(5 * time.Second) + for { + if a, p := h.acked(); a == 2 && p == 0 { + break + } + if time.Now().After(deadline) { + a, p := h.acked() + t.Fatalf("withdrawals acknowledged %d, pending %d; want 2 and 0", a, p) + } + time.Sleep(time.Millisecond) + } + + h.Upsert("vpcC", 300, pA, sid) + if ups, _ := collect(t, stream, 0); ups[0] != "vpcC 2001:db8:a::1/128 2001:db8:0:2:e000:1::" { + t.Errorf("change = %v", ups) + } +} + +func TestHandoffResendsAfterReconnecting(t *testing.T) { + first, second := newFakeStream(), newFakeStream() + first.hold = true // nothing is acknowledged on the first stream + streams := make(chan *fakeStream, 2) + streams <- first + streams <- second + h := NewWithConnect(func(ctx context.Context) (Stream, func(), error) { + select { + case s := <-streams: + return s, func() {}, nil + case <-ctx.Done(): + return nil, nil, ctx.Err() + } + }) + h.Upsert("vpcA", 100, pA, sid) + h.Withdraw("vpcA", pB) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go h.Run(ctx) + + collect(t, first, 1) + first.fail <- errors.New("worker restarted") + // The route and the withdrawal are both sent again. + ups, wds := collect(t, second, 1) + if len(ups) != 1 || len(wds) != 1 || wds[0] != "vpcA 2001:db8:a::2/128" { + t.Fatalf("after reconnecting: upserts %v, withdrawals %v", ups, wds) + } +} + +func TestHandoffIgnoresRoutesWithoutAVPC(t *testing.T) { + h := NewWithConnect(nil) + h.Upsert("", 100, pA, sid) + h.Withdraw("", pA) + if h.next() != nil { + t.Fatal("a route without a VPC was queued") + } +} + +func TestHandoffResendsAcknowledgedWithdrawalsUntilTheyExpire(t *testing.T) { + now := time.Unix(1_700_000_000, 0) + h := NewWithConnect(nil) + h.now = func() time.Time { return now } + h.Upsert("vpcA", 100, pA, sid) + h.Withdraw("vpcA", pA) + req := h.next() + h.ack(&workerv1alpha1.RoutesSyncResponse{Sequence: req.GetSequence()}) + if a, _ := h.acked(); a != 1 { + t.Fatal("acknowledged withdrawal not kept") + } + + // A new stream sends it again: Worker may have restarted before removing + // the entry. + h.mu.Lock() + for k := range h.withdrawn { + h.dirty[k] = struct{}{} + } + h.mu.Unlock() + if req := h.next(); len(req.GetWithdrawals()) != 1 { + t.Fatalf("resent request = %v, want the withdrawal", req) + } + + now = now.Add(withdrawalRetention + time.Second) + h.ack(&workerv1alpha1.RoutesSyncResponse{Sequence: 99}) + if a, p := h.acked(); a != 0 || p != 0 { + t.Fatalf("expired withdrawal kept: acknowledged %d, pending %d", a, p) + } +} + +func TestHandoffForgetTableWithdrawsOnlyThatTable(t *testing.T) { + h := NewWithConnect(nil) + h.Upsert("vpcA", 100, pA, sid) + h.Upsert("vpcA", 100, pB, sid) + h.Upsert("vpcB", 100, pA, sid) + h.Upsert("vpcA", 200, netip.MustParsePrefix("2001:db8:a::3/128"), sid) + h.ForgetTable("vpcA", 100) + h.mu.Lock() + defer h.mu.Unlock() + if len(h.routes) != 2 || len(h.withdrawn) != 2 { + t.Fatalf("routes %v, withdrawn %v; want vpcA's table 100 routes withdrawn only", h.routes, h.withdrawn) + } +} diff --git a/internal/worker/api.go b/internal/worker/api.go index 5367b2d7..9a096dad 100644 --- a/internal/worker/api.go +++ b/internal/worker/api.go @@ -25,8 +25,11 @@ import ( // servedVersions are the API versions this Worker serves, oldest first. var servedVersions = []string{workerv1alpha1.Version} -// Register registers the Worker service on s. -func (w *Worker) Register(s *grpc.Server) { workerv1alpha1.RegisterWorkerServer(s, w) } +// Register registers the Worker and Routes services on s. +func (w *Worker) Register(s *grpc.Server) { + workerv1alpha1.RegisterWorkerServer(s, w) + workerv1alpha1.RegisterRoutesServer(s, &routesServer{w: w}) +} // GetCapabilities implements workerv1alpha1.WorkerServer. func (w *Worker) GetCapabilities(ctx context.Context, req *workerv1alpha1.GetCapabilitiesRequest) ( diff --git a/internal/worker/fabric/datapath.go b/internal/worker/fabric/datapath.go new file mode 100644 index 00000000..16f0d98c --- /dev/null +++ b/internal/worker/fabric/datapath.go @@ -0,0 +1,211 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package fabric + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/netip" + "sync" + + "sigs.k8s.io/controller-runtime/pkg/client" + + "go.datum.net/galactic/internal/nodelocator" + "go.datum.net/galactic/internal/nodevpc" + "go.datum.net/galactic/internal/plumbing/ebpf/egressroutemap" + "go.datum.net/galactic/internal/plumbing/ebpf/uformat" + "go.datum.net/galactic/internal/plumbing/ebpf/usidmap" + "go.datum.net/galactic/internal/plumbing/vrf" +) + +// Datapath is the host state the Fabric module reads and writes. +type Datapath interface { + // Loaded returns nil when the pinned uSID maps are open. + Loaded() error + // TableID returns vpc's host VRF table, and false when the VRF does not + // exist. + TableID(vpc string) (uint32, bool, error) + // Lookup returns the egress_route_table entry a packet to prefix in + // table matches: the longest entry no longer than prefix. + Lookup(table uint32, prefix netip.Prefix) (egressroutemap.Row, bool, error) + // Write installs or replaces the entry for prefix in table, + // encapsulating toward sid, resolving and soliciting its next hop. + Write(table uint32, prefix netip.Prefix, sid netip.Addr) error + // Remove deletes prefix's own entry in table when it encapsulates toward + // sid, or toward any SID when sid is invalid. It keeps a pass-through or + // shard group entry. + Remove(table uint32, prefix netip.Prefix, sid netip.Addr) error + // NextHop resolves sid's underlay next hop without soliciting it. Its + // error wraps egressroutemap.ErrNoUplinkRoute or + // egressroutemap.ErrNeighborUnresolved. + NextHop(sid netip.Addr) (egressroutemap.NextHop, error) + // ReturnPath returns nil when this node decapsulates vpc's SID into + // vpc's table: its locator is in locator_table, and its BGPVRFInstance + // and vrf_table row for vpc are in place. + ReturnPath(ctx context.Context, vpc string) error +} + +// Kernel is the Datapath of the calling process's network namespace, which +// must be the host's. +type Kernel struct { + // PinDir is the bpffs directory of the pinned uSID maps. + PinDir string + // K8s reads BGPRouters and BGPVRFInstances. + K8s client.Client + // Namespace holds the node's BGPRouter and BGPVRFInstances. + Namespace string + // Node is this node. + Node string + + mu sync.Mutex + table *egressroutemap.EgressRouteTable + closer io.Closer + registry *usidmap.Registry + regClose io.Closer +} + +var _ Datapath = (*Kernel)(nil) + +// open returns the pinned egress_route_table and uSID registry, opening them +// on first use and after a failure. +func (k *Kernel) open() (*egressroutemap.EgressRouteTable, *usidmap.Registry, error) { + k.mu.Lock() + defer k.mu.Unlock() + if k.table == nil { + t, c, err := egressroutemap.OpenPinnedEgressRouteTable(k.PinDir) + if err != nil { + return nil, nil, err + } + k.table, k.closer = t, c + } + if k.registry == nil { + r, c, err := usidmap.OpenPinnedRegistry(k.PinDir) + if err != nil { + return nil, nil, err + } + k.registry, k.regClose = r, c + } + return k.table, k.registry, nil +} + +// Close releases the maps this process opened. +func (k *Kernel) Close() error { + k.mu.Lock() + defer k.mu.Unlock() + var errs []error + if k.closer != nil { + errs = append(errs, k.closer.Close()) + } + if k.regClose != nil { + errs = append(errs, k.regClose.Close()) + } + k.table, k.closer, k.registry, k.regClose = nil, nil, nil, nil + return errors.Join(errs...) +} + +// Loaded implements Datapath. +func (k *Kernel) Loaded() error { + _, _, err := k.open() + return err +} + +// TableID implements Datapath. +func (k *Kernel) TableID(vpc string) (uint32, bool, error) { + id, err := vrf.TableID(vpc) + switch { + case errors.Is(err, vrf.ErrNotFound): + return 0, false, nil + case err != nil: + return 0, false, err + } + return id, true, nil +} + +// Lookup implements Datapath. +func (k *Kernel) Lookup(table uint32, prefix netip.Prefix) (egressroutemap.Row, bool, error) { + t, _, err := k.open() + if err != nil { + return egressroutemap.Row{}, false, err + } + return t.LookupRow(table, ipNet(prefix)) +} + +// Write implements Datapath. +func (k *Kernel) Write(table uint32, prefix netip.Prefix, sid netip.Addr) error { + t, _, err := k.open() + if err != nil { + return err + } + return t.Register(table, ipNet(prefix), net.IP(sid.AsSlice())) +} + +// Remove implements Datapath. +func (k *Kernel) Remove(table uint32, prefix netip.Prefix, sid netip.Addr) error { + t, _, err := k.open() + if err != nil { + return err + } + var want net.IP + if sid.IsValid() { + want = net.IP(sid.AsSlice()) + } + return t.UnregisterIfSID(table, ipNet(prefix), want) +} + +// NextHop implements Datapath. +func (k *Kernel) NextHop(sid netip.Addr) (egressroutemap.NextHop, error) { + return egressroutemap.ResolveNextHop(net.IP(sid.AsSlice())) +} + +// ReturnPath implements Datapath. +func (k *Kernel) ReturnPath(ctx context.Context, vpc string) error { + _, reg, err := k.open() + if err != nil { + return err + } + own, err := nodelocator.Prefix(ctx, k.K8s, k.Namespace, k.Node) + if err != nil { + return err + } + if !own.IsValid() { + return errors.New("this node has no SRv6 locator yet") + } + block, err := uformat.Block(own.Addr()) + if err != nil { + return err + } + nodeID, err := uformat.NodeID(own.Addr()) + if err != nil { + return err + } + if _, ok, err := reg.Locator.Get(block, nodeID); err != nil { + return err + } else if !ok { + return fmt.Errorf("locator_table has no row for this node's locator %s", own) + } + cfg := nodevpc.Config{K8s: k.K8s, Namespace: k.Namespace, Node: k.Node, PinDir: k.PinDir} + ok, err := nodevpc.InPlace(ctx, cfg, vpc) + if err != nil { + return err + } + if !ok { + return errors.New("this node's BGPVRFInstance or vrf_table row for the VPC is not in place") + } + return nil +} + +// ipNet converts a prefix to the net.IPNet egressroutemap takes, in the +// address family of its address. +func ipNet(p netip.Prefix) *net.IPNet { + addr := p.Addr().Unmap() + bits := 128 + if addr.Is4() { + bits = 32 + } + return &net.IPNet{IP: net.IP(addr.AsSlice()), Mask: net.CIDRMask(p.Bits(), bits)} +} diff --git a/internal/worker/fabric/fabric.go b/internal/worker/fabric/fabric.go new file mode 100644 index 00000000..0e0d6d33 --- /dev/null +++ b/internal/worker/fabric/fabric.go @@ -0,0 +1,540 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package fabric is Galactic Worker's Fabric delivery module. It delivers to +// destinations on other nodes through the existing SRv6 encapsulator and the +// tenant routes galactic-router learns from EVPN. +// +// The module is the single writer of the tenant routes in egress_route_table +// for every VPC scope Worker owns. galactic-router hands it each route it +// learns through the Routes service; the module writes the route under the +// VPC's fence, re-resolves its underlay next hop when routes or neighbors +// change, and removes it only when galactic-router withdraws it. It never +// touches an entry it did not learn from galactic-router, such as CNI's +// pass-through entries, egress shard entries or Galactic Edge's entries. +// +// The tenant routes of a VPC are shared by every claim in it. A claim's +// Fabric destinations own no entries: the module reports, per destination, +// whether the entry usid_egress would match for each of the destination's +// addresses leads to the destination's node (forward), and whether this node +// decapsulates the VPC's SID into the VPC's table (return). +// +// A route whose next hop stops resolving keeps the entry already written for +// it rather than losing it, so a shorter entry, such as the VPC's default +// toward an egress shard, never catches traffic meant for the unreachable +// node. A route that cannot be written yet, because no entry exists or its +// SID changed, is retried on the next route or neighbor event or periodic +// resync; until then its traffic matches whatever entry it matched before, +// as when galactic-router writes it, and its destinations report why. Neither +// fails the scope's commit, so one unreachable node never stops the VPC's +// other destinations from being programmed. +package fabric + +import ( + "bytes" + "context" + "errors" + "fmt" + "log/slog" + "net" + "net/netip" + "sync" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + "go.datum.net/galactic/internal/plumbing/ebpf/egressroutemap" + "go.datum.net/galactic/internal/worker/module" + "go.datum.net/galactic/internal/worker/routes" +) + +// Name is the module's name. +const Name = "fabric" + +// Config configures a Module. +type Config struct { + // Routes is the tenant route table the Routes service records into. + Routes *routes.Table + // Datapath is the host state the module reads and writes. + Datapath Datapath + // OwnLocator returns this node's SRv6 locator, or the zero prefix while + // it has none. + OwnLocator func(ctx context.Context) (netip.Prefix, error) + Log *slog.Logger +} + +// written is what the module last wrote for a route. +type written struct { + table uint32 + sid netip.Addr +} + +// hop identifies an underlay next hop a route's entry uses. +type hop struct { + link int + gateway netip.Addr +} + +// Module is the Fabric delivery module. +type Module struct { + cfg Config + + mu sync.Mutex + // desired is each scope's claims from its last Plan, which Read reports + // on. + desired map[string][]module.Claim + // written is what Apply last wrote for each route, so a route whose + // VPC's table moved has its old entry removed. + written map[routes.Key]written + // failed is the last write error of each route not yet written. + failed map[routes.Key]error + // sids and hops index each VPC's routes by SID and by the next hop + // their entries used, so a route or neighbor event requeues only the + // VPCs it touches. + sids map[string]map[netip.Addr]struct{} + hops map[string]map[hop]struct{} +} + +var _ module.Module = (*Module)(nil) + +// New returns a Fabric module. +func New(cfg Config) (*Module, error) { + if cfg.Routes == nil || cfg.Datapath == nil || cfg.OwnLocator == nil { + return nil, errors.New("fabric: routes, datapath and own locator are required") + } + if cfg.Log == nil { + cfg.Log = slog.Default() + } + cfg.Log = cfg.Log.With("module", Name) + return &Module{ + cfg: cfg, + desired: map[string][]module.Claim{}, + written: map[routes.Key]written{}, + failed: map[routes.Key]error{}, + sids: map[string]map[netip.Addr]struct{}{}, + hops: map[string]map[hop]struct{}{}, + }, nil +} + +// Name implements module.Module. +func (m *Module) Name() string { return Name } + +// Mode implements module.Module. +func (m *Module) Mode() workerv1alpha1.Mode { return workerv1alpha1.Mode_MODE_FABRIC } + +// Capabilities implements module.Module. Both capabilities need the pinned +// uSID maps. +func (m *Module) Capabilities() []module.Capability { + caps := []module.Capability{ + {Name: workerv1alpha1.CapabilityFabricSRv6, Available: true}, + {Name: workerv1alpha1.CapabilityFabricRoutes, Available: true}, + } + if err := m.cfg.Datapath.Loaded(); err != nil { + for i := range caps { + caps[i].Available = false + caps[i].Reason = workerv1alpha1.ReasonDatapathNotLoaded + caps[i].Message = err.Error() + } + } + return caps +} + +// removal is one entry the change removes. +type removal struct { + table uint32 + prefix netip.Prefix + sid netip.Addr + // withdrawal is set when removing it confirms a withdrawal. + withdrawal *routes.Route + // moved is set when it is the old entry of a route whose table moved. + moved *routes.Key +} + +// change is the Fabric module's change for one scope. +type change struct { + vpc string + writes []routes.Route + removes []removal +} + +// Empty implements module.Change. +func (c *change) Empty() bool { return len(c.writes) == 0 && len(c.removes) == 0 } + +// Plan implements module.Module. It records desired for Read and plans the +// scope's tenant routes: it writes each route whose entry is missing, leads +// elsewhere or has an out-of-date next hop, and removes each withdrawn route's +// entry. A route recorded for another table than the VPC's current one, or +// toward this node's own locator, is skipped. +func (m *Module) Plan(ctx context.Context, scope module.Scope, desired []module.Claim) (module.Change, error) { + vpc := scope.VPC + m.mu.Lock() + if len(desired) > 0 { + m.desired[vpc] = desired + } else { + delete(m.desired, vpc) + } + wrote := false + for k := range m.written { + if k.VPC == vpc { + wrote = true + break + } + } + m.mu.Unlock() + + ch := &change{vpc: vpc} + if len(desired) == 0 && !wrote && !m.cfg.Routes.HasWork(vpc) { + return ch, nil + } + table, ok, err := m.cfg.Datapath.TableID(vpc) + if err != nil { + return nil, fmt.Errorf("fabric: read VPC %s table: %w", vpc, err) + } + if !ok { + // No VRF, so no table to write into. A withdrawn route's entry is + // still removed from the table it was recorded for. + m.planWithdrawals(ch, vpc, 0) + return ch, nil + } + own, err := m.cfg.OwnLocator(ctx) + if err != nil { + return nil, fmt.Errorf("fabric: read this node's locator: %w", err) + } + + sids := map[netip.Addr]struct{}{} + hops := map[hop]struct{}{} + m.mu.Lock() + prev := make(map[routes.Key]written, len(m.written)) + for k, w := range m.written { + if k.VPC == vpc { + prev[k] = w + } + } + m.mu.Unlock() + + for _, r := range m.cfg.Routes.Routes(vpc) { + if err := m.planRoute(ch, r, table, own, prev, sids, hops); err != nil { + return nil, err + } + } + + m.planWithdrawals(ch, vpc, table) + + m.mu.Lock() + m.sids[vpc], m.hops[vpc] = sids, hops + m.mu.Unlock() + return ch, nil +} + +// planRoute plans one tenant route of ch's VPC, whose current table is table: +// it skips a route recorded for another table or toward own, the node's own +// locator, removes the entry a route left in its VPC's earlier table, and +// writes the route unless this incarnation wrote its own entry and that entry +// still leads to its SID over a current next hop. It records the route's SID +// and next hop in sids and hops. +func (m *Module) planRoute(ch *change, r routes.Route, table uint32, own netip.Prefix, + prev map[routes.Key]written, sids map[netip.Addr]struct{}, hops map[hop]struct{}) error { + switch { + case r.TableID != table: + skipped(reasonTableMismatch) + m.cfg.Log.Debug("skipping a tenant route recorded for another table than the VPC's", + "vpc", ch.vpc, "prefix", r.Prefix, "routeTable", r.TableID, "table", table) + return nil + case own.IsValid() && own.Contains(r.SID): + skipped(reasonOwnLocator) + m.cfg.Log.Debug("skipping a tenant route toward this node's own locator", "vpc", ch.vpc, "prefix", r.Prefix) + return nil + } + sids[r.SID] = struct{}{} + if w, ok := prev[r.Key]; ok && w.table != table { + // The SID carries the remote node's Argument for this VPC, so the + // old table's entry is removed only while it is this route's. + k := r.Key + ch.removes = append(ch.removes, removal{table: w.table, prefix: r.Prefix, sid: w.sid, moved: &k}) + } + row, have, err := m.cfg.Datapath.Lookup(table, r.Prefix) + if err != nil { + return fmt.Errorf("fabric: read entry %s in table %d: %w", r.Prefix, table, err) + } + next, hopErr := m.cfg.Datapath.NextHop(r.SID) + if hopErr == nil || errors.Is(hopErr, egressroutemap.ErrNeighborUnresolved) { + if gw, ok := netip.AddrFromSlice(next.Gateway); ok { + hops[hop{link: next.LinkIndex, gateway: gw.Unmap()}] = struct{}{} + } + } + // The lookup is longest-prefix, so a shorter entry toward the same + // SID can answer for the route; only an entry this incarnation + // wrote for the route's own prefix counts. After a restart every + // route is written once again. + w, wroteIt := prev[r.Key] + current := wroteIt && w.table == table && w.sid == r.SID && + have && !row.PassThrough && !row.Sentinel && sameSID(row.SID, r.SID) + switch { + case !current && errors.Is(hopErr, egressroutemap.ErrNoUplinkRoute): + // Writing would fail; whatever entry the route has is kept. + m.setFailed(r.Key, hopErr) + case !current: + ch.writes = append(ch.writes, r) + case hopErr == nil && !sameHop(row, next): + ch.writes = append(ch.writes, r) + default: + // Current, or a next hop that does not resolve now: the entry + // is kept as it is. + } + return nil +} + +// planWithdrawals plans removing the entry of each withdrawn route not yet +// confirmed removed: from the table it was recorded for, and from table, the +// VPC's current one, when that differs and is not 0. A withdrawal with no +// recorded table is removed from table only. A removal names the route's SID +// when it is known, which carries the remote node's Argument for this VPC, so +// it never removes another VPC's entry from a table that moved to it. +func (m *Module) planWithdrawals(ch *change, vpc string, table uint32) { + for _, w := range m.cfg.Routes.Withdrawn(vpc) { + if w.Removed { + continue + } + t := w.TableID + if t == 0 { + t = table + } + if t == 0 { + continue + } + route := w.Route + ch.removes = append(ch.removes, removal{table: t, prefix: w.Prefix, sid: w.SID, withdrawal: &route}) + if table != 0 && t != table { + ch.removes = append(ch.removes, removal{table: table, prefix: w.Prefix, sid: w.SID}) + } + } +} + +// Apply implements module.Module. It writes and removes entries one at a +// time. A route that fails is recorded for Read and retried by a later +// pass; it does not fail the commit. An error is returned only when the +// datapath is not loaded. +func (m *Module) Apply(_ context.Context, scope module.Scope, c module.Change) error { + ch, ok := c.(*change) + if !ok || ch.vpc != scope.VPC { + return fmt.Errorf("fabric: change for another scope than %s", scope.VPC) + } + if err := m.cfg.Datapath.Loaded(); err != nil { + return fmt.Errorf("fabric: datapath not loaded: %w", err) + } + var confirmed []routes.Route + for _, rm := range ch.removes { + if err := m.cfg.Datapath.Remove(rm.table, rm.prefix, rm.sid); err != nil { + m.cfg.Log.Warn("removing a tenant route failed; retrying", "vpc", ch.vpc, "prefix", rm.prefix, + "table", rm.table, "error", err) + continue + } + m.cfg.Log.Debug("removed a tenant route", "vpc", ch.vpc, "prefix", rm.prefix, "table", rm.table, + "sid", rm.sid) + m.mu.Lock() + switch { + case rm.withdrawal != nil: + confirmed = append(confirmed, *rm.withdrawal) + delete(m.written, rm.withdrawal.Key) + delete(m.failed, rm.withdrawal.Key) + case rm.moved != nil: + if w, ok := m.written[*rm.moved]; ok && w.table == rm.table { + delete(m.written, *rm.moved) + } + } + m.mu.Unlock() + } + m.cfg.Routes.ConfirmRemoved(confirmed) + for _, r := range ch.writes { + if err := m.cfg.Datapath.Write(r.TableID, r.Prefix, r.SID); err != nil { + m.setFailed(r.Key, err) + m.cfg.Log.Info("writing a tenant route failed; retrying on the next route or neighbor change", + "vpc", ch.vpc, "prefix", r.Prefix, "sid", r.SID, "error", err) + continue + } + m.cfg.Log.Debug("wrote a tenant route", "vpc", ch.vpc, "prefix", r.Prefix, "table", r.TableID, "sid", r.SID) + m.mu.Lock() + m.written[r.Key] = written{table: r.TableID, sid: r.SID} + delete(m.failed, r.Key) + m.mu.Unlock() + } + return nil +} + +func (m *Module) setFailed(k routes.Key, err error) { + m.mu.Lock() + m.failed[k] = err + m.mu.Unlock() +} + +// Read implements module.Module. It reports every Fabric destination of the +// claims of the scope's last Plan. +func (m *Module) Read(ctx context.Context, scope module.Scope) (module.State, error) { + vpc := scope.VPC + m.mu.Lock() + claims := m.desired[vpc] + m.mu.Unlock() + out := module.State{} + if len(claims) == 0 { + return out, nil + } + table, haveTable, err := m.cfg.Datapath.TableID(vpc) + if err != nil { + return nil, fmt.Errorf("fabric: read VPC %s table: %w", vpc, err) + } + own, err := m.cfg.OwnLocator(ctx) + if err != nil { + return nil, fmt.Errorf("fabric: read this node's locator: %w", err) + } + returnErr := errors.New("the VPC has no host VRF yet") + if haveTable { + returnErr = m.cfg.Datapath.ReturnPath(ctx, vpc) + } + for _, c := range claims { + states := map[string]module.DestinationState{} + for _, d := range c.Destinations { + st := m.readDestination(vpc, table, haveTable, own, d) + st.Generation = c.Generation + // Return readiness depends only on this node decapsulating the + // VPC's SID, whatever the forward path's state. + st.ReturnReady = returnErr == nil + st.Ready = st.ForwardReady && st.ReturnReady + if st.ForwardReady && returnErr != nil { + st.Reason = workerv1alpha1.ReasonReturnPathNotProgrammed + st.Message = returnErr.Error() + } + states[d.ID] = st + } + out[c.Key] = states + } + return out, nil +} + +// readDestination reports one destination's Programmed and ForwardReady, and +// the reason for the first that is false. +func (m *Module) readDestination(vpc string, table uint32, haveTable bool, own netip.Prefix, + d module.Destination) module.DestinationState { + st := module.DestinationState{} + fail := func(reason, msg string) module.DestinationState { + st.Reason, st.Message = reason, msg + return st + } + switch { + case own.IsValid() && own == d.Locator: + return fail(workerv1alpha1.ReasonRouteMismatch, "the destination is on this node") + case len(d.Addresses) == 0: + return fail(workerv1alpha1.ReasonDestinationAddressUnknown, "the destination's attachment records no address") + case !haveTable: + return fail(workerv1alpha1.ReasonPending, "the VPC has no host VRF yet") + } + type check struct { + sid netip.Addr + row egressroutemap.Row + } + checks := make([]check, 0, len(d.Addresses)) + for _, addr := range d.Addresses { + route, haveRoute, wd, haveWd := m.cfg.Routes.Covering(vpc, addr) + switch { + case !haveRoute && haveWd: + return fail(workerv1alpha1.ReasonRouteWithdrawn, + fmt.Sprintf("the route %s covering %s was withdrawn", wd.Prefix, addr)) + case !haveRoute: + return fail(workerv1alpha1.ReasonForwardRouteMissing, fmt.Sprintf("no tenant route covers %s", addr)) + case !d.Locator.Contains(route.SID): + return fail(workerv1alpha1.ReasonRouteMismatch, + fmt.Sprintf("the route %s covering %s leads to %s, outside the destination's locator %s", + route.Prefix, addr, route.SID, d.Locator)) + } + row, have, err := m.cfg.Datapath.Lookup(table, netip.PrefixFrom(addr, addr.BitLen())) + if err != nil { + return fail(workerv1alpha1.ReasonProgrammingFailed, err.Error()) + } + if !have || row.PassThrough || row.Sentinel || !sameSID(row.SID, route.SID) { + m.mu.Lock() + ferr := m.failed[route.Key] + m.mu.Unlock() + if ferr != nil { + reason := hopReason(ferr) + if reason == "" { + reason = workerv1alpha1.ReasonProgrammingFailed + } + return fail(reason, fmt.Sprintf("the route %s is not written: %v", route.Prefix, ferr)) + } + return fail(workerv1alpha1.ReasonPending, fmt.Sprintf("the route %s is not written yet", route.Prefix)) + } + checks = append(checks, check{sid: route.SID, row: row}) + } + st.Programmed = true + for _, c := range checks { + next, err := m.cfg.Datapath.NextHop(c.sid) + if err != nil { + reason := hopReason(err) + if reason == "" { + reason = workerv1alpha1.ReasonUnderlayUnreachable + } + return fail(reason, err.Error()) + } + if !sameHop(c.row, next) { + return fail(workerv1alpha1.ReasonNeighborUnresolved, + fmt.Sprintf("the entry toward %s carries an out-of-date next hop", c.sid)) + } + } + st.ForwardReady = true + return st +} + +// hopReason maps a next hop resolution error to a contract reason, or "". +func hopReason(err error) string { + switch { + case errors.Is(err, egressroutemap.ErrNoUplinkRoute): + return workerv1alpha1.ReasonUnderlayUnreachable + case errors.Is(err, egressroutemap.ErrNeighborUnresolved): + return workerv1alpha1.ReasonNeighborUnresolved + } + return "" +} + +func sameSID(a net.IP, b netip.Addr) bool { + got, ok := netip.AddrFromSlice(a) + return ok && got.Unmap() == b.Unmap() +} + +func sameHop(row egressroutemap.Row, next egressroutemap.NextHop) bool { + return row.LinkIndex == next.LinkIndex && bytes.Equal(row.DMAC, next.DMAC) && bytes.Equal(row.SMAC, next.SMAC) +} + +// affectedByRoute returns the VPCs whose routes an underlay route change for dst +// touches: those with a SID dst contains, or every VPC with routes when dst +// is a default route. +func (m *Module) affectedByRoute(dst netip.Prefix) []string { + m.mu.Lock() + defer m.mu.Unlock() + var out []string + for vpc, sids := range m.sids { + if dst.Bits() == 0 { + out = append(out, vpc) + continue + } + for sid := range sids { + if dst.Contains(sid) { + out = append(out, vpc) + break + } + } + } + return out +} + +// affectedByNeighbor returns the VPCs whose entries use the next hop gateway +// on link. +func (m *Module) affectedByNeighbor(link int, gateway netip.Addr) []string { + m.mu.Lock() + defer m.mu.Unlock() + var out []string + h := hop{link: link, gateway: gateway.Unmap()} + for vpc, hops := range m.hops { + if _, ok := hops[h]; ok { + out = append(out, vpc) + } + } + return out +} diff --git a/internal/worker/fabric/fabric_test.go b/internal/worker/fabric/fabric_test.go new file mode 100644 index 00000000..21d386b7 --- /dev/null +++ b/internal/worker/fabric/fabric_test.go @@ -0,0 +1,423 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package fabric + +import ( + "context" + "errors" + "fmt" + "net" + "net/netip" + "slices" + "testing" + + "github.com/vishvananda/netlink" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + "go.datum.net/galactic/internal/plumbing/ebpf/egressroutemap" + "go.datum.net/galactic/internal/worker/fabric/fabrictest" + "go.datum.net/galactic/internal/worker/module" + "go.datum.net/galactic/internal/worker/routes" +) + +// Test topology. Addresses are documentation prefixes. VPCs A and B use the +// same guest addresses; only their tables tell them apart. +const ( + vpcA = "vpcA" + vpcB = "vpcB" + tableA = 100 + tableB = 200 + linkB = 7 + linkC = 8 +) + +var ( + locOwn = netip.MustParsePrefix("2001:db8:0:1::/64") + locB = netip.MustParsePrefix("2001:db8:0:2::/64") + locC = netip.MustParsePrefix("2001:db8:0:3::/64") + + sidOwnA = netip.MustParseAddr("2001:db8:0:1:e000:1::") + sidBA = netip.MustParseAddr("2001:db8:0:2:e000:1::") + sidBB = netip.MustParseAddr("2001:db8:0:2:e000:2::") + sidCA = netip.MustParseAddr("2001:db8:0:3:e000:1::") + + guest2 = netip.MustParseAddr("2001:db8:a::2") + guest3 = netip.MustParseAddr("2001:db8:a::3") + guest4 = netip.MustParseAddr("2001:db8:a::4") + + host2 = netip.PrefixFrom(guest2, 128) + host3 = netip.PrefixFrom(guest3, 128) + host4 = netip.PrefixFrom(guest4, 128) + + claimA = module.ClaimKey{Node: "node-a", VPC: vpcA, AttachmentUID: "svc-a"} +) + +type fixture struct { + t *testing.T + dp *fabrictest.Datapath + rt *routes.Table + m *Module +} + +func newFixture(t *testing.T) *fixture { + t.Helper() + dp := fabrictest.New() + dp.SetTable(vpcA, tableA) + dp.SetTable(vpcB, tableB) + dp.Reach(locB, linkB, "02:00:00:00:00:0b") + dp.Reach(locC, linkC, "02:00:00:00:00:0c") + rt := routes.New() + m, err := New(Config{Routes: rt, Datapath: dp, + OwnLocator: func(context.Context) (netip.Prefix, error) { return locOwn, nil }}) + if err != nil { + t.Fatal(err) + } + return &fixture{t: t, dp: dp, rt: rt, m: m} +} + +func route(vpc string, table uint32, p netip.Prefix, sid netip.Addr) routes.Route { + return routes.Route{Key: routes.Key{VPC: vpc, Prefix: p}, TableID: table, SID: sid} +} + +func dest(id string, loc netip.Prefix, addrs ...netip.Addr) module.Destination { + return module.Destination{ID: id, Mode: workerv1alpha1.Mode_MODE_FABRIC, VPC: vpcA, Locator: loc, Addresses: addrs} +} + +// claim is claimA with a destination on node B (guest2) and one on node C +// (guest3). +func claim() []module.Claim { + return []module.Claim{{Key: claimA, Generation: 3, Handoff: workerv1alpha1.Handoff_HANDOFF_VETH_PAIR, + Destinations: []module.Destination{dest("dB", locB, guest2), dest("dC", locC, guest3)}}} +} + +// reconcile runs Plan, Apply and Read for vpc, as Worker does. +func (f *fixture) reconcile(vpc string, desired []module.Claim) (module.Change, module.State) { + f.t.Helper() + ctx := context.Background() + scope := module.Scope{VPC: vpc} + ch, err := f.m.Plan(ctx, scope, desired) + if err != nil { + f.t.Fatalf("Plan(%s): %v", vpc, err) + } + if !ch.Empty() { + if err := f.m.Apply(ctx, scope, ch); err != nil { + f.t.Fatalf("Apply(%s): %v", vpc, err) + } + } + st, err := f.m.Read(ctx, scope) + if err != nil { + f.t.Fatalf("Read(%s): %v", vpc, err) + } + return ch, st +} + +func (f *fixture) sid(table uint32, p netip.Prefix) netip.Addr { + f.t.Helper() + r, ok := f.dp.Entries(table)[p] + if !ok { + return netip.Addr{} + } + a, _ := netip.AddrFromSlice(r.SID) + return a +} + +func wantState(t *testing.T, st module.State, id string, programmed, forward, ret bool, reason string) { + t.Helper() + got, ok := st[claimA][id] + if !ok { + t.Fatalf("no state for %s", id) + } + if got.Programmed != programmed || got.ForwardReady != forward || got.ReturnReady != ret || + got.Ready != (forward && ret) || got.Reason != reason || got.Generation != 3 { + t.Fatalf("%s = %+v; want programmed=%t forward=%t return=%t reason=%q", id, got, programmed, forward, ret, + reason) + } +} + +func TestPlanWritesTenantRoutesPerVPCTable(t *testing.T) { + f := newFixture(t) + f.rt.Apply([]routes.Route{ + route(vpcA, tableA, host2, sidBA), route(vpcA, tableA, host3, sidCA), + route(vpcB, tableB, host2, sidBB), + }, nil) + + ch, st := f.reconcile(vpcA, claim()) + if ch.Empty() { + t.Fatal("first plan wrote nothing") + } + f.reconcile(vpcB, nil) + if got := f.sid(tableA, host2); got != sidBA { + t.Errorf("table A %s -> %s, want %s", host2, got, sidBA) + } + if got := f.sid(tableB, host2); got != sidBB { + t.Errorf("table B %s -> %s, want %s", host2, got, sidBB) + } + if got := f.sid(tableA, host3); got != sidCA { + t.Errorf("table A %s -> %s, want %s", host3, got, sidCA) + } + wantState(t, st, "dB", true, true, true, "") + wantState(t, st, "dC", true, true, true, "") + + if ch, _ := f.reconcile(vpcA, claim()); !ch.Empty() { + t.Errorf("second plan is not empty: %+v", ch) + } +} + +func TestPlanSkipsForeignTableAndOwnLocator(t *testing.T) { + f := newFixture(t) + f.rt.Apply([]routes.Route{ + // Recorded for VPC B's table: writing it would cross VPCs. + route(vpcA, tableB, host2, sidBA), + // Toward this node: writing it would loop. + route(vpcA, tableA, host4, sidOwnA), + }, nil) + ch, _ := f.reconcile(vpcA, nil) + if !ch.Empty() { + t.Fatalf("plan is not empty: %+v", ch) + } + if n := len(f.dp.Entries(tableA)) + len(f.dp.Entries(tableB)); n != 0 { + t.Fatalf("%d entries written, want none", n) + } +} + +func TestWithdrawalRemovesOnlyItsEntry(t *testing.T) { + f := newFixture(t) + passThrough := netip.MustParsePrefix("2001:db8:a::10/128") + def := netip.MustParsePrefix("::/0") + f.dp.Put(tableA, passThrough, egressroutemap.Row{PassThrough: true}) + f.dp.Put(tableA, def, egressroutemap.Row{Sentinel: true, LinkIndex: -1}) + f.rt.Apply([]routes.Route{route(vpcA, tableA, host2, sidBA), route(vpcA, tableA, host3, sidCA)}, nil) + f.reconcile(vpcA, nil) + + f.rt.Apply(nil, []routes.Key{ + {VPC: vpcA, Prefix: host2}, + // Never held: removes only an encapsulating entry of its own. + {VPC: vpcA, Prefix: passThrough}, + }) + f.reconcile(vpcA, nil) + got := f.dp.Entries(tableA) + if _, ok := got[host2]; ok { + t.Error("withdrawn route's entry kept") + } + for _, p := range []netip.Prefix{host3, passThrough, def} { + if _, ok := got[p]; !ok { + t.Errorf("entry %s removed", p) + } + } + for _, w := range f.rt.Withdrawn(vpcA) { + if !w.Removed { + t.Errorf("withdrawal %s not confirmed", w.Prefix) + } + } + if f.rt.HasWork(vpcA) != true { + t.Error("VPC A has routes but no work") + } +} + +func TestReadinessChangesOnlyForTheAffectedDestination(t *testing.T) { + f := newFixture(t) + f.rt.Apply([]routes.Route{route(vpcA, tableA, host2, sidBA), route(vpcA, tableA, host3, sidCA)}, nil) + _, st := f.reconcile(vpcA, claim()) + wantState(t, st, "dC", true, true, true, "") + + // The underlay route toward node C is lost: its entry is kept, not + // deleted, and only dC stops being ready. + f.dp.Unreach(locC, fmt.Errorf("%w: test", egressroutemap.ErrNoUplinkRoute)) + writes, removes := f.dp.Counts() + _, st = f.reconcile(vpcA, claim()) + if w, r := f.dp.Counts(); w != writes || r != removes { + t.Errorf("underlay loss wrote %d and removed %d entries, want none", w-writes, r-removes) + } + if _, ok := f.dp.Entries(tableA)[host3]; !ok { + t.Error("entry toward the unreachable node was removed") + } + wantState(t, st, "dB", true, true, true, "") + wantState(t, st, "dC", true, false, true, workerv1alpha1.ReasonUnderlayUnreachable) + + // It comes back through another neighbor: the entry is rewritten and dC + // recovers. + f.dp.Reach(locC, linkB, "02:00:00:00:00:0d") + _, st = f.reconcile(vpcA, claim()) + if r := f.dp.Entries(tableA)[host3]; r.LinkIndex != linkB || r.DMAC.String() != "02:00:00:00:00:0d" { + t.Errorf("entry toward node C not moved to the new next hop: %+v", r) + } + wantState(t, st, "dC", true, true, true, "") + + // The neighbor toward node B is lost. + f.dp.Unreach(locB, fmt.Errorf("%w: test", egressroutemap.ErrNeighborUnresolved)) + _, st = f.reconcile(vpcA, claim()) + wantState(t, st, "dB", true, false, true, workerv1alpha1.ReasonNeighborUnresolved) + wantState(t, st, "dC", true, true, true, "") +} + +func TestRouteWithdrawnThenRestored(t *testing.T) { + f := newFixture(t) + r2 := route(vpcA, tableA, host2, sidBA) + f.rt.Apply([]routes.Route{r2, route(vpcA, tableA, host3, sidCA)}, nil) + f.reconcile(vpcA, claim()) + + f.rt.Apply(nil, []routes.Key{r2.Key}) + _, st := f.reconcile(vpcA, claim()) + wantState(t, st, "dB", false, false, true, workerv1alpha1.ReasonRouteWithdrawn) + wantState(t, st, "dC", true, true, true, "") + + f.rt.Apply([]routes.Route{r2}, nil) + _, st = f.reconcile(vpcA, claim()) + wantState(t, st, "dB", true, true, true, "") +} + +func TestRouteToAnotherNodeIsAMismatch(t *testing.T) { + f := newFixture(t) + // guest2 is on node B, but the route covering it leads to node C. + f.rt.Apply([]routes.Route{route(vpcA, tableA, host2, sidCA), route(vpcA, tableA, host3, sidCA)}, nil) + _, st := f.reconcile(vpcA, claim()) + wantState(t, st, "dB", false, false, true, workerv1alpha1.ReasonRouteMismatch) + wantState(t, st, "dC", true, true, true, "") +} + +func TestMissingRouteAndAddresses(t *testing.T) { + f := newFixture(t) + f.rt.Apply([]routes.Route{route(vpcA, tableA, host3, sidCA)}, nil) + desired := []module.Claim{{Key: claimA, Generation: 3, Destinations: []module.Destination{ + dest("dB", locB, guest2), dest("dNone", locB), + }}} + _, st := f.reconcile(vpcA, desired) + wantState(t, st, "dB", false, false, true, workerv1alpha1.ReasonForwardRouteMissing) + wantState(t, st, "dNone", false, false, true, workerv1alpha1.ReasonDestinationAddressUnknown) +} + +func TestReturnPathIsIndependentOfForward(t *testing.T) { + f := newFixture(t) + f.rt.Apply([]routes.Route{route(vpcA, tableA, host2, sidBA), route(vpcA, tableA, host3, sidCA)}, nil) + f.dp.SetReturn(vpcA, errors.New("no vrf_table row")) + _, st := f.reconcile(vpcA, claim()) + wantState(t, st, "dB", true, true, false, workerv1alpha1.ReasonReturnPathNotProgrammed) + f.dp.SetReturn(vpcA, nil) + _, st = f.reconcile(vpcA, claim()) + wantState(t, st, "dB", true, true, true, "") +} + +func TestTableMoveRemovesTheOldEntry(t *testing.T) { + f := newFixture(t) + f.rt.Apply([]routes.Route{route(vpcA, tableA, host2, sidBA)}, nil) + f.reconcile(vpcA, nil) + const moved = 150 + f.dp.SetTable(vpcA, moved) + f.rt.Apply([]routes.Route{route(vpcA, moved, host2, sidBA)}, nil) + f.reconcile(vpcA, nil) + if _, ok := f.dp.Entries(tableA)[host2]; ok { + t.Error("entry in the old table kept") + } + if got := f.sid(moved, host2); got != sidBA { + t.Errorf("new table %s -> %s, want %s", host2, got, sidBA) + } +} + +func TestWriteFailureIsIsolated(t *testing.T) { + f := newFixture(t) + f.dp.Unreach(locC, fmt.Errorf("%w: test", egressroutemap.ErrNeighborUnresolved)) + f.rt.Apply([]routes.Route{route(vpcA, tableA, host2, sidBA), route(vpcA, tableA, host3, sidCA)}, nil) + _, st := f.reconcile(vpcA, claim()) + wantState(t, st, "dB", true, true, true, "") + wantState(t, st, "dC", false, false, true, workerv1alpha1.ReasonNeighborUnresolved) +} + +func TestEventsRequeueOnlyTouchedVPCs(t *testing.T) { + f := newFixture(t) + f.rt.Apply([]routes.Route{ + route(vpcA, tableA, host2, sidBA), route(vpcA, tableA, host3, sidCA), + route(vpcB, tableB, host2, sidBB), + }, nil) + f.reconcile(vpcA, nil) + f.reconcile(vpcB, nil) + + ipnet := func(p netip.Prefix) *net.IPNet { + return &net.IPNet{IP: net.IP(p.Addr().AsSlice()), Mask: net.CIDRMask(p.Bits(), 128)} + } + cases := []struct { + name string + got []string + want []string + }{ + {"RouteToC", f.m.routeEvent(netlink.Route{Dst: ipnet(locC)}), []string{vpcA}}, + {"RouteToB", f.m.routeEvent(netlink.Route{Dst: ipnet(locB)}), []string{vpcA, vpcB}}, + {"Default", f.m.routeEvent(netlink.Route{}), []string{vpcA, vpcB}}, + {"OtherTable", f.m.routeEvent(netlink.Route{Dst: ipnet(locC), Table: 1000}), nil}, + {"NeighborC", f.m.neighborEvent(netlink.Neigh{LinkIndex: linkC, IP: net.IP(locC.Addr().AsSlice())}), + []string{vpcA}}, + {"NeighborElsewhere", f.m.neighborEvent(netlink.Neigh{LinkIndex: 99, IP: net.IP(locC.Addr().AsSlice())}), + nil}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + slices.Sort(c.got) + if !slices.Equal(c.got, c.want) { + t.Errorf("got %v, want %v", c.got, c.want) + } + }) + } +} + +func TestCapabilitiesNeedTheDatapath(t *testing.T) { + f := newFixture(t) + for _, c := range f.m.Capabilities() { + if !c.Available { + t.Errorf("%s unavailable with the datapath loaded", c.Name) + } + } + f.dp.SetLoaded(errors.New("no pinned maps")) + got := f.m.Capabilities() + if len(got) != 2 { + t.Fatalf("got %d capabilities, want fabric.srv6 and fabric.routes", len(got)) + } + for _, c := range got { + if c.Available || c.Reason != workerv1alpha1.ReasonDatapathNotLoaded { + t.Errorf("%s = %+v, want DatapathNotLoaded", c.Name, c) + } + } +} + +func TestPlanOfAQuietVPCReadsNothing(t *testing.T) { + f := newFixture(t) + m, err := New(Config{Routes: f.rt, Datapath: f.dp, + OwnLocator: func(context.Context) (netip.Prefix, error) { return netip.Prefix{}, errors.New("API down") }}) + if err != nil { + t.Fatal(err) + } + ch, err := m.Plan(context.Background(), module.Scope{VPC: vpcA}, nil) + if err != nil || !ch.Empty() { + t.Fatalf("Plan of a VPC with no routes = %v, %v; want empty and no read", ch, err) + } +} + +func TestRouteUnderACoveringRouteGetsItsOwnEntry(t *testing.T) { + f := newFixture(t) + p64 := netip.MustParsePrefix("2001:db8:a::/64") + f.rt.Apply([]routes.Route{route(vpcA, tableA, p64, sidBA), route(vpcA, tableA, host2, sidBA)}, nil) + f.reconcile(vpcA, nil) + if _, ok := f.dp.Entries(tableA)[host2]; !ok { + t.Fatal("the /128 got no entry of its own under a /64 toward the same SID") + } + f.rt.Apply(nil, []routes.Key{{VPC: vpcA, Prefix: p64}}) + f.reconcile(vpcA, nil) + if got := f.sid(tableA, host2); got != sidBA { + t.Fatalf("after the /64's withdrawal the /128 leads to %s, want %s", got, sidBA) + } +} + +func TestWithdrawalAfterTheVRFIsGone(t *testing.T) { + f := newFixture(t) + f.rt.Apply([]routes.Route{route(vpcA, tableA, host2, sidBA)}, nil) + f.reconcile(vpcA, nil) + f.dp.SetTable(vpcA, 0) + f.rt.Apply(nil, []routes.Key{{VPC: vpcA, Prefix: host2}}) + f.reconcile(vpcA, nil) + if _, ok := f.dp.Entries(tableA)[host2]; ok { + t.Fatal("withdrawn entry kept in the old table once the VRF was gone") + } + if f.rt.HasWork(vpcA) { + t.Fatal("the withdrawal is still pending") + } +} diff --git a/internal/worker/fabric/fabrictest/fabrictest.go b/internal/worker/fabric/fabrictest/fabrictest.go new file mode 100644 index 00000000..4ef44faf --- /dev/null +++ b/internal/worker/fabric/fabrictest/fabrictest.go @@ -0,0 +1,213 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package fabrictest is an in-memory fabric.Datapath for tests. +package fabrictest + +import ( + "context" + "errors" + "fmt" + "maps" + "net" + "net/netip" + "sync" + + "go.datum.net/galactic/internal/plumbing/ebpf/egressroutemap" +) + +// Datapath is an in-memory fabric.Datapath. Lookup is a longest-prefix match +// over exact entries, as egress_route_table's is. +type Datapath struct { + mu sync.Mutex + loadErr error + tables map[string]uint32 + rows map[uint32]map[netip.Prefix]egressroutemap.Row + hops map[netip.Prefix]hopResult + returnErr map[string]error + writes int + removes int +} + +type hopResult struct { + hop egressroutemap.NextHop + err error +} + +// New returns an empty datapath. +func New() *Datapath { + return &Datapath{ + tables: map[string]uint32{}, + rows: map[uint32]map[netip.Prefix]egressroutemap.Row{}, + hops: map[netip.Prefix]hopResult{}, + returnErr: map[string]error{}, + } +} + +// SetLoaded sets the error Loaded returns. +func (d *Datapath) SetLoaded(err error) { + d.mu.Lock() + defer d.mu.Unlock() + d.loadErr = err +} + +// SetTable gives vpc a host VRF on table, or removes it when table is 0. +func (d *Datapath) SetTable(vpc string, table uint32) { + d.mu.Lock() + defer d.mu.Unlock() + if table == 0 { + delete(d.tables, vpc) + return + } + d.tables[vpc] = table +} + +// Reach makes every SID in locator resolve through link toward a neighbor +// with MAC dmac. +func (d *Datapath) Reach(locator netip.Prefix, link int, dmac string) { + mac, err := net.ParseMAC(dmac) + if err != nil { + panic(err) + } + d.mu.Lock() + defer d.mu.Unlock() + d.hops[locator] = hopResult{hop: egressroutemap.NextHop{ + LinkIndex: link, Gateway: net.IP(locator.Addr().AsSlice()), DMAC: mac, + SMAC: net.HardwareAddr{0x02, 0, 0, 0, 0, byte(link)}, + }} +} + +// Unreach makes every SID in locator fail to resolve with err, which wraps +// ErrNoUplinkRoute or ErrNeighborUnresolved. A neighbor failure keeps the +// link and gateway. +func (d *Datapath) Unreach(locator netip.Prefix, err error) { + d.mu.Lock() + defer d.mu.Unlock() + r := d.hops[locator] + r.err = err + if errors.Is(err, egressroutemap.ErrNoUplinkRoute) { + r.hop = egressroutemap.NextHop{} + } else { + r.hop.DMAC = nil + } + d.hops[locator] = r +} + +// SetReturn sets the error ReturnPath returns for vpc. +func (d *Datapath) SetReturn(vpc string, err error) { + d.mu.Lock() + defer d.mu.Unlock() + d.returnErr[vpc] = err +} + +// Put writes an entry directly, as another writer would. +func (d *Datapath) Put(table uint32, prefix netip.Prefix, row egressroutemap.Row) { + d.mu.Lock() + defer d.mu.Unlock() + if d.rows[table] == nil { + d.rows[table] = map[netip.Prefix]egressroutemap.Row{} + } + d.rows[table][prefix.Masked()] = row +} + +// Entries returns table's exact entries. +func (d *Datapath) Entries(table uint32) map[netip.Prefix]egressroutemap.Row { + d.mu.Lock() + defer d.mu.Unlock() + return maps.Clone(d.rows[table]) +} + +// Counts returns how many writes and removals ran. +func (d *Datapath) Counts() (writes, removes int) { + d.mu.Lock() + defer d.mu.Unlock() + return d.writes, d.removes +} + +// Loaded implements fabric.Datapath. +func (d *Datapath) Loaded() error { + d.mu.Lock() + defer d.mu.Unlock() + return d.loadErr +} + +// TableID implements fabric.Datapath. +func (d *Datapath) TableID(vpc string) (uint32, bool, error) { + d.mu.Lock() + defer d.mu.Unlock() + t, ok := d.tables[vpc] + return t, ok, nil +} + +// Lookup implements fabric.Datapath. +func (d *Datapath) Lookup(table uint32, prefix netip.Prefix) (egressroutemap.Row, bool, error) { + d.mu.Lock() + defer d.mu.Unlock() + var ( + best egressroutemap.Row + bits = -1 + ) + for p, r := range d.rows[table] { + if p.Bits() <= prefix.Bits() && p.Contains(prefix.Addr()) && p.Bits() > bits { + best, bits = r, p.Bits() + } + } + return best, bits >= 0, nil +} + +// Write implements fabric.Datapath. +func (d *Datapath) Write(table uint32, prefix netip.Prefix, sid netip.Addr) error { + hop, err := d.NextHop(sid) + if err != nil && !errors.Is(err, egressroutemap.ErrNeighborUnresolved) { + return err + } + if err != nil { + return fmt.Errorf("solicit: %w", err) + } + d.mu.Lock() + defer d.mu.Unlock() + if d.rows[table] == nil { + d.rows[table] = map[netip.Prefix]egressroutemap.Row{} + } + d.rows[table][prefix.Masked()] = egressroutemap.Row{ + SID: net.IP(sid.AsSlice()), LinkIndex: hop.LinkIndex, DMAC: hop.DMAC, SMAC: hop.SMAC, + } + d.writes++ + return nil +} + +// Remove implements fabric.Datapath. +func (d *Datapath) Remove(table uint32, prefix netip.Prefix, sid netip.Addr) error { + d.mu.Lock() + defer d.mu.Unlock() + r, ok := d.rows[table][prefix.Masked()] + if !ok || r.PassThrough || r.Sentinel { + return nil + } + if got, _ := netip.AddrFromSlice(r.SID); sid.IsValid() && got.Unmap() != sid { + return nil + } + delete(d.rows[table], prefix.Masked()) + d.removes++ + return nil +} + +// NextHop implements fabric.Datapath. +func (d *Datapath) NextHop(sid netip.Addr) (egressroutemap.NextHop, error) { + d.mu.Lock() + defer d.mu.Unlock() + for loc, r := range d.hops { + if loc.Contains(sid) { + return r.hop, r.err + } + } + return egressroutemap.NextHop{}, fmt.Errorf("%w: no route to %s", egressroutemap.ErrNoUplinkRoute, sid) +} + +// ReturnPath implements fabric.Datapath. +func (d *Datapath) ReturnPath(_ context.Context, vpc string) error { + d.mu.Lock() + defer d.mu.Unlock() + return d.returnErr[vpc] +} diff --git a/internal/worker/fabric/kernel_root_test.go b/internal/worker/fabric/kernel_root_test.go new file mode 100644 index 00000000..83559aaa --- /dev/null +++ b/internal/worker/fabric/kernel_root_test.go @@ -0,0 +1,271 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package fabric + +import ( + "context" + "fmt" + "net" + "net/netip" + "os" + "slices" + "sync" + "testing" + "time" + + "github.com/containernetworking/plugins/pkg/ns" + "github.com/vishvananda/netlink" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + "go.datum.net/galactic/internal/config" + "go.datum.net/galactic/internal/plumbing/ebpf/attach" + "go.datum.net/galactic/internal/plumbing/ebpf/egressroutemap" + "go.datum.net/galactic/internal/plumbing/intf" + "go.datum.net/galactic/internal/worker/module" + "go.datum.net/galactic/internal/worker/routes" +) + +func requireRoot(t *testing.T) { + t.Helper() + if os.Geteuid() != 0 { + t.Skip("test requires root to load the datapath and create a network namespace; run task test:unit-root") + } +} + +// rootKernel is the Kernel datapath with the return path, which reads the +// node's BGPRouter and BGPVRFInstances, stubbed out. +type rootKernel struct{ *Kernel } + +func (rootKernel) ReturnPath(context.Context, string) error { return nil } + +const ( + uplinkName = "fab0" + uplinkAddr = "2001:db8:ff::1/64" +) + +var ( + gwB = net.ParseIP("2001:db8:ff::b") + gwC = net.ParseIP("2001:db8:ff::c") + macB = net.HardwareAddr{0x02, 0, 0, 0, 0, 0x0b} + macC = net.HardwareAddr{0x02, 0, 0, 0, 0, 0x0c} + macD = net.HardwareAddr{0x02, 0, 0, 0, 0, 0x0d} +) + +// rootNet is a network namespace with an SRv6 uplink, a VRF for each of two +// VPCs, and underlay routes toward the locators of nodes B and C. +type rootNet struct { + t *testing.T + ns ns.NetNS + uplink int +} + +func newRootNet(t *testing.T) *rootNet { + t.Helper() + nsObj, err := ns.TempNetNS() + if err != nil { + t.Fatalf("create netns: %v", err) + } + t.Cleanup(func() { _ = nsObj.Close() }) + t.Setenv(config.EnvCNIEBPFInterfaces, uplinkName) + attach.InvalidateUplinkIndexes() + t.Cleanup(attach.InvalidateUplinkIndexes) + n := &rootNet{t: t, ns: nsObj} + n.do(func() error { + dummy := &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: uplinkName}} + if err := netlink.LinkAdd(dummy); err != nil { + return err + } + if err := netlink.LinkSetUp(dummy); err != nil { + return err + } + addr, _ := netlink.ParseAddr(uplinkAddr) + if err := netlink.AddrAdd(dummy, addr); err != nil { + return err + } + link, err := netlink.LinkByName(uplinkName) + if err != nil { + return err + } + n.uplink = link.Attrs().Index + for vpc, table := range map[string]uint32{vpcA: tableA, vpcB: tableB} { + v := &netlink.Vrf{LinkAttrs: netlink.LinkAttrs{Name: intf.GenerateInterfaceNameVRF(vpc)}, Table: table} + if err := netlink.LinkAdd(v); err != nil { + return fmt.Errorf("add VRF for %s: %w", vpc, err) + } + } + return nil + }) + n.underlay(locB, gwB, macB) + n.underlay(locC, gwC, macC) + return n +} + +func (n *rootNet) do(fn func() error) { + n.t.Helper() + if err := n.ns.Do(func(ns.NetNS) error { return fn() }); err != nil { + n.t.Fatal(err) + } +} + +func ipnet(p netip.Prefix) *net.IPNet { + return &net.IPNet{IP: net.IP(p.Addr().AsSlice()), Mask: net.CIDRMask(p.Bits(), 128)} +} + +// underlay routes loc via gw and resolves gw to mac. +func (n *rootNet) underlay(loc netip.Prefix, gw net.IP, mac net.HardwareAddr) { + n.t.Helper() + n.do(func() error { + if err := netlink.RouteReplace(&netlink.Route{LinkIndex: n.uplink, Dst: ipnet(loc), Gw: gw}); err != nil { + return fmt.Errorf("route %s: %w", loc, err) + } + return n.neighbor(gw, mac) + }) +} + +func (n *rootNet) neighbor(gw net.IP, mac net.HardwareAddr) error { + return netlink.NeighSet(&netlink.Neigh{LinkIndex: n.uplink, Family: netlink.FAMILY_V6, + State: netlink.NUD_PERMANENT, IP: gw, HardwareAddr: mac}) +} + +func TestKernelDatapathRoot(t *testing.T) { + requireRoot(t) + pinDir := fmt.Sprintf("/sys/fs/bpf/galactic-fabric-test-%d", os.Getpid()) + t.Cleanup(func() { _ = os.RemoveAll(pinDir) }) + objs, err := attach.Load(pinDir) + if err != nil { + t.Fatalf("load datapath: %v", err) + } + t.Cleanup(func() { _ = objs.Close() }) + + n := newRootNet(t) + kernel := &Kernel{PinDir: pinDir} + t.Cleanup(func() { _ = kernel.Close() }) + rt := routes.New() + m, err := New(Config{Routes: rt, Datapath: rootKernel{kernel}, + OwnLocator: func(context.Context) (netip.Prefix, error) { return locOwn, nil }}) + if err != nil { + t.Fatal(err) + } + f := &fixture{t: t, rt: rt, m: m} + reconcile := func(vpc string, desired []module.Claim) (st module.State) { + n.do(func() error { _, st = f.reconcile(vpc, desired); return nil }) + return st + } + entry := func(table uint32, p netip.Prefix) (egressroutemap.Row, bool) { + var ( + row egressroutemap.Row + ok bool + ) + n.do(func() error { + var err error + row, ok, err = kernel.Lookup(table, p) + if ok && row.Sentinel { + ok = false + } + return err + }) + return row, ok + } + exact := func(table uint32, p netip.Prefix, sid netip.Addr, mac net.HardwareAddr) { + t.Helper() + row, ok := entry(table, p) + if !ok || !sameSID(row.SID, sid) || row.LinkIndex != n.uplink || row.DMAC.String() != mac.String() { + t.Fatalf("table %d %s = %+v (present %t), want %s via %s", table, p, row, ok, sid, mac) + } + } + + rt.Apply([]routes.Route{ + route(vpcA, tableA, host2, sidBA), route(vpcA, tableA, host3, sidCA), + route(vpcB, tableB, host2, sidBB), + }, nil) + st := reconcile(vpcA, claim()) + reconcile(vpcB, nil) + exact(tableA, host2, sidBA, macB) + exact(tableA, host3, sidCA, macC) + exact(tableB, host2, sidBB, macB) + wantState(t, st, "dB", true, true, true, "") + wantState(t, st, "dC", true, true, true, "") + + // Requeues from now on name only the VPCs an event touches. + var ( + mu sync.Mutex + changed []string + ) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + n.do(func() error { + return m.Watch(ctx, func(vpc string) { + mu.Lock() + defer mu.Unlock() + if !slices.Contains(changed, vpc) { + changed = append(changed, vpc) + } + }, func(err error) { t.Errorf("subscription: %v", err) }) + }) + waitChanged := func(want []string) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for { + mu.Lock() + got := slices.Clone(changed) + mu.Unlock() + slices.Sort(got) + if slices.Equal(got, want) { + break + } + if time.Now().After(deadline) { + t.Fatalf("requeued %v, want %v", got, want) + } + time.Sleep(10 * time.Millisecond) + } + time.Sleep(100 * time.Millisecond) + mu.Lock() + defer mu.Unlock() + if got := slices.Sorted(slices.Values(changed)); !slices.Equal(got, want) { + t.Fatalf("requeued %v, want %v", got, want) + } + changed = nil + } + + // Node C's neighbor is lost: only dC, and only VPC A, are affected; the + // entry is kept. + n.do(func() error { + return netlink.NeighDel(&netlink.Neigh{LinkIndex: n.uplink, Family: netlink.FAMILY_V6, IP: gwC}) + }) + waitChanged([]string{vpcA}) + st = reconcile(vpcA, claim()) + wantState(t, st, "dB", true, true, true, "") + wantState(t, st, "dC", true, false, true, workerv1alpha1.ReasonNeighborUnresolved) + exact(tableA, host3, sidCA, macC) + + // It comes back with a new MAC: the entry follows it and dC recovers. + n.do(func() error { return n.neighbor(gwC, macD) }) + waitChanged([]string{vpcA}) + st = reconcile(vpcA, claim()) + exact(tableA, host3, sidCA, macD) + wantState(t, st, "dC", true, true, true, "") + + // The underlay route toward node B is withdrawn: both VPCs use it. + n.do(func() error { return netlink.RouteDel(&netlink.Route{LinkIndex: n.uplink, Dst: ipnet(locB), Gw: gwB}) }) + waitChanged([]string{vpcA, vpcB}) + st = reconcile(vpcA, claim()) + wantState(t, st, "dB", true, false, true, workerv1alpha1.ReasonUnderlayUnreachable) + wantState(t, st, "dC", true, true, true, "") + exact(tableA, host2, sidBA, macB) + + n.underlay(locB, gwB, macB) + waitChanged([]string{vpcA, vpcB}) + st = reconcile(vpcA, claim()) + wantState(t, st, "dB", true, true, true, "") + + // VPC A's route to the shared guest address is withdrawn; VPC B's stays. + rt.Apply(nil, []routes.Key{{VPC: vpcA, Prefix: host2}}) + st = reconcile(vpcA, claim()) + if _, ok := entry(tableA, host2); ok { + t.Error("withdrawn entry kept in VPC A's table") + } + exact(tableB, host2, sidBB, macB) + wantState(t, st, "dB", false, false, true, workerv1alpha1.ReasonRouteWithdrawn) +} diff --git a/internal/worker/fabric/metrics.go b/internal/worker/fabric/metrics.go new file mode 100644 index 00000000..f941a3a8 --- /dev/null +++ b/internal/worker/fabric/metrics.go @@ -0,0 +1,37 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package fabric + +import ( + "github.com/prometheus/client_golang/prometheus" + "go.miloapis.com/instrumentation/labels" + "go.miloapis.com/instrumentation/metrics" +) + +// Reasons a tenant route is skipped rather than written. +const ( + // reasonTableMismatch is a route galactic-router recorded for another + // table than the VPC's current one: a VRF that moved, or a route that + // would cross into another VPC's table. + reasonTableMismatch = "TableMismatch" + // reasonOwnLocator is a route toward this node's own locator, which + // would loop back into this node. + reasonOwnLocator = "OwnLocator" +) + +var routesSkippedTotal = metrics.NewCounterVec(prometheus.CounterOpts{ + Name: "galactic_worker_fabric_routes_skipped_total", + Help: "Tenant routes the Fabric module did not write, by reason, counted on each reconcile pass that skipped them.", +}, []string{labels.Reason}) + +// InitMetrics creates the module's series at zero, so a scrape shows them +// before the first skip. +func InitMetrics() { + for _, r := range []string{reasonTableMismatch, reasonOwnLocator} { + routesSkippedTotal.WithLabelValues(r) + } +} + +func skipped(reason string) { routesSkippedTotal.WithLabelValues(reason).Inc() } diff --git a/internal/worker/fabric/watch.go b/internal/worker/fabric/watch.go new file mode 100644 index 00000000..08447853 --- /dev/null +++ b/internal/worker/fabric/watch.go @@ -0,0 +1,100 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package fabric + +import ( + "context" + "fmt" + "net/netip" + + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" +) + +// Watch calls changed with each VPC whose tenant routes an underlay event in +// the calling process's network namespace touches: a main-table route change +// covering one of the VPC's SIDs, or a neighbor change for a next hop its +// entries use. It returns once subscribed and stops when ctx is done. A lost +// subscription is reported through errs; Worker's periodic resync still +// covers every VPC. +func (m *Module) Watch(ctx context.Context, changed func(vpc string), errs func(error)) error { + routeUpdates := make(chan netlink.RouteUpdate, 256) + neighUpdates := make(chan netlink.NeighUpdate, 256) + done := make(chan struct{}) + // Closing the subscriptions once ctx is done fails their reads, which is + // no error. + report := errs + errs = func(err error) { + if ctx.Err() == nil && report != nil { + report(err) + } + } + if err := netlink.RouteSubscribeWithOptions(routeUpdates, done, netlink.RouteSubscribeOptions{ + ErrorCallback: errs, + }); err != nil { + close(done) + return fmt.Errorf("subscribe to route events: %w", err) + } + if err := netlink.NeighSubscribeWithOptions(neighUpdates, done, netlink.NeighSubscribeOptions{ + ErrorCallback: errs, + }); err != nil { + close(done) + return fmt.Errorf("subscribe to neighbor events: %w", err) + } + go func() { + defer close(done) + for { + select { + case <-ctx.Done(): + return + case u, ok := <-routeUpdates: + if !ok { + return + } + for _, vpc := range m.routeEvent(u.Route) { + changed(vpc) + } + case u, ok := <-neighUpdates: + if !ok { + return + } + for _, vpc := range m.neighborEvent(u.Neigh) { + changed(vpc) + } + } + } + }() + return nil +} + +// routeEvent returns the VPCs a change to r touches. Only main-table routes +// carry the underlay toward other nodes' locators. +func (m *Module) routeEvent(r netlink.Route) []string { + if r.Table != 0 && r.Table != unix.RT_TABLE_MAIN { + return nil + } + dst := netip.PrefixFrom(netip.IPv6Unspecified(), 0) + if r.Dst != nil { + addr, ok := netip.AddrFromSlice(r.Dst.IP) + if !ok { + return nil + } + ones, _ := r.Dst.Mask.Size() + dst = netip.PrefixFrom(addr.Unmap(), ones) + } + if !dst.Addr().Is6() { + return nil + } + return m.affectedByRoute(dst) +} + +// neighborEvent returns the VPCs whose entries use the neighbor n. +func (m *Module) neighborEvent(n netlink.Neigh) []string { + addr, ok := netip.AddrFromSlice(n.IP) + if !ok { + return nil + } + return m.affectedByNeighbor(n.LinkIndex, addr) +} diff --git a/internal/worker/fabric_test.go b/internal/worker/fabric_test.go new file mode 100644 index 00000000..b5050d07 --- /dev/null +++ b/internal/worker/fabric_test.go @@ -0,0 +1,200 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package worker + +import ( + "context" + "errors" + "io" + "net/netip" + "testing" + + "google.golang.org/grpc/codes" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + "go.datum.net/galactic/internal/worker/fabric" + "go.datum.net/galactic/internal/worker/fabric/fabrictest" + "go.datum.net/galactic/internal/worker/module" + "go.datum.net/galactic/internal/worker/module/fake" + "go.datum.net/galactic/internal/worker/routes" + "go.datum.net/galactic/internal/worker/sources" +) + +const ( + tableA = 100 + uidGuestA1 = "guest-a1" + uidGuestA2 = "guest-a2" +) + +var ( + guestA1Addr = netip.MustParseAddr("2001:db8:a::1") + guestA2Addr = netip.MustParseAddr("2001:db8:a::2") + sidA2 = netip.MustParseAddr("2001:db8:0:2:e000:1::") +) + +// fabricEnv is an env running the real Fabric module over an in-memory +// datapath, with guest addresses on the attachments. +func fabricEnv(t *testing.T, withLocal bool) (*env, *fabrictest.Datapath) { + t.Helper() + e := newEnv(t) + dp := fabrictest.New() + dp.SetTable(vpcA, tableA) + dp.Reach(locB, 7, "02:00:00:00:00:0b") + e.routes = routes.New() + e.src.SetAttachment(sources.Attachment{UID: uidGuestA1, Kind: sources.KindVPCAttachment, VPC: vpcA, Node: node, + Admitted: true, Addresses: []netip.Addr{guestA1Addr}}) + e.src.SetAttachment(sources.Attachment{UID: uidGuestA2, Kind: sources.KindVPCAttachment, VPC: vpcA, + Node: otherNode, Admitted: true, Addresses: []netip.Addr{guestA2Addr}}) + e.modules = func(h *fake.Host) []module.Module { + fab, err := fabric.New(fabric.Config{Routes: e.routes, Datapath: dp, + OwnLocator: func(ctx context.Context) (netip.Prefix, error) { return e.src.Locator(ctx, node) }}) + if err != nil { + t.Fatal(err) + } + mods := []module.Module{fab} + if withLocal { + e.local = fake.New("local", workerv1alpha1.Mode_MODE_LOCAL, h) + mods = append(mods, e.local) + } + return mods + } + return e, dp +} + +// syncRoutes sends one request on a new Routes.Sync stream as caller and +// returns the response. +func (e *env) syncRoutes(caller string, req *workerv1alpha1.RoutesSyncRequest) ( + *workerv1alpha1.RoutesSyncResponse, error) { + e.t.Helper() + ctx, cancel := context.WithCancel(e.ctx(caller)) + e.t.Cleanup(cancel) + stream, err := workerv1alpha1.NewRoutesClient(e.cc).Sync(ctx) + if err != nil { + return nil, err + } + // A stream the server already ended fails Send with io.EOF; Recv then + // returns its status. + if err := stream.Send(req); err != nil && !errors.Is(err, io.EOF) { + return nil, err + } + return stream.Recv() +} + +// upsert is a request upserting one route of VPC A in its table. +func upsert(seq uint64, prefix, sid string) *workerv1alpha1.RoutesSyncRequest { + return &workerv1alpha1.RoutesSyncRequest{Sequence: seq, Upserts: []*workerv1alpha1.TenantRoute{ + {Vpc: vpcA, TableId: tableA, Prefix: prefix, Sid: sid}, + }} +} + +func TestRoutesSyncRefusesACallerWithoutTheCapability(t *testing.T) { + e, _ := fabricEnv(t, false) + e.start() + _, err := e.syncRoutes(natSA, upsert(1, "2001:db8:a::2/128", sidA2.String())) + wantRefusal(t, err, codes.PermissionDenied, workerv1alpha1.ReasonCapabilityNotAllowed) + if got := e.routes.Routes(vpcA); len(got) != 0 { + t.Fatalf("refused stream recorded %v", got) + } +} + +func TestRoutesSyncWritesRoutesOfAScopeWithoutClaims(t *testing.T) { + e, dp := fabricEnv(t, false) + e.start() + host := netip.PrefixFrom(guestA2Addr, 128) + + req := upsert(4, host.String(), sidA2.String()) + req.Upserts = append(req.Upserts, &workerv1alpha1.TenantRoute{Vpc: vpcA, TableId: tableA, + Prefix: "not-a-prefix", Sid: sidA2.String()}) + resp, err := e.syncRoutes(routerSA, req) + e.mustOK(err) + if resp.GetSequence() != 4 || len(resp.GetRejected()) != 1 || + resp.GetRejected()[0].GetReason() != workerv1alpha1.ReasonMalformedRoute { + t.Fatalf("response = %v, want sequence 4 with one MalformedRoute rejection", resp) + } + e.eventually("route written", func() bool { _, ok := dp.Entries(tableA)[host]; return ok }) + + _, err = e.syncRoutes(routerSA, &workerv1alpha1.RoutesSyncRequest{Sequence: 5, + Withdrawals: []*workerv1alpha1.TenantRouteKey{{Vpc: vpcA, Prefix: host.String()}}}) + e.mustOK(err) + e.eventually("route removed", func() bool { _, ok := dp.Entries(tableA)[host]; return !ok }) +} + +func TestMixedLocalAndFabricDestinationsBecomeReady(t *testing.T) { + e, _ := fabricEnv(t, true) + e.start() + _, err := e.syncRoutes(routerSA, upsert(1, netip.PrefixFrom(guestA2Addr, 128).String(), + sidA2.String())) + e.mustOK(err) + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth(uidGuestA1, uidGuestA2))) + + st := e.waitReady(mkey(vpcA, "svc-a"), 1) + for _, d := range st.GetDestinations() { + if !d.GetReady() || !d.GetForwardReady() || !d.GetReturnReady() { + t.Errorf("destination %v = %+v, want ready both ways", d.GetDestination(), d) + } + } + if got := st.GetDestinations()[1].GetMode(); got != workerv1alpha1.Mode_MODE_FABRIC { + t.Errorf("guest-a2 resolved to %s, want MODE_FABRIC", got) + } +} + +func TestLocalDestinationNeverFallsBackToFabric(t *testing.T) { + e, dp := fabricEnv(t, false) + e.start() + // A route covering the local guest's address exists, so Fabric could + // deliver to it; it must not. + _, err := e.syncRoutes(routerSA, upsert(1, "2001:db8:a::/120", sidA2.String())) + e.mustOK(err) + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth(uidGuestA1, uidGuestA2))) + + st := e.waitObserved(mkey(vpcA, "svc-a")) + e.eventually("fabric destination ready", func() bool { + st = e.status(mkey(vpcA, "svc-a")) + return len(st.GetDestinations()) == 2 && st.GetDestinations()[1].GetReady() + }) + local := st.GetDestinations()[0] + if local.GetMode() != workerv1alpha1.Mode_MODE_LOCAL || local.GetProgrammed() || + local.GetReason() != workerv1alpha1.ReasonModuleNotRunning { + t.Fatalf("local destination = %+v, want MODE_LOCAL unprogrammed with ModuleNotRunning", local) + } + if isTrue(st, workerv1alpha1.ConditionReady) { + t.Fatal("claim Ready with its Local destination unprogrammed") + } + if _, ok := dp.Entries(tableA)[netip.PrefixFrom(guestA1Addr, 128)]; ok { + t.Fatal("Fabric wrote an entry for the local guest") + } +} + +// TestWithdrawalDuringRepairKeepsSharedTenantRoutes is the contract's +// withdrawal-during-repair transition for Fabric: a claim owns no tenant +// route, so the snapshot that releases it removes nothing of the VPC's +// routes, which other claims and the VPC's own traffic still use. +func TestWithdrawalDuringRepairKeepsSharedTenantRoutes(t *testing.T) { + e, dp := fabricEnv(t, true) + e.start() + host := netip.PrefixFrom(guestA2Addr, 128) + _, err := e.syncRoutes(routerSA, upsert(1, host.String(), sidA2.String())) + e.mustOK(err) + e.eventually("route written", func() bool { _, ok := dp.Entries(tableA)[host]; return ok }) + e.idle() + + e.local.FailApply(1, 0) + k := mkey(vpcA, "svc-a") + entered := e.local.PausePlan() + e.mustOK(e.request(natSA, key(vpcA, "svc-a"), 1, veth(uidGuestA1, uidGuestA2))) + <-entered + entered = e.local.RepausePlan() + <-entered + if !e.isRepairing(vpcA) { + t.Fatal("scope is not in repair after the failed commit") + } + e.mustOK(e.release(natSA, key(vpcA, "svc-a"), 1)) + e.local.ResumePlan() + + e.waitGone(k) + if _, ok := dp.Entries(tableA)[host]; !ok { + t.Fatal("releasing a claim removed the VPC's tenant route") + } +} diff --git a/internal/worker/harness_test.go b/internal/worker/harness_test.go index 53dd4314..84f25b7a 100644 --- a/internal/worker/harness_test.go +++ b/internal/worker/harness_test.go @@ -32,6 +32,7 @@ import ( "go.datum.net/galactic/internal/worker/module" "go.datum.net/galactic/internal/worker/module/fake" "go.datum.net/galactic/internal/worker/refusal" + "go.datum.net/galactic/internal/worker/routes" "go.datum.net/galactic/internal/worker/sources" "go.datum.net/galactic/internal/worker/store" ) @@ -43,9 +44,10 @@ const ( vpcA = "vpcA" vpcB = "vpcB" - natSA = "galactic-system/nat" - gwSA = "galactic-system/gateway" - otherSA = "galactic-system/other" + natSA = "galactic-system/nat" + gwSA = "galactic-system/gateway" + otherSA = "galactic-system/other" + routerSA = "galactic-system/galactic-router" callerMD = "x-test-caller" ) @@ -55,9 +57,10 @@ var ( locB = netip.MustParsePrefix("2001:db8:0:2::/64") testCaps = map[string][]string{ - natSA: {workerv1alpha1.CapabilityLocalVethPair, workerv1alpha1.CapabilityFabricSRv6}, - gwSA: {workerv1alpha1.CapabilityLocalTap}, - otherSA: {workerv1alpha1.CapabilityLocalVethPair, workerv1alpha1.CapabilityFabricSRv6}, + natSA: {workerv1alpha1.CapabilityLocalVethPair, workerv1alpha1.CapabilityFabricSRv6}, + gwSA: {workerv1alpha1.CapabilityLocalTap}, + otherSA: {workerv1alpha1.CapabilityLocalVethPair, workerv1alpha1.CapabilityFabricSRv6}, + routerSA: {workerv1alpha1.CapabilityFabricRoutes}, } ) @@ -119,6 +122,9 @@ type env struct { // hostState, when set, is the VPCs' canonical host state each // incarnation reconciles. hostState hostscope.Host + // routes, when set, is the tenant route table the Routes service + // records into. + routes *routes.Table // pauseOnStart pauses the Local module's Plan before Start, so the // first reconcile of a restart holds until the test resumes it. pauseOnStart bool @@ -211,7 +217,7 @@ func (e *env) start() { e.t.Fatal(err) } w, err := New(e.cfg, Deps{Store: e.store, Fence: e.fence, Sources: e.src, Modules: reg, Auth: metaAuth{}, - Host: e.hostState, Now: e.clock.Now}) + Host: e.hostState, Routes: e.routes, Now: e.clock.Now}) if err != nil { e.t.Fatal(err) } diff --git a/internal/worker/module/fake/fake.go b/internal/worker/module/fake/fake.go index 31982f68..b193c439 100644 --- a/internal/worker/module/fake/fake.go +++ b/internal/worker/module/fake/fake.go @@ -354,6 +354,7 @@ func (m *Module) Read(_ context.Context, scope module.Scope) (module.State, erro ok, reason := ready(k, id) st.Ready, st.Reason = ok, reason } + st.ForwardReady, st.ReturnReady = st.Ready, st.Ready out[k][id] = st } } diff --git a/internal/worker/module/module.go b/internal/worker/module/module.go index 381f1eaf..ed3c7c82 100644 --- a/internal/worker/module/module.go +++ b/internal/worker/module/module.go @@ -55,6 +55,10 @@ type Destination struct { Node string // Locator is the SRv6 locator of the destination's node. Locator netip.Prefix + // Addresses are the destination's addresses, from its trusted source. + // They are empty when the attachment records none, such as for a guest + // that manages its own addressing. + Addresses []netip.Addr } // DestinationID is the identity of a destination within a claim. @@ -91,8 +95,15 @@ type DestinationState struct { // the host, confirmed by read-back. Programmed bool // Ready is true when observed state, which the module did not write, - // shows the handoff can carry traffic. + // shows the handoff can carry traffic. It equals ForwardReady and + // ReturnReady both being true. Ready bool + // ForwardReady is true when traffic from the handoff can reach the + // destination. + ForwardReady bool + // ReturnReady is true when traffic from the destination can come back to + // the handoff. + ReturnReady bool // Reason explains Programmed or Ready being false, with a contract // reason such as HandoffInterfaceMissing. Reason string diff --git a/internal/worker/reconcile.go b/internal/worker/reconcile.go index 9d2ffb40..e7178c8c 100644 --- a/internal/worker/reconcile.go +++ b/internal/worker/reconcile.go @@ -89,10 +89,7 @@ func (w *Worker) Reconcile(ctx context.Context, vpc string) error { if w.d.Host != nil { return w.reconcileHost(ctx, vpc) } - w.mu.Lock() - delete(w.scopes, vpc) - w.mu.Unlock() - return nil + return w.reconcileUnclaimed(ctx, vpc) } evals, desired, err := w.resolve(ctx, vpc, claims) @@ -117,17 +114,10 @@ func (w *Worker) Reconcile(ctx context.Context, vpc string) error { return err } - mods := w.d.Modules.Modules() - changes := make([]module.Change, len(mods)) - empty := true - for i, m := range mods { - ch, err := m.Plan(ctx, scope, desired[m.Mode()]) - if err != nil { - w.reportFailure(evals, workerv1alpha1.ReasonProgrammingFailed, err) - return fmt.Errorf("plan %s: %w", m.Name(), err) - } - changes[i] = ch - empty = empty && ch.Empty() + mods, changes, empty, err := w.planModules(ctx, scope, desired) + if err != nil { + w.reportFailure(evals, workerv1alpha1.ReasonProgrammingFailed, err) + return err } if !empty || ensure || t.Resync { mutate := func() error { @@ -138,15 +128,7 @@ func (w *Worker) Reconcile(ctx context.Context, vpc string) error { return fmt.Errorf("ensure host state: %w", err) } } - for i, m := range mods { - if changes[i].Empty() { - continue - } - if err := m.Apply(ctx, scope, changes[i]); err != nil { - return fmt.Errorf("apply %s: %w", m.Name(), err) - } - } - return nil + return applyModules(ctx, scope, mods, changes) } // Every change carries the complete desired state of the scope, so // a snapshot commit is always correct; an incremental commit is used @@ -224,7 +206,8 @@ func (w *Worker) reconcileHost(ctx context.Context, vpc string) error { } reclaim := w.unusedFor(vpc, hp.Referenced()) - t, err := w.acquire(module.Scope{VPC: vpc}) + scope := module.Scope{VPC: vpc} + t, err := w.acquire(scope) if errors.Is(err, errOwnedElsewhere) || errors.Is(err, ownership.ErrTransitioning) || errors.Is(err, ownership.ErrNotOwner) { w.log.Debug("leaving host state of a VPC another writer owns", "vpc", vpc, "reason", err) @@ -233,11 +216,20 @@ func (w *Worker) reconcileHost(ctx context.Context, vpc string) error { if err != nil { return err } - if !reclaim && !t.Resync { + // A module can hold state for a scope with no claims, such as the + // tenant routes the Fabric module writes for every VPC on the node. + mods, changes, empty, err := w.planModules(ctx, scope, nil) + if err != nil { + return err + } + if !reclaim && !t.Resync && empty { return nil } gone := false mutate := func() error { + if err := applyModules(ctx, scope, mods, changes); err != nil { + return err + } // A claim accepted since the plan keeps the state; its own // reconcile follows this one. if !reclaim || w.hasClaims(vpc) { @@ -261,6 +253,72 @@ func (w *Worker) reconcileHost(ctx context.Context, vpc string) error { return nil } +// reconcileUnclaimed reconciles a scope Worker holds no claims in when it +// manages no host state: it commits whatever the modules hold for the scope +// without claims, and stops tracking a scope that needs nothing. +func (w *Worker) reconcileUnclaimed(ctx context.Context, vpc string) error { + scope := module.Scope{VPC: vpc} + mods, changes, empty, err := w.planModules(ctx, scope, nil) + if err != nil { + return err + } + if empty { + w.mu.Lock() + if !w.hasClaimsLocked(vpc) { + delete(w.scopes, vpc) + } + w.mu.Unlock() + return nil + } + t, err := w.acquire(scope) + if errors.Is(err, errOwnedElsewhere) || errors.Is(err, ownership.ErrTransitioning) || + errors.Is(err, ownership.ErrNotOwner) { + return nil + } + if err != nil { + return err + } + commit := w.d.Fence.Commit + if t.Resync { + commit = w.d.Fence.CommitSnapshot + } + if _, err := commit(t, func() error { return applyModules(ctx, scope, mods, changes) }); err != nil { + return fmt.Errorf("commit %s: %w", scope.Name(), err) + } + return nil +} + +// planModules plans every module for scope against desired, by mode, and +// reports whether every change is empty. +func (w *Worker) planModules(ctx context.Context, scope module.Scope, + desired map[workerv1alpha1.Mode][]module.Claim) ([]module.Module, []module.Change, bool, error) { + mods := w.d.Modules.Modules() + changes := make([]module.Change, len(mods)) + empty := true + for i, m := range mods { + ch, err := m.Plan(ctx, scope, desired[m.Mode()]) + if err != nil { + return nil, nil, false, fmt.Errorf("plan %s: %w", m.Name(), err) + } + changes[i] = ch + empty = empty && ch.Empty() + } + return mods, changes, empty, nil +} + +// applyModules applies every non-empty change. It runs inside a fence commit. +func applyModules(ctx context.Context, scope module.Scope, mods []module.Module, changes []module.Change) error { + for i, m := range mods { + if changes[i].Empty() { + continue + } + if err := m.Apply(ctx, scope, changes[i]); err != nil { + return fmt.Errorf("apply %s: %w", m.Name(), err) + } + } + return nil +} + // completeTransfer takes over vpc's scope when it is transitioning away from // an earlier Worker incarnation, committing a snapshot that writes nothing. func (w *Worker) completeTransfer(vpc string) error { @@ -307,6 +365,11 @@ func (w *Worker) unusedFor(vpc string, referenced bool) bool { func (w *Worker) hasClaims(vpc string) bool { w.mu.Lock() defer w.mu.Unlock() + return w.hasClaimsLocked(vpc) +} + +// hasClaimsLocked is hasClaims with w.mu held. +func (w *Worker) hasClaimsLocked(vpc string) bool { for k := range w.claims { if k.VPC == vpc { return true @@ -448,7 +511,8 @@ func (w *Worker) resolveDestination(ctx context.Context, vpc string, own netip.P return module.Destination{}, nil } de.mode, de.resolved = mode, true - return module.Destination{ID: de.id, Ref: d, Mode: mode, VPC: att.VPC, Node: att.Node, Locator: loc}, nil + return module.Destination{ID: de.id, Ref: d, Mode: mode, VPC: att.VPC, Node: att.Node, Locator: loc, + Addresses: slices.Clone(att.Addresses)}, nil } func modeName(m workerv1alpha1.Mode) string { @@ -584,6 +648,7 @@ func (w *Worker) applyStatus(c *claim, e *claimEval, state map[workerv1alpha1.Mo default: ds.Programmed = true ds.Ready = got.Ready + ds.ForwardReady, ds.ReturnReady = got.ForwardReady, got.ReturnReady if !got.Ready { ds.Reason, ds.Message = got.Reason, got.Message ready = false diff --git a/internal/worker/routes/routes.go b/internal/worker/routes/routes.go new file mode 100644 index 00000000..31a860a4 --- /dev/null +++ b/internal/worker/routes/routes.go @@ -0,0 +1,227 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package routes holds the tenant routes galactic-router hands Worker +// through the Routes service. Worker records them here; the Fabric module +// writes them into egress_route_table and removes withdrawn ones. +// +// The table lives in memory only. galactic-router sends every route it holds +// whenever it opens a stream, so a Worker restart loses nothing it needs, and +// a route the table has never heard of is never removed from the datapath +// except by an explicit withdrawal. +package routes + +import ( + "net/netip" + "sort" + "sync" + "time" +) + +// Key names a tenant route: a prefix in one VPC. +type Key struct { + VPC string + Prefix netip.Prefix +} + +// Route is one tenant route. +type Route struct { + Key + // TableID is the kernel VRF table galactic-router resolved for the VPC. + TableID uint32 + // SID is the IPv6 SID the route encapsulates toward. A withdrawal of a + // route the table never held has no SID. + SID netip.Addr +} + +// Withdrawal is a withdrawn route whose removal from the datapath the Fabric +// module has not confirmed yet, or has confirmed (Removed). +type Withdrawal struct { + Route + Removed bool + // removedAt is when the removal was confirmed. + removedAt time.Time +} + +// removedRetention is how long a confirmed withdrawal is kept for status. +const removedRetention = time.Hour + +// Table is the set of tenant routes by VPC and the routes withdrawn from it. +// It is safe for concurrent use. +type Table struct { + // now is the clock, time.Now outside tests. + now func() time.Time + mu sync.Mutex + routes map[string]map[netip.Prefix]Route + withdrawn map[string]map[netip.Prefix]Withdrawal +} + +// New returns an empty table. +func New() *Table { + return &Table{ + now: time.Now, + routes: map[string]map[netip.Prefix]Route{}, + withdrawn: map[string]map[netip.Prefix]Withdrawal{}, + } +} + +// Apply records upserts and withdrawals, in that order, and returns the VPCs +// whose routes changed, sorted. An upsert replaces the route with the same +// key and clears its withdrawal. A withdrawal of a held route keeps the +// route's table and SID, so only the row written for it is removed; a +// withdrawal of a route the table never held has no SID. +func (t *Table) Apply(upserts []Route, withdrawals []Key) []string { + t.mu.Lock() + defer t.mu.Unlock() + changed := map[string]struct{}{} + for _, r := range upserts { + r.Prefix = r.Prefix.Masked() + if cur, ok := t.routes[r.VPC][r.Prefix]; ok && cur == r { + continue + } + if t.routes[r.VPC] == nil { + t.routes[r.VPC] = map[netip.Prefix]Route{} + } + t.routes[r.VPC][r.Prefix] = r + delete(t.withdrawn[r.VPC], r.Prefix) + changed[r.VPC] = struct{}{} + } + for _, k := range withdrawals { + k.Prefix = k.Prefix.Masked() + r, held := t.routes[k.VPC][k.Prefix] + if !held { + if w, ok := t.withdrawn[k.VPC][k.Prefix]; ok && !w.Removed { + continue + } + r = Route{Key: k} + } + delete(t.routes[k.VPC], k.Prefix) + if len(t.routes[k.VPC]) == 0 { + delete(t.routes, k.VPC) + } + if t.withdrawn[k.VPC] == nil { + t.withdrawn[k.VPC] = map[netip.Prefix]Withdrawal{} + } + t.withdrawn[k.VPC][k.Prefix] = Withdrawal{Route: r} + changed[k.VPC] = struct{}{} + } + t.pruneLocked() + out := make([]string, 0, len(changed)) + for v := range changed { + out = append(out, v) + } + sort.Strings(out) + return out +} + +// Routes returns vpc's routes, sorted by prefix. +func (t *Table) Routes(vpc string) []Route { + t.mu.Lock() + defer t.mu.Unlock() + out := make([]Route, 0, len(t.routes[vpc])) + for _, r := range t.routes[vpc] { + out = append(out, r) + } + sort.Slice(out, func(i, j int) bool { return lessPrefix(out[i].Prefix, out[j].Prefix) }) + return out +} + +// Withdrawn returns vpc's withdrawals, sorted by prefix. +func (t *Table) Withdrawn(vpc string) []Withdrawal { + t.mu.Lock() + defer t.mu.Unlock() + out := make([]Withdrawal, 0, len(t.withdrawn[vpc])) + for _, w := range t.withdrawn[vpc] { + out = append(out, w) + } + sort.Slice(out, func(i, j int) bool { return lessPrefix(out[i].Prefix, out[j].Prefix) }) + return out +} + +// ConfirmRemoved marks the withdrawals in removed as removed from the +// datapath, provided each is still withdrawn with the same table and SID. A +// removed withdrawal is kept for removedRetention so status can report the +// destination's route as withdrawn rather than never learned. +func (t *Table) ConfirmRemoved(removed []Route) { + t.mu.Lock() + defer t.mu.Unlock() + for _, r := range removed { + w, ok := t.withdrawn[r.VPC][r.Prefix] + if ok && w.Route == r { + w.Removed, w.removedAt = true, t.now() + t.withdrawn[r.VPC][r.Prefix] = w + } + } +} + +// pruneLocked drops confirmed withdrawals older than removedRetention. +func (t *Table) pruneLocked() { + now := t.now() + for vpc, ws := range t.withdrawn { + for p, w := range ws { + if w.Removed && now.Sub(w.removedAt) > removedRetention { + delete(ws, p) + } + } + if len(ws) == 0 { + delete(t.withdrawn, vpc) + } + } +} + +// Forget drops every route and withdrawal of vpc, for a VPC whose host VRF is +// gone and whose rows went with it. +func (t *Table) Forget(vpc string) { + t.mu.Lock() + defer t.mu.Unlock() + delete(t.routes, vpc) + delete(t.withdrawn, vpc) +} + +// HasWork reports whether vpc has a route or an unconfirmed withdrawal. +func (t *Table) HasWork(vpc string) bool { + t.mu.Lock() + defer t.mu.Unlock() + if len(t.routes[vpc]) > 0 { + return true + } + for _, w := range t.withdrawn[vpc] { + if !w.Removed { + return true + } + } + return false +} + +// Covering returns the route of vpc with the longest prefix containing addr, +// and the withdrawal with the longest prefix containing it. +func (t *Table) Covering(vpc string, addr netip.Addr) (Route, bool, Withdrawal, bool) { + t.mu.Lock() + defer t.mu.Unlock() + var ( + route Route + haveR bool + withdr Withdrawal + haveW bool + addrUnm = addr.Unmap() + ) + for p, r := range t.routes[vpc] { + if p.Contains(addrUnm) && (!haveR || p.Bits() > route.Prefix.Bits()) { + route, haveR = r, true + } + } + for p, w := range t.withdrawn[vpc] { + if p.Contains(addrUnm) && (!haveW || p.Bits() > withdr.Prefix.Bits()) { + withdr, haveW = w, true + } + } + return route, haveR, withdr, haveW +} + +func lessPrefix(a, b netip.Prefix) bool { + if c := a.Addr().Compare(b.Addr()); c != 0 { + return c < 0 + } + return a.Bits() < b.Bits() +} diff --git a/internal/worker/routes/routes_test.go b/internal/worker/routes/routes_test.go new file mode 100644 index 00000000..17f8beb0 --- /dev/null +++ b/internal/worker/routes/routes_test.go @@ -0,0 +1,115 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package routes + +import ( + "net/netip" + "slices" + "testing" + "time" +) + +var ( + p1 = netip.MustParsePrefix("2001:db8:a::1/128") + p64 = netip.MustParsePrefix("2001:db8:a::/64") + sid1 = netip.MustParseAddr("2001:db8:0:2:e000:1::") + sid2 = netip.MustParseAddr("2001:db8:0:3:e000:1::") +) + +func TestApplyAndWithdraw(t *testing.T) { + tab := New() + r1 := Route{Key: Key{VPC: "a", Prefix: p1}, TableID: 10, SID: sid1} + if got := tab.Apply([]Route{r1, {Key: Key{VPC: "b", Prefix: p1}, TableID: 20, SID: sid2}}, nil); !slices.Equal( + got, []string{"a", "b"}) { + t.Fatalf("changed = %v", got) + } + if got := tab.Apply([]Route{r1}, nil); len(got) != 0 { + t.Errorf("repeating an upsert changed %v", got) + } + if got := tab.Apply(nil, []Key{{VPC: "a", Prefix: p1}}); !slices.Equal(got, []string{"a"}) { + t.Fatalf("changed = %v", got) + } + if len(tab.Routes("a")) != 0 || len(tab.Routes("b")) != 1 { + t.Fatalf("routes a=%v b=%v", tab.Routes("a"), tab.Routes("b")) + } + w := tab.Withdrawn("a") + if len(w) != 1 || w[0].Route != r1 || w[0].Removed { + t.Fatalf("withdrawn = %+v, want r1 unconfirmed", w) + } + if !tab.HasWork("a") { + t.Error("an unconfirmed withdrawal is no work") + } + tab.ConfirmRemoved([]Route{r1}) + if tab.HasWork("a") || !tab.Withdrawn("a")[0].Removed { + t.Error("confirmed withdrawal still pending") + } + if _, _, w, ok := tab.Covering("a", p1.Addr()); !ok || !w.Removed { + t.Error("confirmed withdrawal not reported as covering") + } + + // An upsert clears the withdrawal. + tab.Apply([]Route{r1}, nil) + if len(tab.Withdrawn("a")) != 0 { + t.Error("upsert kept the withdrawal") + } +} + +func TestWithdrawalOfAnUnknownRouteHasNoSID(t *testing.T) { + tab := New() + tab.Apply(nil, []Key{{VPC: "a", Prefix: p1}}) + w := tab.Withdrawn("a") + if len(w) != 1 || w[0].SID.IsValid() || w[0].TableID != 0 { + t.Fatalf("withdrawn = %+v, want one with no SID or table", w) + } +} + +func TestConfirmIgnoresAReplacedWithdrawal(t *testing.T) { + tab := New() + r1 := Route{Key: Key{VPC: "a", Prefix: p1}, TableID: 10, SID: sid1} + tab.Apply([]Route{r1}, nil) + tab.Apply(nil, []Key{r1.Key}) + // Re-learned toward another SID and withdrawn again before the first + // removal is confirmed. + r2 := r1 + r2.SID = sid2 + tab.Apply([]Route{r2}, []Key{r2.Key}) + tab.ConfirmRemoved([]Route{r1}) + if w := tab.Withdrawn("a"); len(w) != 1 || w[0].Removed || w[0].SID != sid2 { + t.Fatalf("withdrawn = %+v, want the second withdrawal unconfirmed", w) + } +} + +func TestCoveringIsLongestMatch(t *testing.T) { + tab := New() + tab.Apply([]Route{ + {Key: Key{VPC: "a", Prefix: p64}, TableID: 10, SID: sid1}, + {Key: Key{VPC: "a", Prefix: p1}, TableID: 10, SID: sid2}, + }, nil) + r, ok, _, _ := tab.Covering("a", p1.Addr()) + if !ok || r.SID != sid2 { + t.Errorf("Covering(%s) = %+v, want the /128", p1.Addr(), r) + } + r, ok, _, _ = tab.Covering("a", netip.MustParseAddr("2001:db8:a::9")) + if !ok || r.SID != sid1 { + t.Errorf("Covering(::9) = %+v, want the /64", r) + } + if _, ok, _, _ := tab.Covering("b", p1.Addr()); ok { + t.Error("route of VPC a covers VPC b") + } +} + +func TestConfirmedWithdrawalsExpire(t *testing.T) { + now := time.Unix(1_700_000_000, 0) + tab := New() + tab.now = func() time.Time { return now } + r1 := Route{Key: Key{VPC: "a", Prefix: p1}, TableID: 10, SID: sid1} + tab.Apply([]Route{r1}, []Key{r1.Key}) + tab.ConfirmRemoved([]Route{r1}) + now = now.Add(removedRetention + time.Second) + tab.Apply(nil, nil) + if w := tab.Withdrawn("a"); len(w) != 0 { + t.Fatalf("withdrawn = %+v, want the expired one dropped", w) + } +} diff --git a/internal/worker/routesapi.go b/internal/worker/routesapi.go new file mode 100644 index 00000000..251c415e --- /dev/null +++ b/internal/worker/routesapi.go @@ -0,0 +1,158 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package worker + +import ( + "errors" + "fmt" + "io" + "net/netip" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + + workerv1alpha1 "go.datum.net/galactic/api/galactic/worker/v1alpha1" + "go.datum.net/galactic/internal/worker/refusal" + "go.datum.net/galactic/internal/worker/routes" +) + +// routesServer serves the Routes service, which records the tenant routes +// galactic-router hands Worker. +type routesServer struct { + workerv1alpha1.UnimplementedRoutesServer + w *Worker +} + +// Sync implements workerv1alpha1.RoutesServer. It refuses a caller not +// allowed fabric.routes, records each request's routes, requeues the VPCs +// they change and answers with the routes it rejected. The open stream +// renews the caller's session. +func (s *routesServer) Sync(stream grpc.BidiStreamingServer[workerv1alpha1.RoutesSyncRequest, + workerv1alpha1.RoutesSyncResponse]) error { + w := s.w + ctx := stream.Context() + caller, err := w.begin(ctx) + if err != nil { + return err + } + if !caller.Allowed(workerv1alpha1.CapabilityFabricRoutes) { + return w.refuse(refusal.New(codes.PermissionDenied, workerv1alpha1.ReasonCapabilityNotAllowed, + caller.ServiceAccount+" may not use "+workerv1alpha1.CapabilityFabricRoutes, nil)) + } + if w.d.Routes == nil { + return w.refuse(refusal.New(codes.Unavailable, workerv1alpha1.ReasonModuleNotRunning, + "no running module serves "+workerv1alpha1.CapabilityFabricRoutes, nil)) + } + if w.shutting.Load() { + return w.refuse(errShuttingDown()) + } + if err := stream.SendHeader(metadata.MD{}); err != nil { + return err + } + + reqs := make(chan *workerv1alpha1.RoutesSyncRequest) + recvErr := make(chan error, 1) + go func() { + for { + req, err := stream.Recv() + if err != nil { + recvErr <- err + return + } + select { + case reqs <- req: + case <-ctx.Done(): + return + } + } + }() + + renew := time.NewTicker(w.cfg.RenewInterval) + defer renew.Stop() + for { + select { + case <-ctx.Done(): + return nil + case err := <-recvErr: + if errors.Is(err, io.EOF) { + return nil + } + return err + case <-renew.C: + if err := w.touchSession(ctx, caller); err != nil { + return err + } + case req := <-reqs: + if err := stream.Send(s.record(req)); err != nil { + return err + } + } + } +} + +// record parses one request, records the routes that parse and requeues the +// VPCs they change. +func (s *routesServer) record(req *workerv1alpha1.RoutesSyncRequest) *workerv1alpha1.RoutesSyncResponse { + resp := &workerv1alpha1.RoutesSyncResponse{Sequence: req.GetSequence()} + reject := func(k *workerv1alpha1.TenantRouteKey, err error) { + resp.Rejected = append(resp.Rejected, &workerv1alpha1.RejectedRoute{ + Key: k, Reason: workerv1alpha1.ReasonMalformedRoute, Message: err.Error(), + }) + } + var upserts []routes.Route + for _, r := range req.GetUpserts() { + key := &workerv1alpha1.TenantRouteKey{Vpc: r.GetVpc(), Prefix: r.GetPrefix()} + route, err := parseRoute(r) + if err != nil { + reject(key, err) + continue + } + upserts = append(upserts, route) + } + var withdrawals []routes.Key + for _, k := range req.GetWithdrawals() { + key, err := parseRouteKey(k.GetVpc(), k.GetPrefix()) + if err != nil { + reject(k, err) + continue + } + withdrawals = append(withdrawals, key) + } + for _, vpc := range s.w.d.Routes.Apply(upserts, withdrawals) { + s.w.RoutesChanged(vpc) + } + return resp +} + +func parseRouteKey(vpc, prefix string) (routes.Key, error) { + if vpc == "" { + return routes.Key{}, errors.New("vpc is empty") + } + p, err := netip.ParsePrefix(prefix) + if err != nil { + return routes.Key{}, fmt.Errorf("prefix: %w", err) + } + return routes.Key{VPC: vpc, Prefix: p.Masked()}, nil +} + +func parseRoute(r *workerv1alpha1.TenantRoute) (routes.Route, error) { + key, err := parseRouteKey(r.GetVpc(), r.GetPrefix()) + if err != nil { + return routes.Route{}, err + } + if r.GetTableId() == 0 { + return routes.Route{}, errors.New("table_id is 0") + } + sid, err := netip.ParseAddr(r.GetSid()) + if err != nil { + return routes.Route{}, fmt.Errorf("sid: %w", err) + } + if !sid.Is6() || sid.Is4In6() || sid.IsUnspecified() { + return routes.Route{}, fmt.Errorf("sid %s is not a specified IPv6 address", sid) + } + return routes.Route{Key: key, TableID: r.GetTableId(), SID: sid}, nil +} diff --git a/internal/worker/sources/sources.go b/internal/worker/sources/sources.go index 67b94a21..f4616381 100644 --- a/internal/worker/sources/sources.go +++ b/internal/worker/sources/sources.go @@ -42,6 +42,10 @@ type Attachment struct { // Admitted is false for an attachment being deleted, and for an // EdgeAttachment whose Admitted condition is not True. Admitted bool + // Addresses are the attachment's interface addresses from its spec, + // without their prefix lengths. A VPCAttachment whose guest manages its + // own addressing records none, and neither does an EdgeAttachment. + Addresses []netip.Addr } // Sources answers Worker's trusted-source lookups. @@ -86,7 +90,7 @@ func (k Kube) Attachment(ctx context.Context, uid string) (Attachment, bool, err if len(vas.Items) == 1 { a := vas.Items[0] return Attachment{UID: uid, Kind: KindVPCAttachment, VPC: a.Status.VPC, Node: a.Status.Node, - Admitted: a.DeletionTimestamp.IsZero()}, true, nil + Admitted: a.DeletionTimestamp.IsZero(), Addresses: interfaceAddresses(a.Spec.Interface.Addresses)}, true, nil } var eas galacticv1alpha1.EdgeAttachmentList if err := k.Reader.List(ctx, &eas, client.MatchingFields{uidIndex: uid}); err != nil { @@ -102,6 +106,21 @@ func (k Kube) Attachment(ctx context.Context, uid string) (Attachment, bool, err return Attachment{}, false, nil } +// interfaceAddresses returns the addresses of CIDR-notation interface +// addresses, skipping any that do not parse. Admission validates them, so a +// skipped one is a value admission would refuse. +func interfaceAddresses(in []cloudv1alpha1.IPAddress) []netip.Addr { + var out []netip.Addr + for _, a := range in { + if p, err := netip.ParsePrefix(string(a)); err == nil { + out = append(out, p.Addr().Unmap()) + } else if addr, err := netip.ParseAddr(string(a)); err == nil { + out = append(out, addr.Unmap()) + } + } + return out +} + // Locator implements Sources. func (k Kube) Locator(ctx context.Context, node string) (netip.Prefix, error) { return nodelocator.Prefix(ctx, k.Reader, k.Namespace, node) diff --git a/internal/worker/sources/sources_test.go b/internal/worker/sources/sources_test.go index 2f57ef76..cdf9784d 100644 --- a/internal/worker/sources/sources_test.go +++ b/internal/worker/sources/sources_test.go @@ -7,6 +7,7 @@ package sources import ( "context" "net/netip" + "reflect" "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,6 +35,9 @@ func TestKube(t *testing.T) { } byUID := func(o client.Object) []string { return []string{string(o.GetUID())} } va := &cloudv1alpha1.VPCAttachment{ObjectMeta: metav1.ObjectMeta{Name: "va", Namespace: "p", UID: uidVA}, + Spec: cloudv1alpha1.VPCAttachmentSpec{Interface: cloudv1alpha1.VPCAttachmentInterface{ + Addresses: []cloudv1alpha1.IPAddress{"2001:db8:10::5/64", "192.0.2.5/24"}, + }}, Status: cloudv1alpha1.VPCAttachmentStatus{VPC: vpcA, Node: nodeA}} admitted := &galacticv1alpha1.EdgeAttachment{ObjectMeta: metav1.ObjectMeta{Name: "ea", Namespace: "p", UID: uidEA}, Status: galacticv1alpha1.EdgeAttachmentStatus{VPC: vpcA, NodeName: nodeA, Conditions: []metav1.Condition{{ @@ -54,12 +58,13 @@ func TestKube(t *testing.T) { ctx := context.Background() for uid, want := range map[string]Attachment{ - uidVA: {UID: uidVA, Kind: KindVPCAttachment, VPC: vpcA, Node: nodeA, Admitted: true}, + uidVA: {UID: uidVA, Kind: KindVPCAttachment, VPC: vpcA, Node: nodeA, Admitted: true, + Addresses: []netip.Addr{netip.MustParseAddr("2001:db8:10::5"), netip.MustParseAddr("192.0.2.5")}}, uidEA: {UID: uidEA, Kind: KindEdgeAttachment, VPC: vpcA, Node: nodeA, Admitted: true}, uidEB: {UID: uidEB, Kind: KindEdgeAttachment, VPC: vpcA, Node: nodeA, Admitted: false}, } { got, ok, err := k.Attachment(ctx, uid) - if err != nil || !ok || got != want { + if err != nil || !ok || !reflect.DeepEqual(got, want) { t.Errorf("Attachment(%s) = %+v, %t, %v; want %+v", uid, got, ok, err, want) } } diff --git a/internal/worker/worker.go b/internal/worker/worker.go index f6b2424f..a8ef1391 100644 --- a/internal/worker/worker.go +++ b/internal/worker/worker.go @@ -35,6 +35,7 @@ import ( "go.datum.net/galactic/internal/worker/auth" "go.datum.net/galactic/internal/worker/hostscope" "go.datum.net/galactic/internal/worker/module" + "go.datum.net/galactic/internal/worker/routes" "go.datum.net/galactic/internal/worker/sources" "go.datum.net/galactic/internal/worker/store" ) @@ -121,7 +122,11 @@ type Deps struct { // its modules for scopes with claims. Host hostscope.Host Auth Authenticator - Log *slog.Logger + // Routes holds the tenant routes the Routes service records, which the + // Fabric module writes. Without it the Routes service refuses every + // stream with ModuleNotRunning. + Routes *routes.Table + Log *slog.Logger // Now is the clock, time.Now when nil. Now func() time.Time } @@ -306,6 +311,16 @@ func (w *Worker) HostChanged(vpc string) { w.queue.Add(vpc) } +// RoutesChanged requeues vpc's scope after the tenant routes recorded for it +// changed, or a datapath event touched the routes programmed for it. It +// starts tracking the scope when Worker did not know it. +func (w *Worker) RoutesChanged(vpc string) { + w.mu.Lock() + w.scope(vpc) + w.mu.Unlock() + w.queue.Add(vpc) +} + // discover starts tracking, and queues, every VPC with host state on the // node. func (w *Worker) discover(ctx context.Context) { diff --git a/pkg/workerclient/client.go b/pkg/workerclient/client.go index cb90123c..75d9d5a6 100644 --- a/pkg/workerclient/client.go +++ b/pkg/workerclient/client.go @@ -63,9 +63,10 @@ type Options struct { // Client is a Worker client. type Client struct { - opts Options - cc *grpc.ClientConn - api workerv1alpha1.WorkerClient + opts Options + cc *grpc.ClientConn + api workerv1alpha1.WorkerClient + routes workerv1alpha1.RoutesClient mu sync.Mutex session string @@ -115,7 +116,8 @@ func Dial(ctx context.Context, opts Options) (*Client, error) { if err != nil { return nil, fmt.Errorf("workerclient: %w", err) } - c := &Client{opts: opts, cc: cc, api: workerv1alpha1.NewWorkerClient(cc), stop: make(chan struct{})} + c := &Client{opts: opts, cc: cc, api: workerv1alpha1.NewWorkerClient(cc), + routes: workerv1alpha1.NewRoutesClient(cc), stop: make(chan struct{})} if err := c.openSession(ctx, false); err != nil { _ = cc.Close() return nil, err @@ -242,6 +244,39 @@ func (c *Client) Observe(ctx context.Context, key *workerv1alpha1.ClaimKey) (*Ob return &Observer{stream: stream, last: map[string]uint64{}}, nil } +// RouteStream is an open Routes.Sync stream. +type RouteStream = grpc.BidiStreamingClient[workerv1alpha1.RoutesSyncRequest, workerv1alpha1.RoutesSyncResponse] + +// SyncRoutes opens a Routes.Sync stream, which hands Worker tenant routes. +// Only a caller allowed fabric.routes may open one. Cancel ctx to close it. +func (c *Client) SyncRoutes(ctx context.Context) (RouteStream, error) { + var stream RouteStream + err := c.retry(ctx, func(ctx context.Context) error { + s, err := c.routes.Sync(ctx) + if err != nil { + return err + } + // Worker sends headers once it admits the stream; a stream that + // ends without them was refused. + md, err := s.Header() + if err != nil { + return err + } + if md == nil { + if _, err := s.Recv(); err != nil { + return err + } + return errors.New("workerclient: Routes.Sync stream ended without headers") + } + stream = s + return nil + }) + if err != nil { + return nil, err + } + return stream, nil +} + // Reason returns the contract reason of a Worker error, or "". func Reason(err error) string { st, ok := status.FromError(err)