diff --git a/internal/controller/internaldns_publisher.go b/internal/controller/internaldns_publisher.go index df48a630..b1ed7854 100644 --- a/internal/controller/internaldns_publisher.go +++ b/internal/controller/internaldns_publisher.go @@ -714,7 +714,12 @@ func (r *InternalDNSPublisherReconciler) SetupWithManager(mgr mcmanager.Manager) } return mcbuilder.ControllerManagedBy(mgr). Named(r.controllerName()). - For(&computev1alpha.Instance{}, mcbuilder.WithEngageWithLocalCluster(false)). + // Only Instance changes that affect publication wake it; the resync + // still bounds how long a missed change can leave it stale. + For(&computev1alpha.Instance{}, + mcbuilder.WithEngageWithLocalCluster(false), + mcbuilder.WithPredicates(instanceChangedForDNSPublication()), + ). Owns(newDNSObject(dnsRegistrationGVK, "", ""), mcbuilder.WithEngageWithLocalCluster(false)). Owns(newDNSObject(dnsContributionGVK, "", ""), mcbuilder.WithEngageWithLocalCluster(false)). Watches(newDNSObject(dnsResolverContextGVK, "", ""), func(clusterName multicluster.ClusterName, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] { diff --git a/internal/controller/watch_predicates.go b/internal/controller/watch_predicates.go new file mode 100644 index 00000000..df76ffd2 --- /dev/null +++ b/internal/controller/watch_predicates.go @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "k8s.io/apimachinery/pkg/api/equality" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + computev1alpha "go.datum.net/compute/api/v1alpha" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +// An Instance or claim is written many times while it starts: finalizers, +// interface status, runtime conditions, DNS status and more, each by a +// different controller. The watchers below only care about a few of those +// fields, so they wake on changes to those and let the rest go by. Creates, +// deletes and generic events always pass. +// +// Each predicate covers at least what its reconciler reads. When a reconciler +// starts reading another field, add it here, or that change will not wake it. +// Periodic cache resyncs (same resourceVersion) still pass, so a missed event +// is eventually picked up. + +// changeFilter builds an update-only predicate for one object type. +func changeFilter[T client.Object](changed func(old, new T) bool) predicate.Predicate { + return predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + old, okOld := e.ObjectOld.(T) + updated, okNew := e.ObjectNew.(T) + if !okOld || !okNew { + return true + } + return old.GetResourceVersion() == updated.GetResourceVersion() || + metadataChanged(old, updated) || changed(old, updated) + }, + } +} + +// metadataChanged covers what every watcher here reads from metadata: which +// object it is, its spec revision, its labels and owners, and whether it is +// being deleted. Finalizer and annotation bookkeeping is left out. +func metadataChanged(old, updated client.Object) bool { + return old.GetUID() != updated.GetUID() || + old.GetGeneration() != updated.GetGeneration() || + !equality.Semantic.DeepEqual(old.GetLabels(), updated.GetLabels()) || + !equality.Semantic.DeepEqual(old.GetOwnerReferences(), updated.GetOwnerReferences()) || + !equality.Semantic.DeepEqual(old.GetDeletionTimestamp(), updated.GetDeletionTimestamp()) +} + +// conditionStatusChanged reports whether the status of any of the given +// condition types differs. +func conditionStatusChanged(old, updated []metav1.Condition, types ...string) bool { + for _, conditionType := range types { + a := apimeta.FindStatusCondition(old, conditionType) + b := apimeta.FindStatusCondition(updated, conditionType) + if (a == nil) != (b == nil) || (a != nil && a.Status != b.Status) { + return true + } + } + return false +} + +// conditionChanged reports whether a condition's status, reason or message +// differs. +func conditionChanged(old, updated []metav1.Condition, conditionType string) bool { + a := apimeta.FindStatusCondition(old, conditionType) + b := apimeta.FindStatusCondition(updated, conditionType) + if a == nil || b == nil { + return (a == nil) != (b == nil) + } + return a.Status != b.Status || a.Reason != b.Reason || a.Message != b.Message +} + +// instanceChangedForDeployment passes the Instance updates the +// WorkloadDeployment reconciler acts on: the instance control actions, gates, +// suspension, revision and the condition rollup into deployment status. +func instanceChangedForDeployment() predicate.Predicate { + return changeFilter(func(old, updated *computev1alpha.Instance) bool { + return old.Status.Suspended != updated.Status.Suspended || + !equality.Semantic.DeepEqual(old.Status.Controller, updated.Status.Controller) || + conditionStatusChanged(old.Status.Conditions, updated.Status.Conditions, + computev1alpha.InstanceReady, + computev1alpha.InstanceProgrammed, + computev1alpha.InstanceQuotaGranted, + computev1alpha.ReferencedDataReady, + ) || + // The deployment reports the reason and message of a start failure. + conditionChanged(old.Status.Conditions, updated.Status.Conditions, computev1alpha.InstanceAvailable) + }) +} + +// networkInterfaceClaimChangedForDeployment passes the claim updates that can +// release an instance's Network gate or explain why it is held. +func networkInterfaceClaimChangedForDeployment() predicate.Predicate { + return changeFilter(func(old, updated *networkingv1alpha.NetworkInterfaceClaim) bool { + for _, conditionType := range []string{ + networkingv1alpha.NetworkInterfaceClaimBound, + networkingv1alpha.NetworkInterfaceClaimAllocated, + networkingv1alpha.NetworkInterfaceClaimPrepared, + } { + if conditionChanged(old.Status.Conditions, updated.Status.Conditions, conditionType) { + return true + } + } + return false + }) +} + +// instanceChangedForDNSPublication passes the Instance updates that can change +// what the internal DNS publisher writes: eligibility (Ready, RuntimeReady), +// the interfaces and their addresses, and the reported publication status, +// which it restores if another writer changes it. +func instanceChangedForDNSPublication() predicate.Predicate { + return changeFilter(func(old, updated *computev1alpha.Instance) bool { + return conditionStatusChanged(old.Status.Conditions, updated.Status.Conditions, + computev1alpha.InstanceReady, + computev1alpha.InstanceRuntimeReady, + ) || + dnsInterfacesChanged(old.Status.NetworkInterfaces, updated.Status.NetworkInterfaces) || + !equality.Semantic.DeepEqual(old.Status.DNS, updated.Status.DNS) + }) +} + +// dnsInterfacesChanged compares the parts of interface status the DNS +// publisher reads: which interface, its reference, whether it is allocated and +// programmed, and its addresses. +func dnsInterfacesChanged(old, updated []computev1alpha.InstanceNetworkInterfaceStatus) bool { + if len(old) != len(updated) { + return true + } + for i := range old { + a, b := old[i], updated[i] + if a.Name != b.Name || + !equality.Semantic.DeepEqual(a.NetworkInterfaceRef, b.NetworkInterfaceRef) || + !equality.Semantic.DeepEqual(a.Addresses, b.Addresses) || + conditionStatusChanged(a.Conditions, b.Conditions, + computev1alpha.InstanceNetworkInterfaceAllocated, + computev1alpha.InstanceNetworkInterfaceProgrammed, + ) { + return true + } + } + return false +} diff --git a/internal/controller/watch_predicates_test.go b/internal/controller/watch_predicates_test.go new file mode 100644 index 00000000..5ee73ae8 --- /dev/null +++ b/internal/controller/watch_predicates_test.go @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "testing" + + "github.com/stretchr/testify/assert" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + computev1alpha "go.datum.net/compute/api/v1alpha" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +const ( + caseNoChange = "no change" + caseFinalizerAdded = "finalizer added" + caseDeleting = "deleting" + msgStarting = "starting" +) + +func baseInstance() *computev1alpha.Instance { + return &computev1alpha.Instance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "web-0", Namespace: "ns", UID: "uid-1", Generation: 1, + Labels: map[string]string{"a": "b"}, + }, + Status: computev1alpha.InstanceStatus{ + Conditions: []metav1.Condition{ + {Type: computev1alpha.InstanceReady, Status: metav1.ConditionFalse, Reason: computev1alpha.InstanceReadyReasonProvisioning, Message: msgStarting}, + {Type: computev1alpha.InstanceAvailable, Status: metav1.ConditionFalse, Reason: "Starting", Message: msgStarting}, + {Type: computev1alpha.InstanceRuntimeReady, Status: metav1.ConditionFalse, Reason: claimTestPendingReason}, + }, + NetworkInterfaces: []computev1alpha.InstanceNetworkInterfaceStatus{{ + Name: defaultInterfaceName, + Conditions: []metav1.Condition{ + {Type: computev1alpha.InstanceNetworkInterfaceAllocated, Status: metav1.ConditionFalse, Reason: claimTestPendingReason}, + }, + }}, + }, + } +} + +func passesUpdate(p predicate.Predicate, old, updated client.Object) bool { + return p.Update(event.UpdateEvent{ObjectOld: old, ObjectNew: updated}) +} + +func TestInstanceChangedForDeployment(t *testing.T) { + p := instanceChangedForDeployment() + + cases := []struct { + name string + mutate func(*computev1alpha.Instance) + want bool + }{ + {caseNoChange, func(*computev1alpha.Instance) {}, false}, + {"resource version only", func(i *computev1alpha.Instance) { i.ResourceVersion = "2" }, false}, + {caseFinalizerAdded, func(i *computev1alpha.Instance) { i.Finalizers = []string{"x"} }, false}, + {"annotation added", func(i *computev1alpha.Instance) { i.Annotations = map[string]string{"x": "y"} }, false}, + {"interface status", func(i *computev1alpha.Instance) { + i.Status.NetworkInterfaces[0].Addresses = []computev1alpha.InstanceNetworkInterfaceAddress{{Address: "fd00::1"}} + }, false}, + {"dns status", func(i *computev1alpha.Instance) { + i.Status.DNS = []computev1alpha.InstanceDNSStatus{{}} + }, false}, + {"runtime ready", func(i *computev1alpha.Instance) { + apimeta.SetStatusCondition(&i.Status.Conditions, metav1.Condition{Type: computev1alpha.InstanceRuntimeReady, Status: metav1.ConditionTrue, Reason: "RuntimeReady"}) + }, false}, + {"ready message only", func(i *computev1alpha.Instance) { + apimeta.SetStatusCondition(&i.Status.Conditions, metav1.Condition{Type: computev1alpha.InstanceReady, Status: metav1.ConditionFalse, Reason: computev1alpha.InstanceReadyReasonProvisioning, Message: "still starting"}) + }, false}, + {"ready true", func(i *computev1alpha.Instance) { + apimeta.SetStatusCondition(&i.Status.Conditions, metav1.Condition{Type: computev1alpha.InstanceReady, Status: metav1.ConditionTrue, Reason: computev1alpha.InstanceReadyReasonAvailable}) + }, true}, + {"programmed appears", func(i *computev1alpha.Instance) { + apimeta.SetStatusCondition(&i.Status.Conditions, metav1.Condition{Type: computev1alpha.InstanceProgrammed, Status: metav1.ConditionTrue, Reason: "Programmed"}) + }, true}, + {"quota denied", func(i *computev1alpha.Instance) { + apimeta.SetStatusCondition(&i.Status.Conditions, metav1.Condition{Type: computev1alpha.InstanceQuotaGranted, Status: metav1.ConditionFalse, Reason: "QuotaExceeded"}) + }, true}, + {"referenced data blocked", func(i *computev1alpha.Instance) { + apimeta.SetStatusCondition(&i.Status.Conditions, metav1.Condition{Type: computev1alpha.ReferencedDataReady, Status: metav1.ConditionFalse, Reason: "AwaitingPropagation"}) + }, true}, + {"available reason", func(i *computev1alpha.Instance) { + apimeta.SetStatusCondition(&i.Status.Conditions, metav1.Condition{Type: computev1alpha.InstanceAvailable, Status: metav1.ConditionFalse, Reason: computev1alpha.InstanceReadyReasonInstanceCrashing, Message: msgStarting}) + }, true}, + {"available message", func(i *computev1alpha.Instance) { + apimeta.SetStatusCondition(&i.Status.Conditions, metav1.Condition{Type: computev1alpha.InstanceAvailable, Status: metav1.ConditionFalse, Reason: "Starting", Message: "crashed"}) + }, true}, + {"observed template hash", func(i *computev1alpha.Instance) { + i.Status.Controller = &computev1alpha.InstanceControllerStatus{ObservedTemplateHash: "h"} + }, true}, + {"suspended", func(i *computev1alpha.Instance) { i.Status.Suspended = true }, true}, + {"spec change", func(i *computev1alpha.Instance) { i.Generation = 2 }, true}, + {"label change", func(i *computev1alpha.Instance) { i.Labels["a"] = "c" }, true}, + {"owner change", func(i *computev1alpha.Instance) { + i.OwnerReferences = []metav1.OwnerReference{{UID: "wd"}} + }, true}, + {caseDeleting, func(i *computev1alpha.Instance) { now := metav1.Now(); i.DeletionTimestamp = &now }, true}, + {"replaced", func(i *computev1alpha.Instance) { i.UID = "uid-2" }, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + old := baseInstance() + updated := old.DeepCopy() + updated.ResourceVersion = "2" + tc.mutate(updated) + assert.Equal(t, tc.want, passesUpdate(p, old, updated)) + }) + } + + // A periodic resync redelivers the same object and must still pass. + assert.True(t, passesUpdate(p, baseInstance(), baseInstance())) + assert.True(t, p.Create(event.CreateEvent{Object: baseInstance()})) + assert.True(t, p.Delete(event.DeleteEvent{Object: baseInstance()})) +} + +func TestNetworkInterfaceClaimChangedForDeployment(t *testing.T) { + p := networkInterfaceClaimChangedForDeployment() + base := func() *networkingv1alpha.NetworkInterfaceClaim { + return &networkingv1alpha.NetworkInterfaceClaim{ + ObjectMeta: metav1.ObjectMeta{Name: "web-0-eth0", UID: "c1", Generation: 1}, + Status: networkingv1alpha.NetworkInterfaceClaimStatus{ + Conditions: []metav1.Condition{ + {Type: networkingv1alpha.NetworkInterfaceClaimBound, Status: metav1.ConditionFalse, Reason: claimTestPendingReason}, + }, + }, + } + } + + cases := []struct { + name string + mutate func(*networkingv1alpha.NetworkInterfaceClaim) + want bool + }{ + {caseNoChange, func(*networkingv1alpha.NetworkInterfaceClaim) {}, false}, + {caseFinalizerAdded, func(c *networkingv1alpha.NetworkInterfaceClaim) { c.Finalizers = []string{"x"} }, false}, + {"unrelated condition", func(c *networkingv1alpha.NetworkInterfaceClaim) { + apimeta.SetStatusCondition(&c.Status.Conditions, metav1.Condition{Type: "Other", Status: metav1.ConditionTrue, Reason: "X"}) + }, false}, + {"bound", func(c *networkingv1alpha.NetworkInterfaceClaim) { + apimeta.SetStatusCondition(&c.Status.Conditions, metav1.Condition{Type: networkingv1alpha.NetworkInterfaceClaimBound, Status: metav1.ConditionTrue, Reason: "Bound"}) + }, true}, + {"allocated appears", func(c *networkingv1alpha.NetworkInterfaceClaim) { + apimeta.SetStatusCondition(&c.Status.Conditions, metav1.Condition{Type: networkingv1alpha.NetworkInterfaceClaimAllocated, Status: metav1.ConditionTrue, Reason: "Allocated"}) + }, true}, + {"prepared rejected", func(c *networkingv1alpha.NetworkInterfaceClaim) { + apimeta.SetStatusCondition(&c.Status.Conditions, metav1.Condition{Type: networkingv1alpha.NetworkInterfaceClaimPrepared, Status: metav1.ConditionFalse, Reason: "Rejected"}) + }, true}, + {"rejection message", func(c *networkingv1alpha.NetworkInterfaceClaim) { + apimeta.SetStatusCondition(&c.Status.Conditions, metav1.Condition{Type: networkingv1alpha.NetworkInterfaceClaimBound, Status: metav1.ConditionFalse, Reason: claimTestPendingReason, Message: "no capacity"}) + }, true}, + {"label change", func(c *networkingv1alpha.NetworkInterfaceClaim) { c.Labels = map[string]string{"a": "b"} }, true}, + {caseDeleting, func(c *networkingv1alpha.NetworkInterfaceClaim) { now := metav1.Now(); c.DeletionTimestamp = &now }, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + old := base() + updated := old.DeepCopy() + updated.ResourceVersion = "2" + tc.mutate(updated) + assert.Equal(t, tc.want, passesUpdate(p, old, updated)) + }) + } +} + +func TestInstanceChangedForDNSPublication(t *testing.T) { + p := instanceChangedForDNSPublication() + + cases := []struct { + name string + mutate func(*computev1alpha.Instance) + want bool + }{ + {caseNoChange, func(*computev1alpha.Instance) {}, false}, + {caseFinalizerAdded, func(i *computev1alpha.Instance) { i.Finalizers = []string{"x"} }, false}, + {"programmed", func(i *computev1alpha.Instance) { + apimeta.SetStatusCondition(&i.Status.Conditions, metav1.Condition{Type: computev1alpha.InstanceProgrammed, Status: metav1.ConditionTrue, Reason: "Programmed"}) + }, false}, + {"ready message only", func(i *computev1alpha.Instance) { + apimeta.SetStatusCondition(&i.Status.Conditions, metav1.Condition{Type: computev1alpha.InstanceReady, Status: metav1.ConditionFalse, Reason: computev1alpha.InstanceReadyReasonProvisioning, Message: "still starting"}) + }, false}, + {"interface condition message", func(i *computev1alpha.Instance) { + i.Status.NetworkInterfaces[0].Conditions[0].Message = "waiting" + }, false}, + {"external address", func(i *computev1alpha.Instance) { + i.Status.NetworkInterfaces[0].ExternalAddresses = []computev1alpha.InstanceNetworkInterfaceExternalAddress{{}} + }, false}, + {"ready true", func(i *computev1alpha.Instance) { + apimeta.SetStatusCondition(&i.Status.Conditions, metav1.Condition{Type: computev1alpha.InstanceReady, Status: metav1.ConditionTrue, Reason: computev1alpha.InstanceReadyReasonAvailable}) + }, true}, + {"runtime ready", func(i *computev1alpha.Instance) { + apimeta.SetStatusCondition(&i.Status.Conditions, metav1.Condition{Type: computev1alpha.InstanceRuntimeReady, Status: metav1.ConditionTrue, Reason: "RuntimeReady"}) + }, true}, + {"interface allocated", func(i *computev1alpha.Instance) { + i.Status.NetworkInterfaces[0].Conditions[0].Status = metav1.ConditionTrue + }, true}, + {"interface address", func(i *computev1alpha.Instance) { + i.Status.NetworkInterfaces[0].Addresses = []computev1alpha.InstanceNetworkInterfaceAddress{{Address: "fd00::1"}} + }, true}, + {"interface ref", func(i *computev1alpha.Instance) { + i.Status.NetworkInterfaces[0].NetworkInterfaceRef = &networkingv1alpha.LocalNetworkInterfaceRef{Name: "nic"} + }, true}, + {"interface added", func(i *computev1alpha.Instance) { + i.Status.NetworkInterfaces = append(i.Status.NetworkInterfaces, computev1alpha.InstanceNetworkInterfaceStatus{Name: "secondary"}) + }, true}, + {"dns status", func(i *computev1alpha.Instance) { + i.Status.DNS = []computev1alpha.InstanceDNSStatus{{}} + }, true}, + {"annotation", func(i *computev1alpha.Instance) { i.Annotations = map[string]string{"x": "y"} }, false}, + {"label", func(i *computev1alpha.Instance) { i.Labels["a"] = "c" }, true}, + {caseDeleting, func(i *computev1alpha.Instance) { now := metav1.Now(); i.DeletionTimestamp = &now }, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + old := baseInstance() + updated := old.DeepCopy() + updated.ResourceVersion = "2" + tc.mutate(updated) + assert.Equal(t, tc.want, passesUpdate(p, old, updated)) + }) + } +} diff --git a/internal/controller/workloaddeployment_controller.go b/internal/controller/workloaddeployment_controller.go index 2b1aec80..dea46520 100644 --- a/internal/controller/workloaddeployment_controller.go +++ b/internal/controller/workloaddeployment_controller.go @@ -1091,7 +1091,9 @@ func (r *WorkloadDeploymentReconciler) SetupWithManager(mgr mcmanager.Manager, o mcbuilder.WithEngageWithLocalCluster(false), mcbuilder.WithPredicates(wdReferencedDataChangedPredicate()), ). - Owns(&computev1alpha.Instance{}) + // An Instance is written many times while it starts and stops. Only the + // changes this reconciler reads wake it. + Owns(&computev1alpha.Instance{}, mcbuilder.WithPredicates(instanceChangedForDeployment())) // Only watch networking resources when the networking integration is enabled. // On cells without network-services-operator these watches would log spurious @@ -1115,7 +1117,7 @@ func (r *WorkloadDeploymentReconciler) SetupWithManager(mgr mcmanager.Manager, o return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []mcreconcile.Request { return enqueueWorkloadDeploymentForClaim(ctx, cl.GetClient(), clusterName, o) }) - }). + }, mcbuilder.WithPredicates(networkInterfaceClaimChangedForDeployment())). // A deployment on a cell that does not yet know its own location waits // without any other wake-up event, and the reconciler does not poll. // Watching ServingLocations re-reconciles those deployments as soon as