From 19dc061a4801f719a69ca4e1bbb6fb841c674366 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 12 Nov 2026 18:57:53 -0600 Subject: [PATCH] fix: wake deployment and DNS publisher only on Instance changes they read An Instance and its network claim are written many times while it starts and stops. The WorkloadDeployment controller and the internal DNS publisher woke on every one of those writes, so a single workload restart caused a few dozen reconciles that did nothing, and restarting a workload about once a minute was enough to trip the reconcile storm alert. Filter Instance and NetworkInterfaceClaim update events down to the fields each reconciler reads. Creates, deletes and periodic resyncs still pass. --- internal/controller/internaldns_publisher.go | 7 +- internal/controller/watch_predicates.go | 148 ++++++++++++ internal/controller/watch_predicates_test.go | 227 ++++++++++++++++++ .../workloaddeployment_controller.go | 6 +- 4 files changed, 385 insertions(+), 3 deletions(-) create mode 100644 internal/controller/watch_predicates.go create mode 100644 internal/controller/watch_predicates_test.go diff --git a/internal/controller/internaldns_publisher.go b/internal/controller/internaldns_publisher.go index df48a630..b1ed7854 100644 --- a/internal/controller/internaldns_publisher.go +++ b/internal/controller/internaldns_publisher.go @@ -714,7 +714,12 @@ func (r *InternalDNSPublisherReconciler) SetupWithManager(mgr mcmanager.Manager) } return mcbuilder.ControllerManagedBy(mgr). Named(r.controllerName()). - For(&computev1alpha.Instance{}, mcbuilder.WithEngageWithLocalCluster(false)). + // Only Instance changes that affect publication wake it; the resync + // still bounds how long a missed change can leave it stale. + For(&computev1alpha.Instance{}, + mcbuilder.WithEngageWithLocalCluster(false), + mcbuilder.WithPredicates(instanceChangedForDNSPublication()), + ). Owns(newDNSObject(dnsRegistrationGVK, "", ""), mcbuilder.WithEngageWithLocalCluster(false)). Owns(newDNSObject(dnsContributionGVK, "", ""), mcbuilder.WithEngageWithLocalCluster(false)). Watches(newDNSObject(dnsResolverContextGVK, "", ""), func(clusterName multicluster.ClusterName, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] { diff --git a/internal/controller/watch_predicates.go b/internal/controller/watch_predicates.go new file mode 100644 index 00000000..df76ffd2 --- /dev/null +++ b/internal/controller/watch_predicates.go @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "k8s.io/apimachinery/pkg/api/equality" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + computev1alpha "go.datum.net/compute/api/v1alpha" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +// An Instance or claim is written many times while it starts: finalizers, +// interface status, runtime conditions, DNS status and more, each by a +// different controller. The watchers below only care about a few of those +// fields, so they wake on changes to those and let the rest go by. Creates, +// deletes and generic events always pass. +// +// Each predicate covers at least what its reconciler reads. When a reconciler +// starts reading another field, add it here, or that change will not wake it. +// Periodic cache resyncs (same resourceVersion) still pass, so a missed event +// is eventually picked up. + +// changeFilter builds an update-only predicate for one object type. +func changeFilter[T client.Object](changed func(old, new T) bool) predicate.Predicate { + return predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + old, okOld := e.ObjectOld.(T) + updated, okNew := e.ObjectNew.(T) + if !okOld || !okNew { + return true + } + return old.GetResourceVersion() == updated.GetResourceVersion() || + metadataChanged(old, updated) || changed(old, updated) + }, + } +} + +// metadataChanged covers what every watcher here reads from metadata: which +// object it is, its spec revision, its labels and owners, and whether it is +// being deleted. Finalizer and annotation bookkeeping is left out. +func metadataChanged(old, updated client.Object) bool { + return old.GetUID() != updated.GetUID() || + old.GetGeneration() != updated.GetGeneration() || + !equality.Semantic.DeepEqual(old.GetLabels(), updated.GetLabels()) || + !equality.Semantic.DeepEqual(old.GetOwnerReferences(), updated.GetOwnerReferences()) || + !equality.Semantic.DeepEqual(old.GetDeletionTimestamp(), updated.GetDeletionTimestamp()) +} + +// conditionStatusChanged reports whether the status of any of the given +// condition types differs. +func conditionStatusChanged(old, updated []metav1.Condition, types ...string) bool { + for _, conditionType := range types { + a := apimeta.FindStatusCondition(old, conditionType) + b := apimeta.FindStatusCondition(updated, conditionType) + if (a == nil) != (b == nil) || (a != nil && a.Status != b.Status) { + return true + } + } + return false +} + +// conditionChanged reports whether a condition's status, reason or message +// differs. +func conditionChanged(old, updated []metav1.Condition, conditionType string) bool { + a := apimeta.FindStatusCondition(old, conditionType) + b := apimeta.FindStatusCondition(updated, conditionType) + if a == nil || b == nil { + return (a == nil) != (b == nil) + } + return a.Status != b.Status || a.Reason != b.Reason || a.Message != b.Message +} + +// instanceChangedForDeployment passes the Instance updates the +// WorkloadDeployment reconciler acts on: the instance control actions, gates, +// suspension, revision and the condition rollup into deployment status. +func instanceChangedForDeployment() predicate.Predicate { + return changeFilter(func(old, updated *computev1alpha.Instance) bool { + return old.Status.Suspended != updated.Status.Suspended || + !equality.Semantic.DeepEqual(old.Status.Controller, updated.Status.Controller) || + conditionStatusChanged(old.Status.Conditions, updated.Status.Conditions, + computev1alpha.InstanceReady, + computev1alpha.InstanceProgrammed, + computev1alpha.InstanceQuotaGranted, + computev1alpha.ReferencedDataReady, + ) || + // The deployment reports the reason and message of a start failure. + conditionChanged(old.Status.Conditions, updated.Status.Conditions, computev1alpha.InstanceAvailable) + }) +} + +// networkInterfaceClaimChangedForDeployment passes the claim updates that can +// release an instance's Network gate or explain why it is held. +func networkInterfaceClaimChangedForDeployment() predicate.Predicate { + return changeFilter(func(old, updated *networkingv1alpha.NetworkInterfaceClaim) bool { + for _, conditionType := range []string{ + networkingv1alpha.NetworkInterfaceClaimBound, + networkingv1alpha.NetworkInterfaceClaimAllocated, + networkingv1alpha.NetworkInterfaceClaimPrepared, + } { + if conditionChanged(old.Status.Conditions, updated.Status.Conditions, conditionType) { + return true + } + } + return false + }) +} + +// instanceChangedForDNSPublication passes the Instance updates that can change +// what the internal DNS publisher writes: eligibility (Ready, RuntimeReady), +// the interfaces and their addresses, and the reported publication status, +// which it restores if another writer changes it. +func instanceChangedForDNSPublication() predicate.Predicate { + return changeFilter(func(old, updated *computev1alpha.Instance) bool { + return conditionStatusChanged(old.Status.Conditions, updated.Status.Conditions, + computev1alpha.InstanceReady, + computev1alpha.InstanceRuntimeReady, + ) || + dnsInterfacesChanged(old.Status.NetworkInterfaces, updated.Status.NetworkInterfaces) || + !equality.Semantic.DeepEqual(old.Status.DNS, updated.Status.DNS) + }) +} + +// dnsInterfacesChanged compares the parts of interface status the DNS +// publisher reads: which interface, its reference, whether it is allocated and +// programmed, and its addresses. +func dnsInterfacesChanged(old, updated []computev1alpha.InstanceNetworkInterfaceStatus) bool { + if len(old) != len(updated) { + return true + } + for i := range old { + a, b := old[i], updated[i] + if a.Name != b.Name || + !equality.Semantic.DeepEqual(a.NetworkInterfaceRef, b.NetworkInterfaceRef) || + !equality.Semantic.DeepEqual(a.Addresses, b.Addresses) || + conditionStatusChanged(a.Conditions, b.Conditions, + computev1alpha.InstanceNetworkInterfaceAllocated, + computev1alpha.InstanceNetworkInterfaceProgrammed, + ) { + return true + } + } + return false +} diff --git a/internal/controller/watch_predicates_test.go b/internal/controller/watch_predicates_test.go new file mode 100644 index 00000000..5ee73ae8 --- /dev/null +++ b/internal/controller/watch_predicates_test.go @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "testing" + + "github.com/stretchr/testify/assert" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + computev1alpha "go.datum.net/compute/api/v1alpha" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +const ( + caseNoChange = "no change" + caseFinalizerAdded = "finalizer added" + caseDeleting = "deleting" + msgStarting = "starting" +) + +func baseInstance() *computev1alpha.Instance { + return &computev1alpha.Instance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "web-0", Namespace: "ns", UID: "uid-1", Generation: 1, + Labels: map[string]string{"a": "b"}, + }, + Status: computev1alpha.InstanceStatus{ + Conditions: []metav1.Condition{ + {Type: computev1alpha.InstanceReady, Status: metav1.ConditionFalse, Reason: computev1alpha.InstanceReadyReasonProvisioning, Message: msgStarting}, + {Type: computev1alpha.InstanceAvailable, Status: metav1.ConditionFalse, Reason: "Starting", Message: msgStarting}, + {Type: computev1alpha.InstanceRuntimeReady, Status: metav1.ConditionFalse, Reason: claimTestPendingReason}, + }, + NetworkInterfaces: []computev1alpha.InstanceNetworkInterfaceStatus{{ + Name: defaultInterfaceName, + Conditions: []metav1.Condition{ + {Type: computev1alpha.InstanceNetworkInterfaceAllocated, Status: metav1.ConditionFalse, Reason: claimTestPendingReason}, + }, + }}, + }, + } +} + +func passesUpdate(p predicate.Predicate, old, updated client.Object) bool { + return p.Update(event.UpdateEvent{ObjectOld: old, ObjectNew: updated}) +} + +func TestInstanceChangedForDeployment(t *testing.T) { + p := instanceChangedForDeployment() + + cases := []struct { + name string + mutate func(*computev1alpha.Instance) + want bool + }{ + {caseNoChange, func(*computev1alpha.Instance) {}, false}, + {"resource version only", func(i *computev1alpha.Instance) { i.ResourceVersion = "2" }, false}, + {caseFinalizerAdded, func(i *computev1alpha.Instance) { i.Finalizers = []string{"x"} }, false}, + {"annotation added", func(i *computev1alpha.Instance) { i.Annotations = map[string]string{"x": "y"} }, false}, + {"interface status", func(i *computev1alpha.Instance) { + i.Status.NetworkInterfaces[0].Addresses = []computev1alpha.InstanceNetworkInterfaceAddress{{Address: "fd00::1"}} + }, false}, + {"dns status", func(i *computev1alpha.Instance) { + i.Status.DNS = []computev1alpha.InstanceDNSStatus{{}} + }, false}, + {"runtime ready", func(i *computev1alpha.Instance) { + apimeta.SetStatusCondition(&i.Status.Conditions, metav1.Condition{Type: computev1alpha.InstanceRuntimeReady, Status: metav1.ConditionTrue, Reason: "RuntimeReady"}) + }, false}, + {"ready message only", func(i *computev1alpha.Instance) { + apimeta.SetStatusCondition(&i.Status.Conditions, metav1.Condition{Type: computev1alpha.InstanceReady, Status: metav1.ConditionFalse, Reason: computev1alpha.InstanceReadyReasonProvisioning, Message: "still starting"}) + }, false}, + {"ready true", func(i *computev1alpha.Instance) { + apimeta.SetStatusCondition(&i.Status.Conditions, metav1.Condition{Type: computev1alpha.InstanceReady, Status: metav1.ConditionTrue, Reason: computev1alpha.InstanceReadyReasonAvailable}) + }, true}, + {"programmed appears", func(i *computev1alpha.Instance) { + apimeta.SetStatusCondition(&i.Status.Conditions, metav1.Condition{Type: computev1alpha.InstanceProgrammed, Status: metav1.ConditionTrue, Reason: "Programmed"}) + }, true}, + {"quota denied", func(i *computev1alpha.Instance) { + apimeta.SetStatusCondition(&i.Status.Conditions, metav1.Condition{Type: computev1alpha.InstanceQuotaGranted, Status: metav1.ConditionFalse, Reason: "QuotaExceeded"}) + }, true}, + {"referenced data blocked", func(i *computev1alpha.Instance) { + apimeta.SetStatusCondition(&i.Status.Conditions, metav1.Condition{Type: computev1alpha.ReferencedDataReady, Status: metav1.ConditionFalse, Reason: "AwaitingPropagation"}) + }, true}, + {"available reason", func(i *computev1alpha.Instance) { + apimeta.SetStatusCondition(&i.Status.Conditions, metav1.Condition{Type: computev1alpha.InstanceAvailable, Status: metav1.ConditionFalse, Reason: computev1alpha.InstanceReadyReasonInstanceCrashing, Message: msgStarting}) + }, true}, + {"available message", func(i *computev1alpha.Instance) { + apimeta.SetStatusCondition(&i.Status.Conditions, metav1.Condition{Type: computev1alpha.InstanceAvailable, Status: metav1.ConditionFalse, Reason: "Starting", Message: "crashed"}) + }, true}, + {"observed template hash", func(i *computev1alpha.Instance) { + i.Status.Controller = &computev1alpha.InstanceControllerStatus{ObservedTemplateHash: "h"} + }, true}, + {"suspended", func(i *computev1alpha.Instance) { i.Status.Suspended = true }, true}, + {"spec change", func(i *computev1alpha.Instance) { i.Generation = 2 }, true}, + {"label change", func(i *computev1alpha.Instance) { i.Labels["a"] = "c" }, true}, + {"owner change", func(i *computev1alpha.Instance) { + i.OwnerReferences = []metav1.OwnerReference{{UID: "wd"}} + }, true}, + {caseDeleting, func(i *computev1alpha.Instance) { now := metav1.Now(); i.DeletionTimestamp = &now }, true}, + {"replaced", func(i *computev1alpha.Instance) { i.UID = "uid-2" }, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + old := baseInstance() + updated := old.DeepCopy() + updated.ResourceVersion = "2" + tc.mutate(updated) + assert.Equal(t, tc.want, passesUpdate(p, old, updated)) + }) + } + + // A periodic resync redelivers the same object and must still pass. + assert.True(t, passesUpdate(p, baseInstance(), baseInstance())) + assert.True(t, p.Create(event.CreateEvent{Object: baseInstance()})) + assert.True(t, p.Delete(event.DeleteEvent{Object: baseInstance()})) +} + +func TestNetworkInterfaceClaimChangedForDeployment(t *testing.T) { + p := networkInterfaceClaimChangedForDeployment() + base := func() *networkingv1alpha.NetworkInterfaceClaim { + return &networkingv1alpha.NetworkInterfaceClaim{ + ObjectMeta: metav1.ObjectMeta{Name: "web-0-eth0", UID: "c1", Generation: 1}, + Status: networkingv1alpha.NetworkInterfaceClaimStatus{ + Conditions: []metav1.Condition{ + {Type: networkingv1alpha.NetworkInterfaceClaimBound, Status: metav1.ConditionFalse, Reason: claimTestPendingReason}, + }, + }, + } + } + + cases := []struct { + name string + mutate func(*networkingv1alpha.NetworkInterfaceClaim) + want bool + }{ + {caseNoChange, func(*networkingv1alpha.NetworkInterfaceClaim) {}, false}, + {caseFinalizerAdded, func(c *networkingv1alpha.NetworkInterfaceClaim) { c.Finalizers = []string{"x"} }, false}, + {"unrelated condition", func(c *networkingv1alpha.NetworkInterfaceClaim) { + apimeta.SetStatusCondition(&c.Status.Conditions, metav1.Condition{Type: "Other", Status: metav1.ConditionTrue, Reason: "X"}) + }, false}, + {"bound", func(c *networkingv1alpha.NetworkInterfaceClaim) { + apimeta.SetStatusCondition(&c.Status.Conditions, metav1.Condition{Type: networkingv1alpha.NetworkInterfaceClaimBound, Status: metav1.ConditionTrue, Reason: "Bound"}) + }, true}, + {"allocated appears", func(c *networkingv1alpha.NetworkInterfaceClaim) { + apimeta.SetStatusCondition(&c.Status.Conditions, metav1.Condition{Type: networkingv1alpha.NetworkInterfaceClaimAllocated, Status: metav1.ConditionTrue, Reason: "Allocated"}) + }, true}, + {"prepared rejected", func(c *networkingv1alpha.NetworkInterfaceClaim) { + apimeta.SetStatusCondition(&c.Status.Conditions, metav1.Condition{Type: networkingv1alpha.NetworkInterfaceClaimPrepared, Status: metav1.ConditionFalse, Reason: "Rejected"}) + }, true}, + {"rejection message", func(c *networkingv1alpha.NetworkInterfaceClaim) { + apimeta.SetStatusCondition(&c.Status.Conditions, metav1.Condition{Type: networkingv1alpha.NetworkInterfaceClaimBound, Status: metav1.ConditionFalse, Reason: claimTestPendingReason, Message: "no capacity"}) + }, true}, + {"label change", func(c *networkingv1alpha.NetworkInterfaceClaim) { c.Labels = map[string]string{"a": "b"} }, true}, + {caseDeleting, func(c *networkingv1alpha.NetworkInterfaceClaim) { now := metav1.Now(); c.DeletionTimestamp = &now }, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + old := base() + updated := old.DeepCopy() + updated.ResourceVersion = "2" + tc.mutate(updated) + assert.Equal(t, tc.want, passesUpdate(p, old, updated)) + }) + } +} + +func TestInstanceChangedForDNSPublication(t *testing.T) { + p := instanceChangedForDNSPublication() + + cases := []struct { + name string + mutate func(*computev1alpha.Instance) + want bool + }{ + {caseNoChange, func(*computev1alpha.Instance) {}, false}, + {caseFinalizerAdded, func(i *computev1alpha.Instance) { i.Finalizers = []string{"x"} }, false}, + {"programmed", func(i *computev1alpha.Instance) { + apimeta.SetStatusCondition(&i.Status.Conditions, metav1.Condition{Type: computev1alpha.InstanceProgrammed, Status: metav1.ConditionTrue, Reason: "Programmed"}) + }, false}, + {"ready message only", func(i *computev1alpha.Instance) { + apimeta.SetStatusCondition(&i.Status.Conditions, metav1.Condition{Type: computev1alpha.InstanceReady, Status: metav1.ConditionFalse, Reason: computev1alpha.InstanceReadyReasonProvisioning, Message: "still starting"}) + }, false}, + {"interface condition message", func(i *computev1alpha.Instance) { + i.Status.NetworkInterfaces[0].Conditions[0].Message = "waiting" + }, false}, + {"external address", func(i *computev1alpha.Instance) { + i.Status.NetworkInterfaces[0].ExternalAddresses = []computev1alpha.InstanceNetworkInterfaceExternalAddress{{}} + }, false}, + {"ready true", func(i *computev1alpha.Instance) { + apimeta.SetStatusCondition(&i.Status.Conditions, metav1.Condition{Type: computev1alpha.InstanceReady, Status: metav1.ConditionTrue, Reason: computev1alpha.InstanceReadyReasonAvailable}) + }, true}, + {"runtime ready", func(i *computev1alpha.Instance) { + apimeta.SetStatusCondition(&i.Status.Conditions, metav1.Condition{Type: computev1alpha.InstanceRuntimeReady, Status: metav1.ConditionTrue, Reason: "RuntimeReady"}) + }, true}, + {"interface allocated", func(i *computev1alpha.Instance) { + i.Status.NetworkInterfaces[0].Conditions[0].Status = metav1.ConditionTrue + }, true}, + {"interface address", func(i *computev1alpha.Instance) { + i.Status.NetworkInterfaces[0].Addresses = []computev1alpha.InstanceNetworkInterfaceAddress{{Address: "fd00::1"}} + }, true}, + {"interface ref", func(i *computev1alpha.Instance) { + i.Status.NetworkInterfaces[0].NetworkInterfaceRef = &networkingv1alpha.LocalNetworkInterfaceRef{Name: "nic"} + }, true}, + {"interface added", func(i *computev1alpha.Instance) { + i.Status.NetworkInterfaces = append(i.Status.NetworkInterfaces, computev1alpha.InstanceNetworkInterfaceStatus{Name: "secondary"}) + }, true}, + {"dns status", func(i *computev1alpha.Instance) { + i.Status.DNS = []computev1alpha.InstanceDNSStatus{{}} + }, true}, + {"annotation", func(i *computev1alpha.Instance) { i.Annotations = map[string]string{"x": "y"} }, false}, + {"label", func(i *computev1alpha.Instance) { i.Labels["a"] = "c" }, true}, + {caseDeleting, func(i *computev1alpha.Instance) { now := metav1.Now(); i.DeletionTimestamp = &now }, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + old := baseInstance() + updated := old.DeepCopy() + updated.ResourceVersion = "2" + tc.mutate(updated) + assert.Equal(t, tc.want, passesUpdate(p, old, updated)) + }) + } +} diff --git a/internal/controller/workloaddeployment_controller.go b/internal/controller/workloaddeployment_controller.go index 2b1aec80..dea46520 100644 --- a/internal/controller/workloaddeployment_controller.go +++ b/internal/controller/workloaddeployment_controller.go @@ -1091,7 +1091,9 @@ func (r *WorkloadDeploymentReconciler) SetupWithManager(mgr mcmanager.Manager, o mcbuilder.WithEngageWithLocalCluster(false), mcbuilder.WithPredicates(wdReferencedDataChangedPredicate()), ). - Owns(&computev1alpha.Instance{}) + // An Instance is written many times while it starts and stops. Only the + // changes this reconciler reads wake it. + Owns(&computev1alpha.Instance{}, mcbuilder.WithPredicates(instanceChangedForDeployment())) // Only watch networking resources when the networking integration is enabled. // On cells without network-services-operator these watches would log spurious @@ -1115,7 +1117,7 @@ func (r *WorkloadDeploymentReconciler) SetupWithManager(mgr mcmanager.Manager, o return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []mcreconcile.Request { return enqueueWorkloadDeploymentForClaim(ctx, cl.GetClient(), clusterName, o) }) - }). + }, mcbuilder.WithPredicates(networkInterfaceClaimChangedForDeployment())). // A deployment on a cell that does not yet know its own location waits // without any other wake-up event, and the reconciler does not poll. // Watching ServingLocations re-reconciles those deployments as soon as